Firecrawl Webhooks Events

Firecrawl Webhooks & Events

Overview

Handle Firecrawl webhooks for real-time notifications on async crawl and batch scrape jobs. Instead of polling checkCrawlStatus, configure a webhook URL and Firecrawl will POST events as pages are scraped and jobs complete. Signed with HMAC-SHA256 via X-Firecrawl-Signature.

Prerequisites

• A signing secret delivered through the secret manager and a handler that preserves the raw request for verification.
• An event ledger for idempotency, reviewed exception queue, approved destinations, and synthetic signed fixtures.
• Redaction rules preventing page content and credentials from entering event logs.

Output

Return a processing receipt with opaque event ID, signature result, handler version, idempotency result, destination state, and redacted error category. Store any permitted content only in the approved destination.

Webhook Event Types

Instructions

Step 1: Start Crawl with Webhook

import FirecrawlApp from "@mendable/firecrawl-js";

const firecrawl = new FirecrawlApp({
  apiKey: process.env.FIRECRAWL_API_KEY!,
});

// Webhook as string (simple)
const job = await firecrawl.asyncCrawlUrl("https://docs.example.com", {
  limit: 100,
  scrapeOptions: { formats: ["markdown"] },
  webhook: "https://api.yourapp.com/webhooks/firecrawl",
});

console.log(`Crawl started: ${job.id}`);

// Webhook as object (with metadata and event filtering)
const job2 = await firecrawl.asyncCrawlUrl("https://docs.example.com", {
  limit: 100,
  scrapeOptions: { formats: ["markdown"] },
  webhook: {
    url: "https://api.yourapp.com/webhooks/firecrawl",
    events: ["completed", "page"],  // only these events
    metadata: {
      projectId: "my-project",
      triggeredBy: "cron",
    },
  },
});

Step 2: Webhook Handler with Signature Verification

import express from "express";
import crypto from "crypto";

const app = express();
app.use(express.json());

function verifySignature(body: string, signature: string): boolean {
  if (!process.env.FIRECRAWL_WEBHOOK_SECRET) return true; // skip if not configured
  const expected = crypto
    .createHmac("sha256", process.env.FIRECRAWL_WEBHOOK_SECRET)
    .update(body)
    .digest("hex");
  return crypto.timingSafeEqual(Buffer.from(signature), Buffer.from(expected));
}

app.post("/webhooks/firecrawl", express.raw({ type: "application/json" }), async (req, res) => {
  const rawBody = req.body.toString();
  const signature = req.headers["x-firecrawl-signature"] as string;

  if (!verifySignature(rawBody, signature)) {
    return res.status(401).json({ error: "Invalid signature" });
  }

  const { type, id, data, metadata } = JSON.parse(rawBody);

  // Respond immediately — process asynchronously
  res.status(200).json({ received: true });

  switch (type) {
    case "crawl.started":
      console.log(`Crawl ${id} started`);
      break;
    case "crawl.page":
      await handlePageScraped(id, data, metadata);
      break;
    case "crawl.completed":
      await handleCrawlComplete(id, data, metadata);
      break;
    case "crawl.failed":
      await handleCrawlFailed(id, data);
      break;
  }
});

Step 3: Process Page Events (Streaming)

async function handlePageScraped(jobId: string, data: any[], metadata: any) {
  for (const page of data) {
    const doc = {
      url: page.metadata?.sourceURL,
      title: page.metadata?.title,
      markdown: page.markdown,
      statusCode: page.metadata?.statusCode,
      crawlJobId: jobId,
      projectId: metadata?.projectId,
      indexedAt: new Date(),
    };

    // Index page immediately — don't wait for full crawl
    await documentStore.upsert(doc);
    console.log(`Indexed: ${doc.url} (${doc.markdown?.length || 0} chars)`);
  }
}

Step 4: Handle Crawl Completion

async function handleCrawlComplete(jobId: string, data: any[], metadata: any) {
  console.log(`Crawl ${jobId} complete: ${data.length} pages`);

  // Build search index from all crawled pages
  const documents = data
    .filter(page => page.markdown && page.markdown.length > 100)
    .map(page => ({
      id: page.metadata?.sourceURL,
      title: page.metadata?.title || "",
      content: page.markdown,
      url: page.metadata?.sourceURL,
    }));

  await searchIndex.indexBatch(documents);
  console.log(`Indexed ${documents.length} documents for project ${metadata?.projectId}`);
}

async function handleCrawlFailed(jobId: string, data: any) {
  console.error(`Crawl ${jobId} failed:`, data.error);

  await alerting.send({
    severity: "high",
    message: `Firecrawl crawl job ${jobId} failed`,
    error: data.error,
    partialResults: data.partialResults?.length || 0,
  });
}

Step 5: Polling as Webhook Fallback

// Fall back to polling if webhook delivery fails
async function pollWithFallback(jobId: string, timeoutMs = 600000) {
  const deadline = Date.now() + timeoutMs;
  let interval = 2000;

  while (Date.now() < deadline) {
    const status = await firecrawl.checkCrawlStatus(jobId);

    if (status.status === "completed") {
      return status.data;
    }
    if (status.status === "failed") {
      throw new Error(`Crawl failed: ${status.error}`);
    }

    console.log(`Polling: ${status.completed}/${status.total} pages`);
    await new Promise(r => setTimeout(r, interval));
    interval = Math.min(interval * 1.5, 30000);
  }

  throw new Error(`Crawl timed out after ${timeoutMs}ms`);
}

Error Handling

Examples

Local Development with ngrok

set -euo pipefail
# Start ngrok tunnel for local webhook testing
ngrok http 3000
# Use the ngrok URL as your webhook endpoint
# https://abc123.ngrok.io/webhooks/firecrawl

Resources

• Firecrawl Webhooks
• Webhook Event Types
• Crawl Endpoint

Next Steps

For deployment setup, see firecrawl-deploy-integration.