Reliability Patterns

Networks fail. Servers crash. Deploys happen. Your webhook handler needs to handle all of this gracefully.

The Idempotency Problem

Webhook providers guarantee at-least-once delivery, not exactly-once. This means:

Stripe sends webhook → Your server times out → Stripe retries → You process twice

The Solution: Idempotent Handlers

Make your handlers produce the same result whether called once or ten times.

// Store processed event IDs
const processedEvents = new Set(); // In production, use Redis or database

app.post('/webhooks/stripe', async (req, res) => {
  const event = stripe.webhooks.constructEvent(req.body, sig, secret);

  // Check if already processed
  if (processedEvents.has(event.id)) {
    console.log(`Event ${event.id} already processed, skipping`);
    return res.status(200).json({ received: true });
  }

  // Process the event
  await handleEvent(event);

  // Mark as processed
  processedEvents.add(event.id);

  res.status(200).json({ received: true });
});

Production-Ready Idempotency

// Using database for idempotency
async function processWebhook(event) {
  const conn = await db.getConnection();

  try {
    await conn.beginTransaction();

    // Try to insert event record (idempotency key)
    const [result] = await conn.query(
      `INSERT IGNORE INTO processed_webhooks (event_id, event_type, created_at)
       VALUES (?, ?, NOW())`,
      [event.id, event.type]
    );

    // If no rows inserted, event was already processed
    if (result.affectedRows === 0) {
      console.log(`Duplicate event ${event.id}, skipping`);
      await conn.rollback();
      return { duplicate: true };
    }

    // Process the event (inside same transaction)
    switch (event.type) {
      case 'payment_intent.succeeded':
        await fulfillOrder(conn, event.data.object);
        break;
      // ... other events
    }

    await conn.commit();
    return { success: true };

  } catch (error) {
    await conn.rollback();
    throw error;
  } finally {
    conn.release();
  }
}

Retry Strategies

How Stripe Retries

Stripe uses exponential backoff:

Response Codes That Trigger Retries

// Stripe retries on these conditions:
// - Network timeout (no response)
// - 5xx status codes (server errors)
// - 429 (rate limited)

// These do NOT trigger retries:
// - 2xx (success)
// - 4xx except 429 (client errors)

Async Processing Pattern

Process webhooks in the background to avoid timeouts:

const Queue = require('bull');
const webhookQueue = new Queue('webhooks', process.env.REDIS_URL);

// Webhook endpoint - just queue and respond
app.post('/webhooks/stripe', async (req, res) => {
  const event = stripe.webhooks.constructEvent(req.body, sig, secret);

  // Queue for async processing
  await webhookQueue.add('process', {
    eventId: event.id,
    eventType: event.type,
    payload: event.data.object
  }, {
    attempts: 3,
    backoff: {
      type: 'exponential',
      delay: 1000
    }
  });

  // Respond immediately
  res.status(200).json({ received: true });
});

// Background worker processes the queue
webhookQueue.process('process', async (job) => {
  const { eventId, eventType, payload } = job.data;

  // Check idempotency
  if (await isAlreadyProcessed(eventId)) {
    return { skipped: true };
  }

  // Process based on event type
  switch (eventType) {
    case 'payment_intent.succeeded':
      await fulfillOrder(payload);
      break;
    case 'customer.subscription.deleted':
      await cancelSubscription(payload);
      break;
  }

  // Mark as processed
  await markAsProcessed(eventId);

  return { processed: true };
});

Dead Letter Queues

When webhooks fail after all retries, store them for later investigation:

// Configure dead letter queue
const webhookQueue = new Queue('webhooks', {
  defaultJobOptions: {
    attempts: 5,
    backoff: { type: 'exponential', delay: 1000 },
    removeOnComplete: true,
    removeOnFail: false // Keep failed jobs
  }
});

const deadLetterQueue = new Queue('webhook-dlq');

// Move failed jobs to DLQ after all retries
webhookQueue.on('failed', async (job, err) => {
  if (job.attemptsMade >= job.opts.attempts) {
    await deadLetterQueue.add('failed', {
      originalJob: job.data,
      error: err.message,
      failedAt: new Date().toISOString(),
      attempts: job.attemptsMade
    });

    // Alert on-call engineer
    await alertSlack(`Webhook failed after ${job.attemptsMade} attempts: ${job.data.eventId}`);
  }
});

// Admin endpoint to retry DLQ items
app.post('/admin/webhooks/dlq/:id/retry', async (req, res) => {
  const dlqJob = await deadLetterQueue.getJob(req.params.id);

  if (!dlqJob) {
    return res.status(404).json({ error: 'Job not found' });
  }

  // Re-queue to main queue
  await webhookQueue.add('process', dlqJob.data.originalJob, {
    attempts: 3
  });

  // Remove from DLQ
  await dlqJob.remove();

  res.json({ success: true, message: 'Webhook re-queued' });
});

Handling Out-of-Order Events

Events can arrive out of order. A subscription.updated might arrive before subscription.created.

Strategy 1: Fetch Current State

async function handleSubscriptionUpdated(event) {
  const subscriptionId = event.data.object.id;

  // Don't trust event data - fetch current state from API
  const subscription = await stripe.subscriptions.retrieve(subscriptionId);

  // Use the fresh data
  await db.query(
    `INSERT INTO subscriptions (id, status, current_period_end)
     VALUES (?, ?, ?)
     ON DUPLICATE KEY UPDATE status = ?, current_period_end = ?`,
    [subscription.id, subscription.status, subscription.current_period_end,
     subscription.status, subscription.current_period_end]
  );
}

Strategy 2: Event Timestamps

async function handleEvent(event) {
  const eventTime = event.created; // Unix timestamp

  // Check if we've seen a more recent event for this resource
  const [existing] = await db.query(
    `SELECT last_event_time FROM resources WHERE id = ?`,
    [event.data.object.id]
  );

  if (existing && existing.last_event_time > eventTime) {
    console.log('Skipping out-of-order event');
    return;
  }

  // Process event and update timestamp
  await processEvent(event);
  await db.query(
    `UPDATE resources SET last_event_time = ? WHERE id = ?`,
    [eventTime, event.data.object.id]
  );
}

Monitoring and Alerting

// Track webhook metrics
const metrics = {
  received: new Counter('webhooks_received_total'),
  processed: new Counter('webhooks_processed_total'),
  failed: new Counter('webhooks_failed_total'),
  processingTime: new Histogram('webhook_processing_seconds'),
  queueDepth: new Gauge('webhook_queue_depth')
};

// Instrument your handler
app.post('/webhooks/stripe', async (req, res) => {
  metrics.received.inc({ source: 'stripe' });
  const startTime = Date.now();

  try {
    await processWebhook(req.body);
    metrics.processed.inc({ source: 'stripe' });
  } catch (error) {
    metrics.failed.inc({ source: 'stripe', error: error.code });
    throw error;
  } finally {
    metrics.processingTime.observe(
      { source: 'stripe' },
      (Date.now() - startTime) / 1000
    );
  }
});

// Alert on queue buildup
setInterval(async () => {
  const depth = await webhookQueue.count();
  metrics.queueDepth.set(depth);

  if (depth > 1000) {
    await alertPagerDuty('Webhook queue depth critical: ' + depth);
  }
}, 60000);

Recovery: Backfilling Missed Events

If your system was down, fetch missed events from the API:

// Stripe: List recent events
async function backfillStripeEvents(since) {
  let hasMore = true;
  let startingAfter = null;

  while (hasMore) {
    const events = await stripe.events.list({
      created: { gte: since },
      limit: 100,
      starting_after: startingAfter
    });

    for (const event of events.data) {
      // Process if not already handled
      if (!await isAlreadyProcessed(event.id)) {
        await processWebhook(event);
      }
    }

    hasMore = events.has_more;
    startingAfter = events.data[events.data.length - 1]?.id;
  }
}

// Run after downtime
await backfillStripeEvents(Math.floor(Date.now() / 1000) - 86400); // Last 24 hours

Key Takeaways

Next up: Building Webhooks - Implementing webhook senders and receivers from scratch.