Async Screenshot Processing with Webhooks: No More Timeouts

The Timeout Problem

Synchronous screenshot APIs have a fundamental issue: HTTP timeouts. When you capture a complex page that takes 15-30 seconds to render, your HTTP connection can time out. Load balancers, proxies, and client libraries all enforce timeout limits.

The solution? Asynchronous processing with webhooks. Submit the job, get a job ID, and receive a notification when it’s done.

How Async Webhooks Work

The flow is simple:

Submit — Send a screenshot request with a webhookUrl
Receive job ID — API returns immediately with a job identifier
Processing — API captures the screenshot in the background
Notification — API sends the result to your webhook URL
Retrieve — Download the screenshot from the provided URL

Submitting Async Requests

Node.js

const axios = require('axios');

async function submitScreenshotJob(url, options = {}) {
  const response = await axios.post(
    'https://api.toolcenter.dev/v1/screenshot',
    {
      url: url,
      width: options.width || 1280,
      height: options.height || 800,
      format: options.format || 'png',
      fullPage: options.fullPage || false,
      webhookUrl: 'https://your-server.com/api/webhook/screenshot',
    },
    {
      headers: { 'Authorization': 'Bearer YOUR_API_KEY' },
    }
  );

  return response.data;
  // { jobId: 'abc123', status: 'queued' }
}

Python

import requests

def submit_screenshot_job(url, webhook_url):
    response = requests.post(
        'https://api.toolcenter.dev/v1/screenshot',
        json={
            'url': url,
            'width': 1280,
            'height': 800,
            'format': 'png',
            'webhookUrl': webhook_url,
        },
        headers={'Authorization': 'Bearer YOUR_API_KEY'}
    )
    return response.json()  # {'jobId': 'abc123', 'status': 'queued'}

Building the Webhook Receiver

Express.js

const express = require('express');
const crypto = require('crypto');
const app = express();

app.use(express.json());

// Store for pending jobs
const pendingJobs = new Map();

app.post('/api/webhook/screenshot', (req, res) => {
  const { jobId, status, screenshotUrl, error } = req.body;

  // Verify webhook signature
  const signature = req.headers['x-webhook-signature'];
  const expectedSig = crypto
    .createHmac('sha256', process.env.WEBHOOK_SECRET)
    .update(JSON.stringify(req.body))
    .digest('hex');

  if (signature !== expectedSig) {
    return res.status(401).json({ error: 'Invalid signature' });
  }

  console.log(`Job ${jobId}: ${status}`);

  if (status === 'completed') {
    // Download and process the screenshot
    processCompletedScreenshot(jobId, screenshotUrl);
  } else if (status === 'failed') {
    console.error(`Job ${jobId} failed: ${error}`);
    handleFailedJob(jobId, error);
  }

  // Always respond 200 to acknowledge receipt
  res.status(200).json({ received: true });
});

async function processCompletedScreenshot(jobId, screenshotUrl) {
  const response = await axios.get(screenshotUrl, { responseType: 'arraybuffer' });
  const filename = `screenshots/${jobId}.png`;
  fs.writeFileSync(filename, response.data);
  console.log(`Saved: ${filename}`);

  // Resolve any pending promises
  const resolver = pendingJobs.get(jobId);
  if (resolver) {
    resolver.resolve(filename);
    pendingJobs.delete(jobId);
  }
}

app.listen(3000, () => console.log('Webhook server ready'));

Flask (Python)

from flask import Flask, request, jsonify
import hmac
import hashlib
import os

app = Flask(__name__)
WEBHOOK_SECRET = os.environ['WEBHOOK_SECRET']

@app.route('/api/webhook/screenshot', methods=['POST'])
def handle_webhook():
    # Verify signature
    signature = request.headers.get('X-Webhook-Signature', '')
    expected = hmac.new(
        WEBHOOK_SECRET.encode(),
        request.data,
        hashlib.sha256
    ).hexdigest()

    if not hmac.compare_digest(signature, expected):
        return jsonify({'error': 'Invalid signature'}), 401

    data = request.json
    job_id = data['jobId']
    status = data['status']

    if status == 'completed':
        screenshot_url = data['screenshotUrl']
        process_screenshot(job_id, screenshot_url)
    elif status == 'failed':
        handle_failure(job_id, data.get('error'))

    return jsonify({'received': True}), 200

def process_screenshot(job_id, url):
    response = requests.get(url)
    with open(f'screenshots/{job_id}.png', 'wb') as f:
        f.write(response.content)
    print(f'Saved screenshot for job {job_id}')

Promise-Based Async Pattern

Create a clean interface that submits the job and resolves when the webhook fires:

class AsyncScreenshotClient {
  constructor(apiKey, webhookBaseUrl) {
    this.apiKey = apiKey;
    this.webhookBaseUrl = webhookBaseUrl;
    this.pending = new Map();
  }

  capture(url, options = {}) {
    return new Promise(async (resolve, reject) => {
      const timeout = setTimeout(() => {
        reject(new Error('Screenshot timed out'));
        this.pending.delete(jobId);
      }, options.timeout || 60000);

      const { jobId } = await submitScreenshotJob(url, {
        ...options,
        webhookUrl: `${this.webhookBaseUrl}/api/webhook/screenshot`,
      });

      this.pending.set(jobId, {
        resolve: (result) => { clearTimeout(timeout); resolve(result); },
        reject: (error) => { clearTimeout(timeout); reject(error); },
      });
    });
  }

  handleWebhook(data) {
    const handler = this.pending.get(data.jobId);
    if (!handler) return;

    if (data.status === 'completed') {
      handler.resolve(data.screenshotUrl);
    } else {
      handler.reject(new Error(data.error));
    }

    this.pending.delete(data.jobId);
  }
}

// Usage
const client = new AsyncScreenshotClient(API_KEY, 'https://your-server.com');

const screenshotUrl = await client.capture('https://example.com', {
  width: 1280,
  height: 800,
  timeout: 30000,
});

Batch Processing with Webhooks

Process thousands of URLs without holding connections open:

async function batchCapture(urls) {
  const jobs = [];

  // Submit all jobs
  for (const url of urls) {
    const job = await submitScreenshotJob(url, {
      webhookUrl: 'https://your-server.com/api/webhook/batch',
    });
    jobs.push({ url, jobId: job.jobId });

    // Small delay to avoid overwhelming the API
    await sleep(50);
  }

  console.log(`Submitted ${jobs.length} jobs`);
  return jobs;
}

The webhook handler processes results as they arrive:

const batchResults = { completed: 0, failed: 0, total: 0 };

app.post('/api/webhook/batch', (req, res) => {
  const { jobId, status, screenshotUrl } = req.body;

  if (status === 'completed') {
    batchResults.completed++;
    downloadAndSave(jobId, screenshotUrl);
  } else {
    batchResults.failed++;
  }

  batchResults.total = batchResults.completed + batchResults.failed;
  console.log(`Batch progress: ${batchResults.total} processed (${batchResults.completed} ok, ${batchResults.failed} failed)`);

  res.status(200).json({ received: true });
});

Webhook Security

Signature Verification

Always verify webhook signatures to prevent spoofing:

function verifyWebhookSignature(payload, signature, secret) {
  const expected = crypto
    .createHmac('sha256', secret)
    .update(typeof payload === 'string' ? payload : JSON.stringify(payload))
    .digest('hex');

  return crypto.timingSafeEqual(
    Buffer.from(signature),
    Buffer.from(expected)
  );
}

IP Allowlisting

Only accept webhooks from known API server IPs:

const ALLOWED_IPS = ['203.0.113.1', '203.0.113.2'];

app.use('/api/webhook/*', (req, res, next) => {
  const clientIp = req.ip || req.connection.remoteAddress;
  if (!ALLOWED_IPS.includes(clientIp)) {
    return res.status(403).json({ error: 'Forbidden' });
  }
  next();
});

Retry and Failure Handling

Webhooks can fail. Build retry logic:

async function handleFailedJob(jobId, error) {
  const job = pendingJobs.get(jobId);
  if (!job) return;

  job.retries = (job.retries || 0) + 1;

  if (job.retries < 3) {
    console.log(`Retrying job ${jobId} (attempt ${job.retries + 1})`);
    await submitScreenshotJob(job.url, {
      webhookUrl: job.webhookUrl,
    });
  } else {
    console.error(`Job ${jobId} permanently failed after 3 attempts`);
    pendingJobs.delete(jobId);
  }
}

When to Use Webhooks vs Synchronous

Use synchronous when:

Single screenshots with fast-loading pages
Real-time user-facing features
Simple scripts and one-off captures

Use webhooks when:

Batch processing hundreds or thousands of URLs
Pages that take 10+ seconds to load
Background processing where immediate results aren’t needed
You want to avoid holding HTTP connections open

Conclusion

Webhooks transform screenshot processing from a blocking operation into an event-driven pipeline. Submit jobs in bulk, let the API process them asynchronously, and handle results as they arrive. This pattern eliminates timeout issues, enables massive parallelism, and makes your screenshot pipeline far more resilient.

The Timeout Problem#

How Async Webhooks Work#

Submitting Async Requests#

Node.js#

Python#

Building the Webhook Receiver#

Express.js#

Flask (Python)#

Promise-Based Async Pattern#

Batch Processing with Webhooks#

Webhook Security#

Signature Verification#

IP Allowlisting#

Retry and Failure Handling#

When to Use Webhooks vs Synchronous#

Conclusion#