Reference: Worker Fleet

API publishes job → Redis (BullMQ) → worker pod(s) → Postgres + outbound HTTP
                                      ↓ failures
                                    DLQ queue + PagerDuty

// Job envelope standard
type JobEnvelope<T> = {
  idempotencyKey: string;
  tenantId: string;
  traceparent?: string;
  payload: T;
};

// src/workers/order-email.worker.ts
import { Worker, Queue } from "bullmq";
import IORedis from "ioredis";
 
const connection = new IORedis(process.env.REDIS_URL!, { maxRetriesPerRequest: null });
 
export const orderEmailQueue = new Queue("order-email", { connection });
 
const worker = new Worker(
  "order-email",
  async (job) => {
    const { idempotencyKey, tenantId, payload } = job.data;
    const done = await redis.set(`idem:${idempotencyKey}`, "1", "EX", 86400, "NX");
    if (!done) {
      job.log("duplicate skipped");
      return;
    }
    await sendEmail(tenantId, payload);
  },
  {
    connection,
    concurrency: 5, // pool max 20 on worker; API uses 10
  },
);
 
worker.on("failed", async (job, err) => {
  if (job && job.attemptsMade >= 3) {
    await dlq.add("failed", { original: job.data, error: String(err) });
  }
});

// src/main-worker.ts - graceful shutdown
import { orderEmailWorker } from "./workers/order-email.worker";
 
async function shutdown(signal: string) {
  console.info({ msg: "shutdown", signal });
  await orderEmailWorker.close();
  process.exit(0);
}
process.on("SIGTERM", () => shutdown("SIGTERM"));

pool max on worker = 20
concurrency = 5 (not 20 - jobs do 2-4 queries each)
API pool max = 10 (separate Deployment)

1. Alert fires on DLQ depth
2. On-call inspects first 5 jobs in Bull Board
3. Fix root cause (template bug, vendor 403)
4. Replay DLQ to primary queue in batches of 50
5. Post-mortem if customer emails missed SLA