Retry & Outbound Resilience

import got from "got";
 
const client = got.extend({
  timeout: { request: 8_000 },
  retry: {
    limit: 3,
    methods: ["GET", "PUT", "HEAD", "DELETE", "OPTIONS", "TRACE"],
    statusCodes: [408, 413, 429, 500, 502, 503, 504],
    calculateDelay: ({ attemptCount }) => Math.min(1000 * 2 ** attemptCount, 10_000) + Math.random() * 200,
  },
  hooks: {
    beforeRetry: [(err, retryCount) => {
      console.warn({ msg: "got_retry", retryCount, err: err.message });
    }],
  },
});

// src/http/partner-client.ts
import got, { type Got } from "got";
 
function buildClient(): Got {
  return got.extend({
    prefixUrl: process.env.PARTNER_API_URL,
    headers: { Authorization: `Bearer ${process.env.PARTNER_TOKEN}` },
    timeout: { request: 8_000 },
    retry: {
      limit: 3,
      methods: ["GET"],
      statusCodes: [429, 500, 502, 503, 504],
      calculateDelay: ({ attemptCount }) => 500 * 2 ** attemptCount + Math.random() * 100,
    },
  });
}
 
export const partnerClient = buildClient();
 
// POST with manual idempotent retry
export async function createPartnerJob(idempotencyKey: string, body: unknown) {
  const maxAttempts = 3;
  let attempt = 0;
  while (attempt < maxAttempts) {
    try {
      return await partnerClient.post("jobs", {
        json: body,
        headers: { "Idempotency-Key": idempotencyKey },
        retry: { limit: 0 },
      }).json();
    } catch (err) {
      attempt++;
      if (!isRetryable(err) || attempt >= maxAttempts) throw err;
      await sleep(500 * 2 ** attempt);
    }
  }
}
 
function isRetryable(err: unknown): boolean {
  if (!got.isHTTPError(err)) return true; // network
  const code = err.response?.statusCode ?? 0;
  return code >= 500 || code === 429;
}

import axios from "axios";
import axiosRetry from "axios-retry";
 
const api = axios.create({ baseURL: process.env.PARTNER_API_URL, timeout: 8000 });
axiosRetry(api, {
  retries: 3,
  retryDelay: axiosRetry.exponentialDelay,
  retryCondition: (err) => axiosRetry.isNetworkOrIdempotentRequestError(err) || err.response?.status === 429,
});

import CircuitBreaker from "opossum";
 
const breaker = new CircuitBreaker(callPartner, {
  timeout: 10_000,
  errorThresholdPercentage: 50,
  resetTimeout: 30_000,
});
 
breaker.fallback(() => ({ status: "degraded" }));

const deadline = Date.now() + 15_000;
while (attempts--) {
  const remaining = deadline - Date.now();
  if (remaining <= 0) throw new Error("deadline_exceeded");
  await client.get("x", { timeout: { request: remaining } });
}