feat: add circuit breaker for downstream payment providers (#948)

backend/docs/CIRCUIT_BREAKER_PROVIDERS.md

@@ -0,0 +1,98 @@
+# Circuit Breaker for Downstream Providers (Issue #948)
+
+Circuit breaker protection for all downstream payment providers (soroban,
+evm, fiat, credit). Each provider is wrapped in an isolated circuit breaker so
+a degraded or failing provider fails fast instead of exhausting timeouts and
+retries, and failures in one provider cannot cascade to others (bulkhead
+isolation).
+
+## What changed
+
+All four payment providers now guard their downstream calls with
+`withCircuitBreaker` from `src/middleware/circuit-breaker.js`:
+
+| Provider | Circuit name | Calls guarded |
+|----------|-------------|---------------|
+| `soroban` | `soroban-provider` | processPayment, refundPayment, getStatus |
+| `evm` | `evm-provider` | processPayment, refundPayment, getStatus |
+| `fiat` | `fiat-provider` | processPayment, refundPayment, getStatus |
+| `credit` | `credit-provider` | processPayment, refundPayment, getStatus |
+
+## File layout
+
+```
+backend/src/services/payments/providers/fiat.ts        # + withCircuitBreaker
+backend/src/services/payments/providers/credit.ts      # + withCircuitBreaker
+backend/src/services/payments/providers/soroban.ts     # already guarded
+backend/src/services/payments/providers/evm.ts         # already guarded
+backend/src/services/payments/providers/__tests__/circuit-breaker-integration.test.ts
+backend/src/middleware/circuit-breaker.js              # existing facade
+backend/src/services/circuitBreaker.ts                 # existing state machine
+backend/docs/CIRCUIT_BREAKER.md                        # existing doc
+backend/docs/CIRCUIT_BREAKER_PROVIDERS.md              # new
+```
+
+## How it works
+
+Each provider method wraps its external call in `withCircuitBreaker(name, fn)`:
+
+```ts
+return fromThrowable(async () =>
+  withCircuitBreaker(CIRCUIT_NAME, async () => {
+    // downstream call (Stripe, Horizon, EVM RPC, ...)
+  }),
+);
+```
+
+Behavior:
+
+- **Closed state** — calls flow to the downstream; outcomes (success/failure)
+  are recorded in a sliding window.
+- **Open state** — calls are short-circuited immediately with a
+  `CircuitBreakerError` (fast-fail), protecting the downstream and the event
+  loop.
+- **Half-open state** — after `waitDurationInOpenState`, a bounded number of
+  trial calls are allowed; enough consecutive successes return to closed.
+
+Because each provider has its own named circuit, a flapping fiat feed cannot
+trip the evm breaker.
+
+## Usage
+
+```ts
+import { withCircuitBreaker } from '../../../middleware/circuit-breaker.js';
+
+const result = await withCircuitBreaker('fiat-provider', () =>
+  fiatPaymentsService.createPayment({ ... }),
+);
+```
+
+When the breaker is open and no fallback is given, `withCircuitBreaker`
+throws `CircuitBreakerError`. The payment router catches this and falls
+through to the next provider.
+
+## Observability
+
+```ts
+import { getCircuitState, getAllCircuits, resetCircuit } from '../../../middleware/circuit-breaker.js';
+
+getCircuitState('fiat-provider');   // { state, metrics, ... }
+getAllCircuits();                   // all provider circuits
+resetCircuit('fiat-provider');      // force back to closed
+```
+
+Circuit state is also surfaced by `GET /api/v1/circuit-breaker`.
+
+## Testing
+
+```bash
+cd backend
+npx vitest run src/services/payments/providers/__tests__/circuit-breaker-integration.test.ts
+npx vitest run src/middleware/__tests__/circuit-breaker.test.ts
+```
+
+## Failure paths covered
+
+- Provider throws → failure recorded; consecutive failures trip the breaker.
+- Breaker open → subsequent calls short-circuit with `CircuitBreakerError`.
+- Provider succeeds → success recorded; breaker stays/closes to closed.

backend/src/services/payments/providers/__tests__/circuit-breaker-integration.test.ts

@@ -0,0 +1,105 @@
+import { describe, it, expect, beforeEach } from 'vitest';
+import { FiatPaymentProvider } from './fiat.js';
+import { CreditPaymentProvider } from './credit.js';
+import { resetAllCircuits, getCircuitState } from '../../../../../middleware/circuit-breaker.js';
+import type { PaymentInput } from './types.js';
+
+const baseInput: PaymentInput = {
+  amount: 10,
+  currency: 'USD',
+  toAddress: 'acc_123',
+  network: 'fiat',
+  tenantId: 'tenant-1',
+  metadata: { bankAccountId: 'ba_1', fiatMethod: 'ach' },
+};
+
+describe('Circuit breaker integration — fiat provider', () => {
+  beforeEach(() => {
+    resetAllCircuits();
+  });
+
+  it('processes a successful payment through the circuit breaker', async () => {
+    const provider = new FiatPaymentProvider();
+    const result = await provider.processPayment(baseInput);
+    expect(result.ok).toBe(true);
+    if (result.ok) expect(result.value.providerId).toBe('fiat');
+    const state = getCircuitState('fiat-provider');
+    expect(state?.metrics.successfulCalls).toBeGreaterThanOrEqual(1);
+  });
+
+  it('records a failure when the underlying call throws', async () => {
+    const provider = new FiatPaymentProvider();
+    // Force a failure via invalid input that throws inside the circuit breaker
+    const result = await provider.processPayment({ ...baseInput, amount: -5 });
+    expect(result.ok).toBe(false);
+  });
+
+  it('refundPayment is guarded by the circuit breaker', async () => {
+    const provider = new FiatPaymentProvider();
+    const result = await provider.refundPayment('tx_1', 5);
+    expect(result.ok).toBe(true);
+    if (result.ok) expect(result.value.refundedAmount).toBe(5);
+  });
+
+  it('getStatus is guarded by the circuit breaker', async () => {
+    const provider = new FiatPaymentProvider();
+    const result = await provider.getStatus('tx_1');
+    expect(result.ok).toBe(true);
+  });
+
+  it('healthCheck returns true', async () => {
+    const provider = new FiatPaymentProvider();
+    expect(await provider.healthCheck()).toBe(true);
+  });
+
+  it('validateConfig returns true with required keys', () => {
+    const provider = new FiatPaymentProvider();
+    expect(provider.validateConfig({ STRIPE_SECRET_KEY: 'sk_test_123' })).toBe(true);
+  });
+});
+
+describe('Circuit breaker integration — credit provider', () => {
+  beforeEach(() => {
+    resetAllCircuits();
+  });
+
+  it('processes a successful payment through the circuit breaker', async () => {
+    const provider = new CreditPaymentProvider();
+    provider.topUp('tenant-1', 100);
+    const result = await provider.processPayment(baseInput);
+    expect(result.ok).toBe(true);
+    if (result.ok) expect(result.value.providerId).toBe('credit');
+    const state = getCircuitState('credit-provider');
+    expect(state?.metrics.successfulCalls).toBeGreaterThanOrEqual(1);
+  });
+
+  it('rejects insufficient credits through the circuit breaker', async () => {
+    const provider = new CreditPaymentProvider();
+    // No credits topped up
+    const result = await provider.processPayment({ ...baseInput, amount: 1000 });
+    expect(result.ok).toBe(false);
+  });
+
+  it('refundPayment is guarded by the circuit breaker', async () => {
+    const provider = new CreditPaymentProvider();
+    const result = await provider.refundPayment('tx_credit', 5);
+    expect(result.ok).toBe(true);
+    if (result.ok) expect(result.value.refundedAmount).toBe(5);
+  });
+
+  it('getStatus is guarded by the circuit breaker', async () => {
+    const provider = new CreditPaymentProvider();
+    const result = await provider.getStatus('tx_credit');
+    expect(result.ok).toBe(true);
+  });
+
+  it('healthCheck returns true', async () => {
+    const provider = new CreditPaymentProvider();
+    expect(await provider.healthCheck()).toBe(true);
+  });
+
+  it('validateConfig returns true', () => {
+    const provider = new CreditPaymentProvider();
+    expect(provider.validateConfig({})).toBe(true);
+  });
+});

backend/src/services/payments/providers/credit.ts

@@ -1,45 +1,54 @@
 import { randomUUID } from 'node:crypto';
 import { fromThrowable } from '../../../lib/result.js';
 import { BaseService } from '../../BaseService.js';
+import { withCircuitBreaker } from '../../../middleware/circuit-breaker.js';
 import type { PaymentProvider, PaymentInput, PaymentOutput, RefundOutput, StatusOutput } from './types.js';
 
 // In-memory credit ledger keyed by tenantId. A real deployment would persist this in Prisma.
 const creditLedger = new Map<string, number>();
 
+const CIRCUIT_NAME = 'credit-provider';
+
 export class CreditPaymentProvider extends BaseService implements PaymentProvider {
   readonly id = 'credit';
 
   async processPayment(input: PaymentInput) {
-    return fromThrowable(async () => {
-      const balance = creditLedger.get(input.tenantId) ?? 0;
-      if (balance < input.amount) {
-        throw Object.assign(new Error('INSUFFICIENT_CREDITS'), { statusCode: 402, code: 'INSUFFICIENT_CREDITS' });
-      }
-      creditLedger.set(input.tenantId, balance - input.amount);
-
-      const txHash = `credit_${randomUUID().replace(/-/g, '')}`;
-      return {
-        txHash,
-        providerId: this.id,
-        network: input.network,
-        status: 'confirmed',
-        raw: { creditsUsed: input.amount, remainingBalance: balance - input.amount },
-      } satisfies PaymentOutput;
-    });
+    return fromThrowable(async () =>
+      withCircuitBreaker(CIRCUIT_NAME, async () => {
+        const balance = creditLedger.get(input.tenantId) ?? 0;
+        if (balance < input.amount) {
+          throw Object.assign(new Error('INSUFFICIENT_CREDITS'), { statusCode: 402, code: 'INSUFFICIENT_CREDITS' });
+        }
+        creditLedger.set(input.tenantId, balance - input.amount);
+
+        const txHash = `credit_${randomUUID().replace(/-/g, '')}`;
+        return {
+          txHash,
+          providerId: this.id,
+          network: input.network,
+          status: 'confirmed',
+          raw: { creditsUsed: input.amount, remainingBalance: balance - input.amount },
+        } satisfies PaymentOutput;
+      }),
+    );
   }
 
   async refundPayment(txId: string, amount?: number) {
-    return fromThrowable(async () => {
-      const refundHash = `credit_refund_${txId}`;
-      return { txHash: refundHash, refundedAmount: amount ?? 0 } satisfies RefundOutput;
-    });
+    return fromThrowable(async () =>
+      withCircuitBreaker(CIRCUIT_NAME, async () => {
+        const refundHash = `credit_refund_${txId}`;
+        return { txHash: refundHash, refundedAmount: amount ?? 0 } satisfies RefundOutput;
+      }),
+    );
   }
 
   async getStatus(txId: string) {
-    return fromThrowable(async () => {
-      // Credit transactions are synchronous — always confirmed immediately
-      return { txHash: txId, status: 'confirmed', confirmations: 1 } satisfies StatusOutput;
-    });
+    return fromThrowable(async () =>
+      withCircuitBreaker(CIRCUIT_NAME, async () => {
+        // Credit transactions are synchronous — always confirmed immediately
+        return { txHash: txId, status: 'confirmed', confirmations: 1 } satisfies StatusOutput;
+      }),
+    );
   }
 
   validateConfig(_config: Record<string, unknown>): boolean {

backend/src/services/payments/providers/fiat.ts

@@ -1,61 +1,70 @@
 import { randomUUID } from 'node:crypto';
 import { fromThrowable } from '../../../lib/result.js';
 import { BaseService } from '../../BaseService.js';
+import { withCircuitBreaker } from '../../../middleware/circuit-breaker.js';
 import { fiatPaymentsService } from '../../fiat-payments.js';
 import type { PaymentProvider, PaymentInput, PaymentOutput, RefundOutput, StatusOutput } from './types.js';
 
+const CIRCUIT_NAME = 'fiat-provider';
+
 export class FiatPaymentProvider extends BaseService implements PaymentProvider {
   readonly id = 'fiat';
 
   async processPayment(input: PaymentInput) {
-    return fromThrowable(async () => {
-      // Fiat provider requires a pre-verified bank account id in metadata
-      const bankAccountId = (input.metadata?.['bankAccountId'] as string | undefined) ?? 'default';
-      const method = (input.metadata?.['fiatMethod'] as 'ach' | 'wire' | undefined) ?? 'ach';
+    return fromThrowable(async () =>
+      withCircuitBreaker(CIRCUIT_NAME, async () => {
+        // Fiat provider requires a pre-verified bank account id in metadata
+        const bankAccountId = (input.metadata?.['bankAccountId'] as string | undefined) ?? 'default';
+        const method = (input.metadata?.['fiatMethod'] as 'ach' | 'wire' | undefined) ?? 'ach';
 
-      const record = fiatPaymentsService.createPayment({
-        bankAccountId,
-        amount: input.amount,
-        currency: input.currency,
-        method,
-        recipient: {
-          name: (input.metadata?.['recipientName'] as string | undefined) ?? 'Unknown',
-          accountNumber: input.toAddress,
-          routingNumber: (input.metadata?.['routingNumber'] as string | undefined) ?? '',
-        },
-      });
+        const record = fiatPaymentsService.createPayment({
+          bankAccountId,
+          amount: input.amount,
+          currency: input.currency,
+          method,
+          recipient: {
+            name: (input.metadata?.['recipientName'] as string | undefined) ?? 'Unknown',
+            accountNumber: input.toAddress,
+            routingNumber: (input.metadata?.['routingNumber'] as string | undefined) ?? '',
+          },
+        });
 
-      return {
-        txHash: record.id,
-        providerId: this.id,
-        network: input.network,
-        status: 'pending',
-        raw: { fiatRecordId: record.id, status: record.status },
-      } satisfies PaymentOutput;
-    });
+        return {
+          txHash: record.id,
+          providerId: this.id,
+          network: input.network,
+          status: 'pending',
+          raw: { fiatRecordId: record.id, status: record.status },
+        } satisfies PaymentOutput;
+      }),
+    );
   }
 
   async refundPayment(txId: string, amount?: number) {
-    return fromThrowable(async () => {
-      const refundId = `fiat_refund_${randomUUID().replace(/-/g, '')}`;
-      return { txHash: refundId, refundedAmount: amount ?? 0 } satisfies RefundOutput;
-    });
+    return fromThrowable(async () =>
+      withCircuitBreaker(CIRCUIT_NAME, async () => {
+        const refundId = `fiat_refund_${randomUUID().replace(/-/g, '')}`;
+        return { txHash: refundId, refundedAmount: amount ?? 0 } satisfies RefundOutput;
+      }),
+    );
   }
 
   async getStatus(txId: string) {
-    return fromThrowable(async () => {
-      const payment = fiatPaymentsService.getPayment(txId);
-      if (!payment) return { txHash: txId, status: 'pending' } satisfies StatusOutput;
-      const statusMap: Record<string, StatusOutput['status']> = {
-        settled: 'confirmed',
-        failed: 'failed',
-        returned: 'failed',
-      };
-      return {
-        txHash: txId,
-        status: statusMap[payment.status] ?? 'pending',
-      } satisfies StatusOutput;
-    });
+    return fromThrowable(async () =>
+      withCircuitBreaker(CIRCUIT_NAME, async () => {
+        const payment = fiatPaymentsService.getPayment(txId);
+        if (!payment) return { txHash: txId, status: 'pending' } satisfies StatusOutput;
+        const statusMap: Record<string, StatusOutput['status']> = {
+          settled: 'confirmed',
+          failed: 'failed',
+          returned: 'failed',
+        };
+        return {
+          txHash: txId,
+          status: statusMap[payment.status] ?? 'pending',
+        } satisfies StatusOutput;
+      }),
+    );
   }
 
   validateConfig(config: Record<string, unknown>): boolean {