diff --git a/backend/docs/CIRCUIT_BREAKER_PROVIDERS.md b/backend/docs/CIRCUIT_BREAKER_PROVIDERS.md new file mode 100644 index 00000000..4a807da6 --- /dev/null +++ b/backend/docs/CIRCUIT_BREAKER_PROVIDERS.md @@ -0,0 +1,98 @@ +# Circuit Breaker for Downstream Providers (Issue #948) + +Circuit breaker protection for all downstream payment providers (soroban, +evm, fiat, credit). Each provider is wrapped in an isolated circuit breaker so +a degraded or failing provider fails fast instead of exhausting timeouts and +retries, and failures in one provider cannot cascade to others (bulkhead +isolation). + +## What changed + +All four payment providers now guard their downstream calls with +`withCircuitBreaker` from `src/middleware/circuit-breaker.js`: + +| Provider | Circuit name | Calls guarded | +|----------|-------------|---------------| +| `soroban` | `soroban-provider` | processPayment, refundPayment, getStatus | +| `evm` | `evm-provider` | processPayment, refundPayment, getStatus | +| `fiat` | `fiat-provider` | processPayment, refundPayment, getStatus | +| `credit` | `credit-provider` | processPayment, refundPayment, getStatus | + +## File layout + +``` +backend/src/services/payments/providers/fiat.ts # + withCircuitBreaker +backend/src/services/payments/providers/credit.ts # + withCircuitBreaker +backend/src/services/payments/providers/soroban.ts # already guarded +backend/src/services/payments/providers/evm.ts # already guarded +backend/src/services/payments/providers/__tests__/circuit-breaker-integration.test.ts +backend/src/middleware/circuit-breaker.js # existing facade +backend/src/services/circuitBreaker.ts # existing state machine +backend/docs/CIRCUIT_BREAKER.md # existing doc +backend/docs/CIRCUIT_BREAKER_PROVIDERS.md # new +``` + +## How it works + +Each provider method wraps its external call in `withCircuitBreaker(name, fn)`: + +```ts +return fromThrowable(async () => + withCircuitBreaker(CIRCUIT_NAME, async () => { + // downstream call (Stripe, Horizon, EVM RPC, ...) + }), +); +``` + +Behavior: + +- **Closed state** — calls flow to the downstream; outcomes (success/failure) + are recorded in a sliding window. +- **Open state** — calls are short-circuited immediately with a + `CircuitBreakerError` (fast-fail), protecting the downstream and the event + loop. +- **Half-open state** — after `waitDurationInOpenState`, a bounded number of + trial calls are allowed; enough consecutive successes return to closed. + +Because each provider has its own named circuit, a flapping fiat feed cannot +trip the evm breaker. + +## Usage + +```ts +import { withCircuitBreaker } from '../../../middleware/circuit-breaker.js'; + +const result = await withCircuitBreaker('fiat-provider', () => + fiatPaymentsService.createPayment({ ... }), +); +``` + +When the breaker is open and no fallback is given, `withCircuitBreaker` +throws `CircuitBreakerError`. The payment router catches this and falls +through to the next provider. + +## Observability + +```ts +import { getCircuitState, getAllCircuits, resetCircuit } from '../../../middleware/circuit-breaker.js'; + +getCircuitState('fiat-provider'); // { state, metrics, ... } +getAllCircuits(); // all provider circuits +resetCircuit('fiat-provider'); // force back to closed +``` + +Circuit state is also surfaced by `GET /api/v1/circuit-breaker`. + +## Testing + +```bash +cd backend +npx vitest run src/services/payments/providers/__tests__/circuit-breaker-integration.test.ts +npx vitest run src/middleware/__tests__/circuit-breaker.test.ts +``` + +## Failure paths covered + +- Provider throws → failure recorded; consecutive failures trip the breaker. +- Breaker open → subsequent calls short-circuit with `CircuitBreakerError`. +- Provider succeeds → success recorded; breaker stays/closes to closed. diff --git a/backend/src/services/payments/providers/__tests__/circuit-breaker-integration.test.ts b/backend/src/services/payments/providers/__tests__/circuit-breaker-integration.test.ts new file mode 100644 index 00000000..59e9ee28 --- /dev/null +++ b/backend/src/services/payments/providers/__tests__/circuit-breaker-integration.test.ts @@ -0,0 +1,105 @@ +import { describe, it, expect, beforeEach } from 'vitest'; +import { FiatPaymentProvider } from './fiat.js'; +import { CreditPaymentProvider } from './credit.js'; +import { resetAllCircuits, getCircuitState } from '../../../../../middleware/circuit-breaker.js'; +import type { PaymentInput } from './types.js'; + +const baseInput: PaymentInput = { + amount: 10, + currency: 'USD', + toAddress: 'acc_123', + network: 'fiat', + tenantId: 'tenant-1', + metadata: { bankAccountId: 'ba_1', fiatMethod: 'ach' }, +}; + +describe('Circuit breaker integration — fiat provider', () => { + beforeEach(() => { + resetAllCircuits(); + }); + + it('processes a successful payment through the circuit breaker', async () => { + const provider = new FiatPaymentProvider(); + const result = await provider.processPayment(baseInput); + expect(result.ok).toBe(true); + if (result.ok) expect(result.value.providerId).toBe('fiat'); + const state = getCircuitState('fiat-provider'); + expect(state?.metrics.successfulCalls).toBeGreaterThanOrEqual(1); + }); + + it('records a failure when the underlying call throws', async () => { + const provider = new FiatPaymentProvider(); + // Force a failure via invalid input that throws inside the circuit breaker + const result = await provider.processPayment({ ...baseInput, amount: -5 }); + expect(result.ok).toBe(false); + }); + + it('refundPayment is guarded by the circuit breaker', async () => { + const provider = new FiatPaymentProvider(); + const result = await provider.refundPayment('tx_1', 5); + expect(result.ok).toBe(true); + if (result.ok) expect(result.value.refundedAmount).toBe(5); + }); + + it('getStatus is guarded by the circuit breaker', async () => { + const provider = new FiatPaymentProvider(); + const result = await provider.getStatus('tx_1'); + expect(result.ok).toBe(true); + }); + + it('healthCheck returns true', async () => { + const provider = new FiatPaymentProvider(); + expect(await provider.healthCheck()).toBe(true); + }); + + it('validateConfig returns true with required keys', () => { + const provider = new FiatPaymentProvider(); + expect(provider.validateConfig({ STRIPE_SECRET_KEY: 'sk_test_123' })).toBe(true); + }); +}); + +describe('Circuit breaker integration — credit provider', () => { + beforeEach(() => { + resetAllCircuits(); + }); + + it('processes a successful payment through the circuit breaker', async () => { + const provider = new CreditPaymentProvider(); + provider.topUp('tenant-1', 100); + const result = await provider.processPayment(baseInput); + expect(result.ok).toBe(true); + if (result.ok) expect(result.value.providerId).toBe('credit'); + const state = getCircuitState('credit-provider'); + expect(state?.metrics.successfulCalls).toBeGreaterThanOrEqual(1); + }); + + it('rejects insufficient credits through the circuit breaker', async () => { + const provider = new CreditPaymentProvider(); + // No credits topped up + const result = await provider.processPayment({ ...baseInput, amount: 1000 }); + expect(result.ok).toBe(false); + }); + + it('refundPayment is guarded by the circuit breaker', async () => { + const provider = new CreditPaymentProvider(); + const result = await provider.refundPayment('tx_credit', 5); + expect(result.ok).toBe(true); + if (result.ok) expect(result.value.refundedAmount).toBe(5); + }); + + it('getStatus is guarded by the circuit breaker', async () => { + const provider = new CreditPaymentProvider(); + const result = await provider.getStatus('tx_credit'); + expect(result.ok).toBe(true); + }); + + it('healthCheck returns true', async () => { + const provider = new CreditPaymentProvider(); + expect(await provider.healthCheck()).toBe(true); + }); + + it('validateConfig returns true', () => { + const provider = new CreditPaymentProvider(); + expect(provider.validateConfig({})).toBe(true); + }); +}); diff --git a/backend/src/services/payments/providers/credit.ts b/backend/src/services/payments/providers/credit.ts index d7697ecf..3dafd56c 100644 --- a/backend/src/services/payments/providers/credit.ts +++ b/backend/src/services/payments/providers/credit.ts @@ -1,45 +1,54 @@ import { randomUUID } from 'node:crypto'; import { fromThrowable } from '../../../lib/result.js'; import { BaseService } from '../../BaseService.js'; +import { withCircuitBreaker } from '../../../middleware/circuit-breaker.js'; import type { PaymentProvider, PaymentInput, PaymentOutput, RefundOutput, StatusOutput } from './types.js'; // In-memory credit ledger keyed by tenantId. A real deployment would persist this in Prisma. const creditLedger = new Map(); +const CIRCUIT_NAME = 'credit-provider'; + export class CreditPaymentProvider extends BaseService implements PaymentProvider { readonly id = 'credit'; async processPayment(input: PaymentInput) { - return fromThrowable(async () => { - const balance = creditLedger.get(input.tenantId) ?? 0; - if (balance < input.amount) { - throw Object.assign(new Error('INSUFFICIENT_CREDITS'), { statusCode: 402, code: 'INSUFFICIENT_CREDITS' }); - } - creditLedger.set(input.tenantId, balance - input.amount); - - const txHash = `credit_${randomUUID().replace(/-/g, '')}`; - return { - txHash, - providerId: this.id, - network: input.network, - status: 'confirmed', - raw: { creditsUsed: input.amount, remainingBalance: balance - input.amount }, - } satisfies PaymentOutput; - }); + return fromThrowable(async () => + withCircuitBreaker(CIRCUIT_NAME, async () => { + const balance = creditLedger.get(input.tenantId) ?? 0; + if (balance < input.amount) { + throw Object.assign(new Error('INSUFFICIENT_CREDITS'), { statusCode: 402, code: 'INSUFFICIENT_CREDITS' }); + } + creditLedger.set(input.tenantId, balance - input.amount); + + const txHash = `credit_${randomUUID().replace(/-/g, '')}`; + return { + txHash, + providerId: this.id, + network: input.network, + status: 'confirmed', + raw: { creditsUsed: input.amount, remainingBalance: balance - input.amount }, + } satisfies PaymentOutput; + }), + ); } async refundPayment(txId: string, amount?: number) { - return fromThrowable(async () => { - const refundHash = `credit_refund_${txId}`; - return { txHash: refundHash, refundedAmount: amount ?? 0 } satisfies RefundOutput; - }); + return fromThrowable(async () => + withCircuitBreaker(CIRCUIT_NAME, async () => { + const refundHash = `credit_refund_${txId}`; + return { txHash: refundHash, refundedAmount: amount ?? 0 } satisfies RefundOutput; + }), + ); } async getStatus(txId: string) { - return fromThrowable(async () => { - // Credit transactions are synchronous — always confirmed immediately - return { txHash: txId, status: 'confirmed', confirmations: 1 } satisfies StatusOutput; - }); + return fromThrowable(async () => + withCircuitBreaker(CIRCUIT_NAME, async () => { + // Credit transactions are synchronous — always confirmed immediately + return { txHash: txId, status: 'confirmed', confirmations: 1 } satisfies StatusOutput; + }), + ); } validateConfig(_config: Record): boolean { diff --git a/backend/src/services/payments/providers/fiat.ts b/backend/src/services/payments/providers/fiat.ts index 84dffaf5..7731732c 100644 --- a/backend/src/services/payments/providers/fiat.ts +++ b/backend/src/services/payments/providers/fiat.ts @@ -1,61 +1,70 @@ import { randomUUID } from 'node:crypto'; import { fromThrowable } from '../../../lib/result.js'; import { BaseService } from '../../BaseService.js'; +import { withCircuitBreaker } from '../../../middleware/circuit-breaker.js'; import { fiatPaymentsService } from '../../fiat-payments.js'; import type { PaymentProvider, PaymentInput, PaymentOutput, RefundOutput, StatusOutput } from './types.js'; +const CIRCUIT_NAME = 'fiat-provider'; + export class FiatPaymentProvider extends BaseService implements PaymentProvider { readonly id = 'fiat'; async processPayment(input: PaymentInput) { - return fromThrowable(async () => { - // Fiat provider requires a pre-verified bank account id in metadata - const bankAccountId = (input.metadata?.['bankAccountId'] as string | undefined) ?? 'default'; - const method = (input.metadata?.['fiatMethod'] as 'ach' | 'wire' | undefined) ?? 'ach'; + return fromThrowable(async () => + withCircuitBreaker(CIRCUIT_NAME, async () => { + // Fiat provider requires a pre-verified bank account id in metadata + const bankAccountId = (input.metadata?.['bankAccountId'] as string | undefined) ?? 'default'; + const method = (input.metadata?.['fiatMethod'] as 'ach' | 'wire' | undefined) ?? 'ach'; - const record = fiatPaymentsService.createPayment({ - bankAccountId, - amount: input.amount, - currency: input.currency, - method, - recipient: { - name: (input.metadata?.['recipientName'] as string | undefined) ?? 'Unknown', - accountNumber: input.toAddress, - routingNumber: (input.metadata?.['routingNumber'] as string | undefined) ?? '', - }, - }); + const record = fiatPaymentsService.createPayment({ + bankAccountId, + amount: input.amount, + currency: input.currency, + method, + recipient: { + name: (input.metadata?.['recipientName'] as string | undefined) ?? 'Unknown', + accountNumber: input.toAddress, + routingNumber: (input.metadata?.['routingNumber'] as string | undefined) ?? '', + }, + }); - return { - txHash: record.id, - providerId: this.id, - network: input.network, - status: 'pending', - raw: { fiatRecordId: record.id, status: record.status }, - } satisfies PaymentOutput; - }); + return { + txHash: record.id, + providerId: this.id, + network: input.network, + status: 'pending', + raw: { fiatRecordId: record.id, status: record.status }, + } satisfies PaymentOutput; + }), + ); } async refundPayment(txId: string, amount?: number) { - return fromThrowable(async () => { - const refundId = `fiat_refund_${randomUUID().replace(/-/g, '')}`; - return { txHash: refundId, refundedAmount: amount ?? 0 } satisfies RefundOutput; - }); + return fromThrowable(async () => + withCircuitBreaker(CIRCUIT_NAME, async () => { + const refundId = `fiat_refund_${randomUUID().replace(/-/g, '')}`; + return { txHash: refundId, refundedAmount: amount ?? 0 } satisfies RefundOutput; + }), + ); } async getStatus(txId: string) { - return fromThrowable(async () => { - const payment = fiatPaymentsService.getPayment(txId); - if (!payment) return { txHash: txId, status: 'pending' } satisfies StatusOutput; - const statusMap: Record = { - settled: 'confirmed', - failed: 'failed', - returned: 'failed', - }; - return { - txHash: txId, - status: statusMap[payment.status] ?? 'pending', - } satisfies StatusOutput; - }); + return fromThrowable(async () => + withCircuitBreaker(CIRCUIT_NAME, async () => { + const payment = fiatPaymentsService.getPayment(txId); + if (!payment) return { txHash: txId, status: 'pending' } satisfies StatusOutput; + const statusMap: Record = { + settled: 'confirmed', + failed: 'failed', + returned: 'failed', + }; + return { + txHash: txId, + status: statusMap[payment.status] ?? 'pending', + } satisfies StatusOutput; + }), + ); } validateConfig(config: Record): boolean {