From f9340937db488e0394cce50c4f5596cf60960392 Mon Sep 17 00:00:00 2001 From: Petar Todorovic Date: Wed, 16 Sep 2026 12:34:03 +0200 Subject: [PATCH] fix(earn): load eligible yields by token to avoid WAF 403 blocks Fetch token-eligible yields on the initial earn page by passing the selected token to the Yield API instead of serializing dozens of yield IDs into the query string. This prevents exceeding AWS WAF's 2KB query string limit (SizeRestrictions_QUERYSTRING). Also reduce the fallback yield IDs chunk size from 100 to 20 for paths that must still fetch yields by explicit identifiers. --- .../state/earn-selection/catalog/catalog.ts | 1 + .../earn/state/earn-selection/catalog/keys.ts | 6 ++- .../state/resource-observations.ts | 1 + .../yield-directory/yield-directory.ts | 36 ++++++++++----- .../src/services/api/resource-sources.ts | 1 + .../src/services/api/yield-resource-source.ts | 1 + .../widget/src/shared/effect/pagination.ts | 1 + .../atoms/atom-family-key-identity.test.ts | 3 ++ .../tests/resources/yield-directory.test.ts | 45 +++++++++++++++++-- 9 files changed, 79 insertions(+), 16 deletions(-) diff --git a/packages/widget/src/features/earn/state/earn-selection/catalog/catalog.ts b/packages/widget/src/features/earn/state/earn-selection/catalog/catalog.ts index c4679a167..c94aa52c9 100644 --- a/packages/widget/src/features/earn/state/earn-selection/catalog/catalog.ts +++ b/packages/widget/src/features/earn/state/earn-selection/catalog/catalog.ts @@ -115,6 +115,7 @@ export const earnYieldCatalogAtom = Atom.family((key: YieldCatalogKey) => { const source = enrichedYieldDirectoryResourceAtom.foreground( new YieldDirectoryKey({ network: key.network, + token: key.token, types: undefined, yieldIds: key.yieldIds, }) diff --git a/packages/widget/src/features/earn/state/earn-selection/catalog/keys.ts b/packages/widget/src/features/earn/state/earn-selection/catalog/keys.ts index 9383277d2..504c92fcb 100644 --- a/packages/widget/src/features/earn/state/earn-selection/catalog/keys.ts +++ b/packages/widget/src/features/earn/state/earn-selection/catalog/keys.ts @@ -14,15 +14,19 @@ export class AvailableYieldCategoriesKey extends Data.TaggedClass( export class YieldCatalogKey extends Data.TaggedClass("YieldCatalogKey")<{ category: DashboardYieldCategory | null; network: Network; + token: string | null; yieldIds: ReadonlyArray; }> { constructor(input: { readonly category: DashboardYieldCategory | null; readonly network: Network; + readonly token?: string | null; readonly yieldIds: ReadonlyArray; }) { super({ - ...input, + category: input.category, + network: input.network, + token: input.token ?? null, yieldIds: [...new Set(input.yieldIds)].sort(), }); } diff --git a/packages/widget/src/features/earn/state/earn-selection/state/resource-observations.ts b/packages/widget/src/features/earn/state/earn-selection/state/resource-observations.ts index f34234b05..acafd9b2a 100644 --- a/packages/widget/src/features/earn/state/earn-selection/state/resource-observations.ts +++ b/packages/widget/src/features/earn/state/earn-selection/state/resource-observations.ts @@ -104,6 +104,7 @@ const readYieldCatalog = ( new YieldCatalogKey({ category: null, network: selectedToken.token.network, + token: selectedToken.token.address || selectedToken.token.symbol, yieldIds: selectedToken.availableYields, }) ) diff --git a/packages/widget/src/resources/yield-directory/yield-directory.ts b/packages/widget/src/resources/yield-directory/yield-directory.ts index 013edc26f..2632a74bb 100644 --- a/packages/widget/src/resources/yield-directory/yield-directory.ts +++ b/packages/widget/src/resources/yield-directory/yield-directory.ts @@ -15,7 +15,9 @@ import { YieldResourceSource } from "../../services/api/resource-sources"; import { withApiResourcePolicy } from "../../shared/effect/api-resource"; import { API_MAX_PAGE_SIZE, + loadAllPages, loadAllPagesByIdChunks, + YIELD_IDS_CHUNK_SIZE, } from "../../shared/effect/pagination"; import { makePresentableResourceFamily } from "../resource-failure-presentation"; import type { YieldProviderError } from "../yield-provider/index"; @@ -27,18 +29,21 @@ type YieldType = EarnYield["mechanics"]["type"]; export class YieldDirectoryKey extends Data.TaggedClass("YieldDirectoryKey")<{ readonly network: Network | null; + readonly token: string | null; readonly types: ReadonlyArray; readonly yieldIds: ReadonlyArray; }> { constructor(input: { readonly network?: Network | null; + readonly token?: string | null; readonly types?: ReadonlyArray; - readonly yieldIds: ReadonlyArray; + readonly yieldIds?: ReadonlyArray; }) { super({ network: input.network ?? null, + token: input.token ?? null, types: [...new Set(input.types ?? [])].sort(), - yieldIds: [...new Set(input.yieldIds)].sort(), + yieldIds: [...new Set(input.yieldIds ?? [])].sort(), }); } } @@ -69,7 +74,7 @@ const yieldDirectoryCanonicalAtom = Atom.family((key: YieldDirectoryKey) => appRuntime .atom(() => Effect.gen(function* () { - if (key.yieldIds?.length === 0) { + if (!key.token && key.yieldIds?.length === 0) { return { items: [], missingYieldIds: [], @@ -82,18 +87,25 @@ const yieldDirectoryCanonicalAtom = Atom.family((key: YieldDirectoryKey) => limit: API_MAX_PAGE_SIZE, offset, ...(key.network ? { network: key.network } : {}), + ...(key.token ? { token: key.token } : {}), ...(key.types.length > 0 ? { types: key.types } : {}), - ...(yieldIds ? { yieldIds } : {}), + ...(yieldIds && yieldIds.length > 0 ? { yieldIds } : {}), }); - const items = yield* loadAllPagesByIdChunks({ - chunkSize: API_MAX_PAGE_SIZE, - concurrency: CONCURRENCY, - fetchPage: ({ ids, offset }) => fetchPage(offset, ids), - getItemId: (yieldModel) => yieldModel.id, - ids: key.yieldIds, - pageSize: API_MAX_PAGE_SIZE, - }); + const items = key.token + ? yield* loadAllPages({ + concurrency: CONCURRENCY, + fetchPage: (offset) => fetchPage(offset), + pageSize: API_MAX_PAGE_SIZE, + }) + : yield* loadAllPagesByIdChunks({ + chunkSize: YIELD_IDS_CHUNK_SIZE, + concurrency: CONCURRENCY, + fetchPage: ({ ids, offset }) => fetchPage(offset, ids), + getItemId: (yieldModel) => yieldModel.id, + ids: key.yieldIds, + pageSize: API_MAX_PAGE_SIZE, + }); const returnedIds = new Set(items.map((yieldModel) => yieldModel.id)); return { diff --git a/packages/widget/src/services/api/resource-sources.ts b/packages/widget/src/services/api/resource-sources.ts index caf3df04a..796f56fdd 100644 --- a/packages/widget/src/services/api/resource-sources.ts +++ b/packages/widget/src/services/api/resource-sources.ts @@ -54,6 +54,7 @@ export type YieldDirectoryRequest = { readonly limit: number; readonly network?: Network; readonly offset: number; + readonly token?: string; readonly types?: ReadonlyArray; readonly yieldIds?: ReadonlyArray; }; diff --git a/packages/widget/src/services/api/yield-resource-source.ts b/packages/widget/src/services/api/yield-resource-source.ts index c96b7184e..f6d0c5946 100644 --- a/packages/widget/src/services/api/yield-resource-source.ts +++ b/packages/widget/src/services/api/yield-resource-source.ts @@ -46,6 +46,7 @@ const toYieldDirectoryParams = (request: YieldDirectoryRequest) => ({ limit: request.limit, ...(request.network ? { network: request.network } : {}), offset: request.offset, + ...(request.token ? { token: request.token } : {}), ...(request.types ? { types: [...request.types] } : {}), ...(request.yieldIds ? { yieldIds: [...request.yieldIds] } : {}), }); diff --git a/packages/widget/src/shared/effect/pagination.ts b/packages/widget/src/shared/effect/pagination.ts index 798b2b752..2529ebdc2 100644 --- a/packages/widget/src/shared/effect/pagination.ts +++ b/packages/widget/src/shared/effect/pagination.ts @@ -26,6 +26,7 @@ type PaginationMetadata = { }; export const API_MAX_PAGE_SIZE = 100; +export const YIELD_IDS_CHUNK_SIZE = 20; export const getNextPageOffset = ({ limit, diff --git a/packages/widget/tests/atoms/atom-family-key-identity.test.ts b/packages/widget/tests/atoms/atom-family-key-identity.test.ts index 34e1b0bd7..d51a927b6 100644 --- a/packages/widget/tests/atoms/atom-family-key-identity.test.ts +++ b/packages/widget/tests/atoms/atom-family-key-identity.test.ts @@ -65,15 +65,18 @@ describe("atom family key identity", () => { const first = new YieldCatalogKey({ category: null, network: "ethereum", + token: "ETH", yieldIds: [secondYieldId, firstYieldId, secondYieldId], }); const equivalent = new YieldCatalogKey({ category: null, network: "ethereum", + token: "ETH", yieldIds: [firstYieldId, secondYieldId], }); expect(first.yieldIds).toEqual([firstYieldId, secondYieldId]); + expect(first.token).toBe("ETH"); expect(earnYieldCatalogAtom(first)).toBe(earnYieldCatalogAtom(equivalent)); }); diff --git a/packages/widget/tests/resources/yield-directory.test.ts b/packages/widget/tests/resources/yield-directory.test.ts index 08911a4ad..4b7a5336a 100644 --- a/packages/widget/tests/resources/yield-directory.test.ts +++ b/packages/widget/tests/resources/yield-directory.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it, vi } from "@effect/vitest"; import { Cause, Deferred, Effect, Layer, Option } from "effect"; import { AsyncResult, Atom, AtomRegistry } from "effect/unstable/reactivity"; import { appRuntime } from "../../src/app/runtime/app-runtime"; +import { YieldId } from "../../src/domain/identity/identifiers"; import { enrichedYieldDirectoryResourceAtom, YieldDirectoryError, @@ -77,13 +78,51 @@ describe("Yield Directory resource", () => { registry.get(yieldDirectoryResourceAtom(equivalent)) ).items ).toHaveLength(101); - expect(listYields).toHaveBeenCalledTimes(2); + expect(listYields).toHaveBeenCalledTimes(6); expect(listYields.mock.calls.map(([request]) => request.offset)).toEqual([ - 0, 0, + 0, 0, 0, 0, 0, 0, ]); expect( listYields.mock.calls.map(([request]) => request.yieldIds?.length) - ).toEqual([100, 1]); + ).toEqual([20, 20, 20, 20, 20, 1]); + }); + + it("loads yields by token using standard pagination without yieldIds in request", () => { + const yields = Array.from({ length: 150 }, (_, index) => + makeYield(`yield-${index}`) + ); + const listYields = vi.fn((request: YieldDirectoryRequest) => { + expect(request.yieldIds).toBeUndefined(); + expect(request.token).toBe("ETH"); + expect(request.network).toBe("ethereum"); + + return Effect.succeed({ + items: yields.slice(request.offset, request.offset + request.limit), + limit: request.limit, + offset: request.offset, + total: yields.length, + }); + }); + const registry = makeRegistry( + YieldResourceSource.of({ listYields } as never) + ); + const missingId = YieldId.make("yield-999"); + const key = new YieldDirectoryKey({ + network: "ethereum", + token: "ETH", + yieldIds: [yields[0]!.id, yields[149]!.id, missingId], + }); + + const result = AsyncResult.getOrThrow( + registry.get(yieldDirectoryResourceAtom(key)) + ); + + expect(result.items).toHaveLength(150); + expect(result.missingYieldIds).toEqual([missingId]); + expect(listYields).toHaveBeenCalledTimes(2); + expect(listYields.mock.calls.map(([request]) => request.offset)).toEqual([ + 0, 100, + ]); }); it("skips empty ID sets and distinguishes explicit directories", () => {