diff --git a/sources/feverup.com/fetch-plan-grid-page.js b/sources/feverup.com/fetch-plan-grid-page.js new file mode 100644 index 00000000..415dba08 --- /dev/null +++ b/sources/feverup.com/fetch-plan-grid-page.js @@ -0,0 +1,55 @@ +const crypto = require("node:crypto"); +const { + fetchJson, + sleep, + withJitter, + withRetry, +} = require("../../common/utils.js"); +const { dailyCache } = require("../../common/cache.js"); + +// Fever's API answers an error with the same `cache-control: public, +// max-age=1800` as a success, so CloudFront keeps a 500 for half an hour and +// hands it to everyone asking for that exact URL. Retrying the URL is then +// pointless: on 11 November 2026 a single 500 for catalogue page 2 voided the +// retrieve, because the two step retries replayed the cached listing page, +// rebuilt byte-for-byte the same URL, and were served the stored error within a +// second each without ever reaching Fever. +// +// The recommendation tracker id is part of that cache key, while Fever's own +// server ignores it - page 2 comes back the same with a made-up id, or one +// that isn't a UUID at all. So each retry asks with a fresh id, which misses +// the stored error and reaches the origin. The first attempt keeps the id the +// listing page gave, which is what Fever's own page sends. +// +// A 500 here is retried because this endpoint is known to serve a transient +// one; it stays permanent everywhere else. +const PLAN_GRID_RETRY_CONFIG = { + retries: 3, + delayMs: 5_000, + backoffFactor: 2, +}; +const RETRYABLE_PLAN_GRID_STATUSES = new Set([500]); + +/** + * One page of the catalogue's plan grid, through the daily cache. Only a + * success is cached, so a failure never pins a later run to the failing id. + */ +function fetchPlanGridPage({ getUrl, trackerId, page, headers, delayMs }) { + return dailyCache(`feverup-catalogue-page-${page}`, async () => { + let attempt = 0; + return withRetry( + async () => { + const id = attempt++ === 0 ? trackerId : crypto.randomUUID(); + await sleep(withJitter(delayMs)); + return fetchJson(getUrl(id, page), { headers }); + }, + { + ...PLAN_GRID_RETRY_CONFIG, + label: `Fever catalogue page ${page}`, + shouldRetry: (error) => RETRYABLE_PLAN_GRID_STATUSES.has(error.status), + }, + ); + }); +} + +module.exports = fetchPlanGridPage; diff --git a/sources/feverup.com/retrieve.js b/sources/feverup.com/retrieve.js index 0da7fd0f..fb0f39e3 100644 --- a/sources/feverup.com/retrieve.js +++ b/sources/feverup.com/retrieve.js @@ -14,6 +14,7 @@ const { extractPlanGrid, isFallbackListing, } = require("./utils"); +const fetchPlanGridPage = require("./fetch-plan-grid-page"); // Fever files a city's plans under "what plan filters" (its categories), and // its Cinema filter can't be relied on to hold the screenings: a dinner served @@ -141,11 +142,13 @@ async function fetchCataloguePlanIds(movieListPage) { if (hasMultiplePages) { let page = 2; while (true) { - const { results, next_page: nextPage } = await fetchCached( - `feverup-catalogue-page-${page}`, - fetchJson, - getPlanGridApiUrl(trackerId, page), - ); + const { results, next_page: nextPage } = await fetchPlanGridPage({ + getUrl: getPlanGridApiUrl, + trackerId, + page, + headers: REQUEST_HEADERS, + delayMs: REQUEST_DELAY_MS, + }); cataloguePlans.push(...results); if (!nextPage) break; page = nextPage; diff --git a/sources/feverup.com/tests/fetch-plan-grid-page.test.js b/sources/feverup.com/tests/fetch-plan-grid-page.test.js new file mode 100644 index 00000000..23493d91 --- /dev/null +++ b/sources/feverup.com/tests/fetch-plan-grid-page.test.js @@ -0,0 +1,70 @@ +const { fetchJson } = require("../../../common/utils"); +const fetchPlanGridPage = require("../fetch-plan-grid-page"); + +jest.mock("../../../common/utils", () => ({ + ...jest.requireActual("../../../common/utils"), + fetchJson: jest.fn(), +})); +jest.mock("../../../common/cache", () => ({ + dailyCache: jest.fn((key, retrieve) => retrieve()), +})); + +const httpError = (status) => + Object.assign(new Error(`Failed - ${status}`), { status }); + +const getUrl = (trackerId, page) => `grid?page=${page}&tracker=${trackerId}`; +const trackerOf = (url) => + new URL(url, "https://x").searchParams.get("tracker"); +// Runs the retries' backoff on fake timers, so the test doesn't wait it out. +const fetchPage = async () => { + const result = fetchPlanGridPage({ + getUrl, + trackerId: "from-listing", + page: 2, + headers: {}, + delayMs: 0, + }); + result.catch(() => {}); + await jest.runAllTimersAsync(); + return result; +}; + +describe("fetchPlanGridPage", () => { + beforeEach(() => { + jest.useFakeTimers(); + fetchJson.mockReset(); + jest.spyOn(console, "log").mockImplementation(() => {}); + }); + + it("asks with the listing page's tracker id first", async () => { + fetchJson.mockResolvedValueOnce({ results: [] }); + await expect(fetchPage()).resolves.toEqual({ results: [] }); + expect(fetchJson).toHaveBeenCalledTimes(1); + expect(trackerOf(fetchJson.mock.calls[0][0])).toBe("from-listing"); + }); + + it("retries a 500 with a fresh tracker id each time, so it misses the cached error", async () => { + fetchJson + .mockRejectedValueOnce(httpError(500)) + .mockRejectedValueOnce(httpError(500)) + .mockResolvedValueOnce({ results: [{ id: 1 }] }); + await expect(fetchPage()).resolves.toEqual({ results: [{ id: 1 }] }); + + const trackers = fetchJson.mock.calls.map(([url]) => trackerOf(url)); + expect(trackers).toHaveLength(3); + expect(trackers[0]).toBe("from-listing"); + expect(new Set(trackers).size).toBe(3); + }); + + it("does not retry other failures", async () => { + fetchJson.mockRejectedValueOnce(httpError(400)); + await expect(fetchPage()).rejects.toMatchObject({ status: 400 }); + expect(fetchJson).toHaveBeenCalledTimes(1); + }); + + it("fails once the retries are spent", async () => { + fetchJson.mockRejectedValue(httpError(500)); + await expect(fetchPage()).rejects.toMatchObject({ status: 500 }); + expect(fetchJson).toHaveBeenCalledTimes(4); + }); +});