From dffbf41bbf40b4b5e455f9e47ae38ec1ba3d4b7c Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Sat, 17 Oct 2026 17:10:12 +0200 Subject: [PATCH] fix(stats): give Cloud and GitHub collectors separate request budgets --- .changeset/stats-collector-schedules.md | 6 ++++ stats/README.md | 7 +++-- stats/src/github.ts | 5 +++- stats/src/index.ts | 38 ++++++++++--------------- stats/test/worker.test.ts | 31 +++++++++++++++++++- stats/wrangler.jsonc | 2 +- 6 files changed, 61 insertions(+), 28 deletions(-) create mode 100644 .changeset/stats-collector-schedules.md diff --git a/.changeset/stats-collector-schedules.md b/.changeset/stats-collector-schedules.md new file mode 100644 index 0000000..478f288 --- /dev/null +++ b/.changeset/stats-collector-schedules.md @@ -0,0 +1,6 @@ +--- +"ftw-webapp": patch +--- + +Collect repository snapshots, daily GitHub traffic and public Cloud feeds in +separate scheduled runs so one source cannot use another's request budget. diff --git a/stats/README.md b/stats/README.md index ab8886e..3fe1e4e 100644 --- a/stats/README.md +++ b/stats/README.md @@ -65,8 +65,11 @@ UTC days. It uses one aggregate query per day to fit the zone's one-day query limit. It sends no browser script and stores no request rows, IP addresses or visitor ids. -The GitHub repository snapshot and site query run each hour. GitHub traffic -runs once a day because GitHub exposes it as a rolling daily window. Cloud usage collection also runs hourly. The retired relay no longer +Repository snapshots run at minute 3 each hour. Public Cloud feeds and site +traffic run at minute 23 in a separate invocation. GitHub traffic runs alone +at 00:17 UTC because GitHub exposes it as a rolling daily window. Keeping +these schedules separate leaves each its own external-request budget; the +combined six-repo snapshot and site queries can reach the 50-request limit. The retired relay no longer supplies an active source. Set `ACCESS_TEAM_DOMAIN` and `ACCESS_AUD` as Worker variables. Protect diff --git a/stats/src/github.ts b/stats/src/github.ts index 4e2fb8b..eec698e 100644 --- a/stats/src/github.ts +++ b/stats/src/github.ts @@ -43,6 +43,7 @@ export async function collectGitHub(env: AppEnv, daily: boolean, nowMs = Date.no let complete = 0 for (const repo of repos) { + if (!daily) { try { const snapshot = await collectSnapshot(owner, repo, token, nowMs) await saveSnapshot(env.DB, snapshot) @@ -56,10 +57,12 @@ export async function collectGitHub(env: AppEnv, daily: boolean, nowMs = Date.no await saveCollectorRun(env.DB, `github:${repo}`, startedAt, false, 'repository request failed') } - if (!daily) continue + continue + } try { await collectTraffic(env.DB, owner, repo, token, nowMs) await saveCollectorRun(env.DB, `github-traffic:${repo}`, startedAt, true, 'traffic window') + complete += 1 } catch (error) { console.error('stats: GitHub traffic collection failed', { repo, diff --git a/stats/src/index.ts b/stats/src/index.ts index 71d4f89..1712d5b 100644 --- a/stats/src/index.ts +++ b/stats/src/index.ts @@ -72,30 +72,22 @@ export default { }, async scheduled(controller: ScheduledController, env: AppEnv): Promise { - const daily = controller.cron === '17 0 * * *' const failures: string[] = [] - try { - await collectGitHub(env, daily, controller.scheduledTime) - } catch (error) { - failures.push('github') - console.error('stats: scheduled GitHub collection failed', { - kind: error instanceof Error ? error.name : 'unknown', - }) - } - try { - await collectSiteTraffic(env, controller.scheduledTime) - } catch (error) { - failures.push('site') - console.error('stats: scheduled site collection failed', { - kind: error instanceof Error ? error.name : 'unknown', - code: siteCollectorFailureCode(error), - }) - } - try { await collectUsage(env) } catch { - failures.push('cloud:usage'); console.error('stats: Cloud usage collection unavailable') - } - try {await collectFleet(env)} catch {failures.push('cloud:fleet')} - await pruneStoredData(env.DB, controller.scheduledTime) + // Each schedule gets its own external-request budget. Six repository + // snapshots plus seven analytics queries can consume the 50-request cap. + if (controller.cron === '3 * * * *' || controller.cron === '17 0 * * *') { + try { await collectGitHub(env, controller.cron === '17 0 * * *', controller.scheduledTime) } + catch { failures.push('github') } + } else if (controller.cron === '23 * * * *') { + try { await collectSiteTraffic(env, controller.scheduledTime) } + catch (error) { + failures.push('site') + console.error('stats: scheduled site collection failed', { code: siteCollectorFailureCode(error) }) + } + try { await collectUsage(env) } catch { failures.push('cloud:usage') } + try { await collectFleet(env) } catch { failures.push('cloud:fleet') } + } else { throw new Error('unknown collection schedule') } + await pruneStoredData(env.DB, controller.scheduledTime) if (failures.length > 0) throw new Error('one or more scheduled collectors failed') }, } satisfies ExportedHandler diff --git a/stats/test/worker.test.ts b/stats/test/worker.test.ts index beda2ee..03e559f 100644 --- a/stats/test/worker.test.ts +++ b/stats/test/worker.test.ts @@ -601,7 +601,7 @@ describe('project stats Worker', () => { }) await worker.scheduled( - { cron: '3 * * * *', scheduledTime: Date.UTC(2026, 7, 16, 9, 3), noRetry() {} }, + { cron: '23 * * * *', scheduledTime: Date.UTC(2026, 7, 16, 9, 23), noRetry() {} }, env ) @@ -616,6 +616,35 @@ describe('project stats Worker', () => { }) }) +it('keeps six-repo snapshots, daily traffic and public Cloud feeds within separate 50-request budgets', async () => { + const repos=['ftw','device-drivers','ftw-cloud','ftw-app','ftw-webapp','ftw-web'] + let requests=0 + vi.stubGlobal('fetch',async(input:RequestInfo|URL)=>{ + if(++requests>50) throw new Error('Too many subrequests.') + const url=new URL(typeof input==='string'?input:input instanceof URL?input:input.url) + if(url.hostname==='api.cloudflare.com') return Response.json({data:{viewer:{zones:[{traffic:[]}]}},errors:null}) + if(url.hostname==='cloud.ftw.energy') return url.pathname==='/v1/stats' + ? new Response(null,{status:503,headers:{'FTW-Statistics-State':'disabled'}}) + : Response.json(fleetView(Date.now())) + if(url.pathname==='/search/issues') return Response.json({total_count:0}) + if(url.pathname.endsWith('/traffic/views')) return Response.json({views:[]}) + if(url.pathname.endsWith('/traffic/clones')) return Response.json({clones:[]}) + if(url.pathname.includes('/traffic/')) return Response.json([]) + if(url.pathname==='/repos/srcfl/ftw/issues' && url.searchParams.get('page')==='1') return Response.json(Array.from({length:100},(_,number)=>({number}))) + if(url.pathname.split('/').length===4) return Response.json({stargazers_count:0,forks_count:0,subscribers_count:0,pushed_at:null}) + return Response.json([]) + }) + const all={...env,GITHUB_REPOS:repos.join(',')} + for(const [cron,count] of [['3 * * * *',43],['23 * * * *',9],['17 0 * * *',24]] as const){ + requests=0 + await worker.scheduled({cron,scheduledTime:Date.now(),noRetry(){}},all) + expect(requests).toBe(count) + } + const runs=await env.DB.prepare('SELECT source,ok FROM collector_runs').all<{source:string;ok:number}>() + expect(runs.results).toHaveLength(15) + expect(runs.results.every(r=>r.ok===1)).toBe(true) +}) + function relayPayload(now: Date, reports: number): RelayIngestBody { return { schema: 'ftw.relay-stats/1', diff --git a/stats/wrangler.jsonc b/stats/wrangler.jsonc index 8578d97..b3268d1 100644 --- a/stats/wrangler.jsonc +++ b/stats/wrangler.jsonc @@ -29,7 +29,7 @@ } ], "triggers": { - "crons": ["3 * * * *", "17 0 * * *"] + "crons": ["3 * * * *", "23 * * * *", "17 0 * * *"] }, "observability": { "enabled": true,