fix(stats): keep scheduled collectors within separate request budgets

.changeset/stats-collector-schedules.md

@@ -0,0 +1,6 @@
+---
+"ftw-webapp": patch
+---
+
+Collect repository snapshots, daily GitHub traffic and public Cloud feeds in
+separate scheduled runs so one source cannot use another's request budget.

stats/README.md

@@ -65,8 +65,11 @@ UTC days. It uses one aggregate query per day to fit the zone's one-day query
 limit. It sends no
 browser script and stores no request rows, IP addresses or visitor ids.
 
-The GitHub repository snapshot and site query run each hour. GitHub traffic
-runs once a day because GitHub exposes it as a rolling daily window. Cloud usage collection also runs hourly. The retired relay no longer
+Repository snapshots run at minute 3 each hour. Public Cloud feeds and site
+traffic run at minute 23 in a separate invocation. GitHub traffic runs alone
+at 00:17 UTC because GitHub exposes it as a rolling daily window. Keeping
+these schedules separate leaves each its own external-request budget; the
+combined six-repo snapshot and site queries can reach the 50-request limit. The retired relay no longer
 supplies an active source.
 
 Set `ACCESS_TEAM_DOMAIN` and `ACCESS_AUD` as Worker variables. Protect

stats/src/github.ts

@@ -43,6 +43,7 @@ export async function collectGitHub(env: AppEnv, daily: boolean, nowMs = Date.no
   let complete = 0
 
   for (const repo of repos) {
+    if (!daily) {
     try {
       const snapshot = await collectSnapshot(owner, repo, token, nowMs)
       await saveSnapshot(env.DB, snapshot)
@@ -56,10 +57,12 @@ export async function collectGitHub(env: AppEnv, daily: boolean, nowMs = Date.no
       await saveCollectorRun(env.DB, `github:${repo}`, startedAt, false, 'repository request failed')
     }
 
-    if (!daily) continue
+    continue
+    }
     try {
       await collectTraffic(env.DB, owner, repo, token, nowMs)
       await saveCollectorRun(env.DB, `github-traffic:${repo}`, startedAt, true, 'traffic window')
+      complete += 1
     } catch (error) {
       console.error('stats: GitHub traffic collection failed', {
         repo,

stats/src/index.ts

@@ -72,30 +72,22 @@ export default {
   },
 
   async scheduled(controller: ScheduledController, env: AppEnv): Promise<void> {
-    const daily = controller.cron === '17 0 * * *'
     const failures: string[] = []
-    try {
-      await collectGitHub(env, daily, controller.scheduledTime)
-    } catch (error) {
-      failures.push('github')
-      console.error('stats: scheduled GitHub collection failed', {
-        kind: error instanceof Error ? error.name : 'unknown',
-      })
-    }
-    try {
-      await collectSiteTraffic(env, controller.scheduledTime)
-    } catch (error) {
-      failures.push('site')
-      console.error('stats: scheduled site collection failed', {
-        kind: error instanceof Error ? error.name : 'unknown',
-        code: siteCollectorFailureCode(error),
-      })
-    }
-    try { await collectUsage(env) } catch {
- failures.push('cloud:usage'); console.error('stats: Cloud usage collection unavailable')
- }
- try {await collectFleet(env)} catch {failures.push('cloud:fleet')}
- await pruneStoredData(env.DB, controller.scheduledTime)
+    // Each schedule gets its own external-request budget. Six repository
+    // snapshots plus seven analytics queries can consume the 50-request cap.
+    if (controller.cron === '3 * * * *' || controller.cron === '17 0 * * *') {
+      try { await collectGitHub(env, controller.cron === '17 0 * * *', controller.scheduledTime) }
+      catch { failures.push('github') }
+    } else if (controller.cron === '23 * * * *') {
+      try { await collectSiteTraffic(env, controller.scheduledTime) }
+      catch (error) {
+        failures.push('site')
+        console.error('stats: scheduled site collection failed', { code: siteCollectorFailureCode(error) })
+      }
+      try { await collectUsage(env) } catch { failures.push('cloud:usage') }
+      try { await collectFleet(env) } catch { failures.push('cloud:fleet') }
+    } else { throw new Error('unknown collection schedule') }
+    await pruneStoredData(env.DB, controller.scheduledTime)
     if (failures.length > 0) throw new Error('one or more scheduled collectors failed')
   },
 } satisfies ExportedHandler<AppEnv>

stats/test/worker.test.ts

@@ -601,7 +601,7 @@ describe('project stats Worker', () => {
     })
 
     await worker.scheduled(
-      { cron: '3 * * * *', scheduledTime: Date.UTC(2026, 7, 16, 9, 3), noRetry() {} },
+      { cron: '23 * * * *', scheduledTime: Date.UTC(2026, 7, 16, 9, 23), noRetry() {} },
       env
     )
 
@@ -616,6 +616,35 @@ describe('project stats Worker', () => {
   })
 })
 
+it('keeps six-repo snapshots, daily traffic and public Cloud feeds within separate 50-request budgets', async () => {
+  const repos=['ftw','device-drivers','ftw-cloud','ftw-app','ftw-webapp','ftw-web']
+  let requests=0
+  vi.stubGlobal('fetch',async(input:RequestInfo|URL)=>{
+    if(++requests>50) throw new Error('Too many subrequests.')
+    const url=new URL(typeof input==='string'?input:input instanceof URL?input:input.url)
+    if(url.hostname==='api.cloudflare.com') return Response.json({data:{viewer:{zones:[{traffic:[]}]}},errors:null})
+    if(url.hostname==='cloud.ftw.energy') return url.pathname==='/v1/stats'
+      ? new Response(null,{status:503,headers:{'FTW-Statistics-State':'disabled'}})
+      : Response.json(fleetView(Date.now()))
+    if(url.pathname==='/search/issues') return Response.json({total_count:0})
+    if(url.pathname.endsWith('/traffic/views')) return Response.json({views:[]})
+    if(url.pathname.endsWith('/traffic/clones')) return Response.json({clones:[]})
+    if(url.pathname.includes('/traffic/')) return Response.json([])
+    if(url.pathname==='/repos/srcfl/ftw/issues' && url.searchParams.get('page')==='1') return Response.json(Array.from({length:100},(_,number)=>({number})))
+    if(url.pathname.split('/').length===4) return Response.json({stargazers_count:0,forks_count:0,subscribers_count:0,pushed_at:null})
+    return Response.json([])
+  })
+  const all={...env,GITHUB_REPOS:repos.join(',')}
+  for(const [cron,count] of [['3 * * * *',43],['23 * * * *',9],['17 0 * * *',24]] as const){
+    requests=0
+    await worker.scheduled({cron,scheduledTime:Date.now(),noRetry(){}},all)
+    expect(requests).toBe(count)
+  }
+  const runs=await env.DB.prepare('SELECT source,ok FROM collector_runs').all<{source:string;ok:number}>()
+  expect(runs.results).toHaveLength(15)
+  expect(runs.results.every(r=>r.ok===1)).toBe(true)
+})
+
 function relayPayload(now: Date, reports: number): RelayIngestBody {
   return {
     schema: 'ftw.relay-stats/1',

stats/wrangler.jsonc

@@ -29,7 +29,7 @@
     }
   ],
   "triggers": {
-    "crons": ["3 * * * *", "17 0 * * *"]
+    "crons": ["3 * * * *", "23 * * * *", "17 0 * * *"]
   },
   "observability": {
     "enabled": true,