diff --git a/.changeset/project-growth-stats.md b/.changeset/project-growth-stats.md new file mode 100644 index 0000000..b2a929d --- /dev/null +++ b/.changeset/project-growth-stats.md @@ -0,0 +1,5 @@ +--- +"ftw-webapp": minor +--- + +Add a private project dashboard and privacy-bounded public growth statistics. diff --git a/.changeset/stats-cloud-coverage.md b/.changeset/stats-cloud-coverage.md new file mode 100644 index 0000000..26142f7 --- /dev/null +++ b/.changeset/stats-cloud-coverage.md @@ -0,0 +1,6 @@ +--- +"ftw-webapp": patch +--- + +Track all six active public FTW repositories and published beta releases. +State which usage figures are missing, and keep legacy relay load in the private view. diff --git a/.changeset/stats-energy-usage.md b/.changeset/stats-energy-usage.md new file mode 100644 index 0000000..eade011 --- /dev/null +++ b/.changeset/stats-energy-usage.md @@ -0,0 +1,7 @@ +--- +"ftw-webapp": patch +--- + +Restore GitHub history across repository coverage changes. Mark relay as +retired and add validated Cloud group cells for reporting installations, +configured capacity and recorded energy, with explicit missing-data states. diff --git a/package-lock.json b/package-lock.json index 84272ea..f7cef3c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,10 +14,13 @@ "@noble/hashes": "^2.2.0", "cbor2": "^2.3.0", "idb": "^8.0.3", + "jose": "^6.2.8", "jsqr": "^1.4.0", "ws": "^8.21.2" }, "devDependencies": { + "@cloudflare/vitest-pool-workers": "^0.21.3", + "@cloudflare/workers-types": "^5.20260814.1", "@sveltejs/vite-plugin-svelte": "^7.2.0", "@testing-library/svelte": "^5.4.2", "@types/node": "^26.1.2", @@ -142,10 +145,29 @@ } } }, + "node_modules/@cloudflare/vitest-pool-workers": { + "version": "0.21.3", + "resolved": "https://registry.npmjs.org/@cloudflare/vitest-pool-workers/-/vitest-pool-workers-0.21.3.tgz", + "integrity": "sha512-jCoGRQ6FlsP5adp1GJISvITOGdTHTpsWOq3KkSGIfeDY/5RKjTUagDwaXjpqBXY5gAk6id/QbgnGuCTAg6lSTQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "cjs-module-lexer": "1.2.3", + "esbuild": "0.28.1", + "miniflare": "5.20260811.1-alpha", + "wrangler": "4.123.0", + "zod": "4.4.3" + }, + "peerDependencies": { + "@vitest/runner": "^4.1.0", + "@vitest/snapshot": "^4.1.0", + "vitest": "^4.1.0" + } + }, "node_modules/@cloudflare/workerd-darwin-64": { - "version": "1.20260730.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-64/-/workerd-darwin-64-1.20260730.1.tgz", - "integrity": "sha512-+MBHmPaiTe2KajryW0T24rZvWFxb41hD3d8anNzQqHzft6vSEb18+sp0znSwxgij7ApPhSM1+vhkNg4f3YMguA==", + "version": "1.20260811.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-64/-/workerd-darwin-64-1.20260811.1.tgz", + "integrity": "sha512-i5jqz+ywtOefr0AJbiAc8qxBLfSim/B0WJG7aW3B+pWnoVfMJdUQvi+BWcFKZJ0MoCci3KadTx6g31VfuEEqpQ==", "cpu": [ "x64" ], @@ -160,9 +182,9 @@ } }, "node_modules/@cloudflare/workerd-darwin-arm64": { - "version": "1.20260730.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-arm64/-/workerd-darwin-arm64-1.20260730.1.tgz", - "integrity": "sha512-SBHKntPkKvNPgaCrTe99xC1CAl8ygJDzlYfK0LbuJ1muKadIw35WnhO0wu894fKBtllsVQdNzDLee+cm0ppLSQ==", + "version": "1.20260811.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-arm64/-/workerd-darwin-arm64-1.20260811.1.tgz", + "integrity": "sha512-NoOUM/nvaDdm2Onlnz33FikWjtatzulNtvwvy4xs0IrHaTCHwC0c8NwIt6s+AI13FkDs02/vm2I3GTPLCT9+hQ==", "cpu": [ "arm64" ], @@ -177,9 +199,9 @@ } }, "node_modules/@cloudflare/workerd-linux-64": { - "version": "1.20260730.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-64/-/workerd-linux-64-1.20260730.1.tgz", - "integrity": "sha512-ouyPOSMbiKPeSwUJUvxtMcxGAXs2J4aPE4T5ABIYX5ClcQx5j5bbHTmnqOQEY8sAuLTPjH7dY+iB6UI5ISlwwA==", + "version": "1.20260811.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-64/-/workerd-linux-64-1.20260811.1.tgz", + "integrity": "sha512-sdYq2jL1AD1supa3fsi5O4zTB28wSjvTHj7Migh6/ts8EROPdvrSwv+rdGHhv8HJNAz/wbIAY3wZsi1Rw4uUIg==", "cpu": [ "x64" ], @@ -194,9 +216,9 @@ } }, "node_modules/@cloudflare/workerd-linux-arm64": { - "version": "1.20260730.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-arm64/-/workerd-linux-arm64-1.20260730.1.tgz", - "integrity": "sha512-YQ+Mi78U3TPdgBPtwq+Sm6rJU+Ihl2y0pjYtuuKkdmUbYzL7oLR6Xqq9wljhasnuCFICssDJaqhMep5WizYoEQ==", + "version": "1.20260811.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-arm64/-/workerd-linux-arm64-1.20260811.1.tgz", + "integrity": "sha512-RIRv4shbu1kg05sD+DHTpSFCNnb5Dl2SkPDMUykqZa508tkPqe7VVw7gO0Q5msTBGyL0FfFrLuRxwwfA8u5Sow==", "cpu": [ "arm64" ], @@ -211,9 +233,9 @@ } }, "node_modules/@cloudflare/workerd-windows-64": { - "version": "1.20260730.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-windows-64/-/workerd-windows-64-1.20260730.1.tgz", - "integrity": "sha512-27fAN+vUECW1oYVc1KOcHYpkL8COM2Uxtxql7TL595kxbjoqS5yckw7NLz7bTf2pALFCZWjqXDjZGJ/xbG4ZKQ==", + "version": "1.20260811.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-windows-64/-/workerd-windows-64-1.20260811.1.tgz", + "integrity": "sha512-g6VquwjASlYAibcNW/0E6Zszht4qLkmnXOGwIjjRHl2A0Qz48kVeMcGvyH6eA0G9U3OzZojjYFpP+YeyQmmdjw==", "cpu": [ "x64" ], @@ -227,6 +249,13 @@ "node": ">=16" } }, + "node_modules/@cloudflare/workers-types": { + "version": "5.20260814.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workers-types/-/workers-types-5.20260814.1.tgz", + "integrity": "sha512-bF5RT5bmxEaHJR2PRrAmcdf09EPxpRXXzPHcOtjX//r84jZNpwslNwx291ofNdBVuqGV129CBJzxiMYsB37DHw==", + "dev": true, + "license": "MIT OR Apache-2.0" + }, "node_modules/@cspotcode/source-map-support": { "version": "0.8.1", "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", @@ -2257,6 +2286,13 @@ "url": "https://paulmillr.com/funding/" } }, + "node_modules/cjs-module-lexer": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-1.2.3.tgz", + "integrity": "sha512-0TNiGstbQmCFwt4akjjBg5pLRTSyj/PkWQ1ZoO2zntmg9yLqSRxwEa4iCfQLGjqhiqBfOJa7W/E8wfGrTDmlZQ==", + "dev": true, + "license": "MIT" + }, "node_modules/clsx": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz", @@ -2578,6 +2614,15 @@ "@types/estree": "^1.0.6" } }, + "node_modules/jose": { + "version": "6.2.8", + "resolved": "https://registry.npmjs.org/jose/-/jose-6.2.8.tgz", + "integrity": "sha512-Bsdjwm3Qsd/P0jR+BHDe3LytDfY7WBq2HmCCLIwuVRHMuEC9ae7/R474GIUdF1NgCyZjzVo/A9DOiOBtXq8ZoQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/panva" + } + }, "node_modules/js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", @@ -2960,16 +3005,16 @@ "license": "CC0-1.0" }, "node_modules/miniflare": { - "version": "5.20260730.0-alpha", - "resolved": "https://registry.npmjs.org/miniflare/-/miniflare-5.20260730.0-alpha.tgz", - "integrity": "sha512-8/dspSXDshP6nSkCpjKO7BYc2qZoYSXm7iM+QxY7qJyJpAB3onnQSaiu0cvKJlfuMGwULl55hG69FJCcCMXU1Q==", + "version": "5.20260811.1-alpha", + "resolved": "https://registry.npmjs.org/miniflare/-/miniflare-5.20260811.1-alpha.tgz", + "integrity": "sha512-DtOG0BeanIxs2sH0smFvExZD89cBQwGckbHiFkRJrrNAUu3NGClZkUxqu+zy7HYfKBAgq935EMY49vIPm3JVdA==", "dev": true, "license": "MIT", "dependencies": { "@cspotcode/source-map-support": "0.8.1", "sharp": "0.35.2", - "undici": "7.28.0", - "workerd": "1.20260730.1", + "undici": "7.29.0", + "workerd": "1.20260811.1", "ws": "8.21.0", "youch": "4.1.0-beta.10" }, @@ -2978,9 +3023,9 @@ } }, "node_modules/miniflare/node_modules/undici": { - "version": "7.28.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-7.28.0.tgz", - "integrity": "sha512-cRZYrTDwWznlnRiPjggAGxZXanty6M8RV1ff8Wm4LWXBp7/IG8v5DnOm74DtUBp9OONpK75YlPnIjQqX0dBDtA==", + "version": "7.29.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.29.0.tgz", + "integrity": "sha512-IDxfleLmmbSskfWSUATiN1nfn2rDuvnMOqb5CWR92iIfojA0Ud+ulOAAEQ57LPr9rWmsreUyf5lwyao+7GNNVw==", "dev": true, "license": "MIT", "engines": { @@ -3020,9 +3065,9 @@ } }, "node_modules/nanoid": { - "version": "3.3.17", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.17.tgz", - "integrity": "sha512-xQLf0A3HOMlgHq0n247/LRuAOYmB7dXJ/DvAxGvsSBij45XtBSmQycu+F8ODbHwns/XyFZagyL1+J0Offw1E0g==", + "version": "3.3.18", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.18.tgz", + "integrity": "sha512-DTg4MJbGMWkfi6VZFdNt2/caMbQy4Ou+Op/hJQvGEWcnVfoA1QA+xzRKAzw9jD6+GVOOeYr/mIcuDSdug6F6+w==", "dev": true, "funding": [ { @@ -3808,9 +3853,9 @@ } }, "node_modules/workerd": { - "version": "1.20260730.1", - "resolved": "https://registry.npmjs.org/workerd/-/workerd-1.20260730.1.tgz", - "integrity": "sha512-zmfNIjwYSWFY5chGBOjWtH3xAE7p97FTC6vR4Ep98290ho6AeAR/NVcBD274YCLEUYzqm8yxdtZlxMybU8a3jA==", + "version": "1.20260811.1", + "resolved": "https://registry.npmjs.org/workerd/-/workerd-1.20260811.1.tgz", + "integrity": "sha512-kh+FFm55JQ4ssxhHZV9VPdMQq3D1nHxNJgwxMtWGD4dGppJvLySdguTRDKgeNTvgq6heSz+6TTXyPSDGj8Yllw==", "dev": true, "hasInstallScript": true, "license": "Apache-2.0", @@ -3821,17 +3866,17 @@ "node": ">=16" }, "optionalDependencies": { - "@cloudflare/workerd-darwin-64": "1.20260730.1", - "@cloudflare/workerd-darwin-arm64": "1.20260730.1", - "@cloudflare/workerd-linux-64": "1.20260730.1", - "@cloudflare/workerd-linux-arm64": "1.20260730.1", - "@cloudflare/workerd-windows-64": "1.20260730.1" + "@cloudflare/workerd-darwin-64": "1.20260811.1", + "@cloudflare/workerd-darwin-arm64": "1.20260811.1", + "@cloudflare/workerd-linux-64": "1.20260811.1", + "@cloudflare/workerd-linux-arm64": "1.20260811.1", + "@cloudflare/workerd-windows-64": "1.20260811.1" } }, "node_modules/wrangler": { - "version": "4.118.0", - "resolved": "https://registry.npmjs.org/wrangler/-/wrangler-4.118.0.tgz", - "integrity": "sha512-9pkBw/b8zWqGx2S+oLhgHMR1M/4VOE8SynUFABnGWiSFGlcOQ4xiI/B71Xf66RYP2xzngU37IQFPtUruij3lYw==", + "version": "4.123.0", + "resolved": "https://registry.npmjs.org/wrangler/-/wrangler-4.123.0.tgz", + "integrity": "sha512-VXo2I1oa0x9aGAKIFPRSQPqTh0RBY5Ktl44YOhNmsJQFUdJKDA2vVTU6Xj+FC2koll6orJqWZN8jbXVIk9O67Q==", "dev": true, "license": "MIT OR Apache-2.0", "dependencies": { @@ -3839,10 +3884,10 @@ "@cloudflare/unenv-preset": "2.16.1", "blake3-wasm": "2.1.5", "esbuild": "0.28.1", - "miniflare": "5.20260730.0-alpha", + "miniflare": "5.20260811.1-alpha", "path-to-regexp": "6.3.0", "unenv": "2.0.0-rc.24", - "workerd": "1.20260730.1" + "workerd": "1.20260811.1" }, "bin": { "cf-wrangler": "bin/cf-wrangler.js", @@ -3856,7 +3901,7 @@ "fsevents": "2.3.3" }, "peerDependencies": { - "@cloudflare/workers-types": "^5.20260730.1" + "@cloudflare/workers-types": "^5.20260811.1" }, "peerDependenciesMeta": { "@cloudflare/workers-types": { @@ -3933,6 +3978,16 @@ "integrity": "sha512-B58NGBEoc8Y9MWWCQGl/gq9xBCe4IiKM0a2x7GZdQKOW5Exr8S1W24J6OgM1njK8xCRGvAJIL/MxXHf6SkmQKQ==", "dev": true, "license": "MIT" + }, + "node_modules/zod": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.4.3.tgz", + "integrity": "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } } } } diff --git a/package.json b/package.json index 6772352..0ac6e62 100644 --- a/package.json +++ b/package.json @@ -11,17 +11,23 @@ "scripts": { "dev": "vite", "build": "vite build", + "build:stats": "WRANGLER_WRITE_LOGS=0 wrangler deploy --dry-run --config stats/wrangler.jsonc --outdir stats/dist", "preview": "vite preview", "relay": "node relay/src/main.ts", "icons": "node scripts/make-icons.mjs", - "check": "svelte-check --tsconfig ./tsconfig.json", + "check": "svelte-check --tsconfig ./tsconfig.json && npm run check:stats", + "check:stats": "tsc -p stats/tsconfig.json && npm run types:stats -- --check", "check:contract": "node scripts/check-contract-drift.mjs", "check:tokens": "node scripts/check-token-drift.mjs", - "test": "vitest run", + "test": "vitest run && npm run test:stats", + "test:stats": "WRANGLER_WRITE_LOGS=0 vitest run --config stats/vitest.config.ts", + "types:stats": "WRANGLER_WRITE_LOGS=0 wrangler types stats/worker-configuration.d.ts --config stats/wrangler.jsonc --include-runtime false", "test:watch": "vitest", - "verify": "npm run check && npm run test && npm run build" + "verify": "npm run check && npm run test && npm run build && npm run build:stats" }, "devDependencies": { + "@cloudflare/vitest-pool-workers": "^0.21.3", + "@cloudflare/workers-types": "^5.20260814.1", "@sveltejs/vite-plugin-svelte": "^7.2.0", "@testing-library/svelte": "^5.4.2", "@types/node": "^26.1.2", @@ -41,6 +47,7 @@ "@noble/hashes": "^2.2.0", "cbor2": "^2.3.0", "idb": "^8.0.3", + "jose": "^6.2.8", "jsqr": "^1.4.0", "ws": "^8.21.2" } diff --git a/relay/deploy/.env.example b/relay/deploy/.env.example new file mode 100644 index 0000000..5b94367 --- /dev/null +++ b/relay/deploy/.env.example @@ -0,0 +1,2 @@ +RELAY_STATS_EXPORT_URL=https://stats.ftw.energy/api/ingest/relay +RELAY_STATS_EXPORT_SECRET=replace_with_the_same_32_byte_secret_as_the_worker diff --git a/relay/deploy/README.md b/relay/deploy/README.md index 565a4b3..6172e51 100644 --- a/relay/deploy/README.md +++ b/relay/deploy/README.md @@ -87,6 +87,13 @@ curl -s http://127.0.0.1:8787/fleet/stats The response says `reports, not unique boxes`. The public `https://relay.ftw.energy/fleet/stats` route returns 404. +Project stats export is optional. Set both `RELAY_STATS_EXPORT_URL` and +`RELAY_STATS_EXPORT_SECRET` in the deploy `.env` file to send aggregate counts +to `stats.ftw.energy` every five minutes. The body has rooms, sockets, frame and +byte counters, process uptime, and up to fourteen days of the daily fleet +totals above. It has no handles, addresses, ids or routed frames. A failed +export does not stop relay traffic. + Two things worth an alert: - `/healthz` failing. The container restarts itself, but a loop means something diff --git a/relay/deploy/compose.yml b/relay/deploy/compose.yml index d71d8cb..fb30d72 100644 --- a/relay/deploy/compose.yml +++ b/relay/deploy/compose.yml @@ -29,6 +29,10 @@ services: # volume. Fleet request bodies and source addresses are never written. RELAY_DEADMAN_PATH: /state/deadman.json RELAY_FLEET_STATS_PATH: /state/fleet-stats.json + # Optional. When both exist, the relay sends aggregate process counters + # and daily fleet totals. It never sends handles, addresses or frames. + RELAY_STATS_EXPORT_URL: ${RELAY_STATS_EXPORT_URL:-} + RELAY_STATS_EXPORT_SECRET: ${RELAY_STATS_EXPORT_SECRET:-} volumes: - relay-state:/state logging: diff --git a/relay/src/main.ts b/relay/src/main.ts index ed3aa0e..3e07393 100644 --- a/relay/src/main.ts +++ b/relay/src/main.ts @@ -7,6 +7,7 @@ */ import { RelayServer } from './server.ts' +import { RelayStatsExporter } from './stats-export.ts' const relay = await RelayServer.start({ port: Number(process.env['PORT'] ?? 8787), @@ -29,8 +30,28 @@ const relay = await RelayServer.start({ console.log(`relay listening on ${relay.url}`) +const statsUrl = process.env['RELAY_STATS_EXPORT_URL'] ?? '' +const statsSecret = process.env['RELAY_STATS_EXPORT_SECRET'] ?? '' +let statsExporter: RelayStatsExporter | null = null +if (statsUrl && statsSecret) { + try { + statsExporter = new RelayStatsExporter({ + url: statsUrl, + secret: statsSecret, + snapshot: () => relay.aggregateStats(), + log: (line) => console.log(new Date().toISOString() + ' ' + line), + }) + statsExporter.start() + } catch { + console.log(new Date().toISOString() + ' relay: stats export configuration rejected') + } +} else if (statsUrl || statsSecret) { + console.log(new Date().toISOString() + ' relay: stats export configuration incomplete') +} + for (const signal of ['SIGINT', 'SIGTERM'] as const) { process.on(signal, () => { + statsExporter?.stop() void relay.stop().then(() => process.exit(0)) }) } diff --git a/relay/src/server.ts b/relay/src/server.ts index 3c93595..28f1bed 100644 --- a/relay/src/server.ts +++ b/relay/src/server.ts @@ -42,7 +42,12 @@ import { } from 'node:http' import { currentEpoch } from './epoch.ts' import { Deadman, rowError } from './deadman.ts' -import { FleetStats, fleetReportError, type FleetReport } from './fleet.ts' +import { + FleetStats, + fleetReportError, + type FleetReport, + type FleetStatsView, +} from './fleet.ts' import { AttemptCounter, EXCHANGE, TokenBucket } from './limits.ts' import { CLOSE_BAD_JOIN, @@ -142,6 +147,19 @@ export interface RelayInspection { fleet: { days: number; reportsToday: number } } +export interface RelayAggregateStats { + schema: 'ftw.relay-stats/1' + observed_at: string + relay: { + uptime_seconds: number + rooms: number + sockets: number + frames_routed: number + bytes_routed: number + } + fleet: FleetStatsView +} + /** The box's uplink, a browser stream, or the box's exchange socket. */ type Role = 'box' | 'app' | 'exchange' @@ -271,6 +289,7 @@ export class RelayServer { Omit > #epoch: number + #startedAtMs: number #rotateStartedAtMs: number | null = null #sockets = 0 /** Concurrent sockets per address, so one client cannot fill the relay. */ @@ -308,6 +327,7 @@ export class RelayServer { } const now = this.#opts.now() + this.#startedAtMs = now this.#epoch = currentEpoch(now) this.#attempts = new AttemptCounter( { limit: this.#opts.attemptLimit, windowMs: this.#opts.attemptWindowMs }, @@ -454,6 +474,31 @@ export class RelayServer { } } + /** + * The only body allowed to leave for project statistics. + * + * It has process counters and the same daily fleet totals the local operator + * route already serves. It has no epoch, handle, address, dead-man state or + * routed payload. Fourteen days are enough to heal a short export outage while + * keeping the signed request small. + */ + aggregateStats(): RelayAggregateStats { + const now = this.#opts.now() + const fleet = this.#fleet.view() + return { + schema: 'ftw.relay-stats/1', + observed_at: new Date(now).toISOString(), + relay: { + uptime_seconds: Math.max(0, Math.floor((now - this.#startedAtMs) / 1000)), + rooms: this.#rooms.size, + sockets: this.#sockets, + frames_routed: this.#framesRouted, + bytes_routed: this.#bytesRouted, + }, + fleet: { ...fleet, days: fleet.days.slice(0, 14) }, + } + } + stop(): Promise { clearInterval(this.#timer) for (const room of this.#rooms.values()) this.#failOpen(room) diff --git a/relay/src/stats-export.test.ts b/relay/src/stats-export.test.ts new file mode 100644 index 0000000..3842978 --- /dev/null +++ b/relay/src/stats-export.test.ts @@ -0,0 +1,91 @@ +// @vitest-environment node + +import { createHmac } from 'node:crypto' +import { describe, expect, it } from 'vitest' +import { RelayServer, type RelayAggregateStats } from './server.ts' +import { RelayStatsExporter } from './stats-export.ts' + +const SECRET = '0123456789abcdef0123456789abcdef' + +function snapshot(): RelayAggregateStats { + return { + schema: 'ftw.relay-stats/1', + observed_at: '2026-08-14T12:00:00.000Z', + relay: { + uptime_seconds: 300, + rooms: 2, + sockets: 3, + frames_routed: 40, + bytes_routed: 5000, + }, + fleet: { + schema: 'ftw.fleet-stats/1', + meaning: 'reports, not unique boxes', + days: [], + }, + } +} + +describe('relay project stats export', () => { + it('signs the exact bounded body without adding an identity', async () => { + let sent: { body: string; headers: Headers } | null = null + const exporter = new RelayStatsExporter({ + url: 'https://stats.ftw.energy/api/ingest/relay', + secret: SECRET, + snapshot, + now: () => Date.UTC(2026, 7, 14, 12), + post: async (_url, init) => { + sent = { body: String(init.body), headers: new Headers(init.headers) } + return { status: 204 } + }, + }) + + expect(await exporter.sendOnce()).toBe(true) + expect(sent).not.toBeNull() + const body = sent!.body + const timestamp = sent!.headers.get('x-ftw-timestamp') + const expected = createHmac('sha256', SECRET).update(`${timestamp}.${body}`).digest('hex') + expect(sent!.headers.get('x-ftw-signature')).toBe(`v1=${expected}`) + expect(JSON.parse(body)).toEqual(snapshot()) + expect(body).not.toMatch(/handle|address|serial|gateway|site_name/) + }) + + it('keeps failures out of the relay path and logs a fixed message', async () => { + const lines: string[] = [] + const exporter = new RelayStatsExporter({ + url: 'https://stats.ftw.energy/api/ingest/relay', + secret: SECRET, + snapshot, + log: (line) => lines.push(line), + post: async () => ({ status: 503 }), + }) + expect(await exporter.sendOnce()).toBe(false) + expect(lines).toEqual(['relay: stats export rejected']) + }) + + it('builds its snapshot from counts and never from room handles', async () => { + const now = Date.UTC(2026, 7, 14, 12) + const relay = await RelayServer.start({ now: () => now }) + try { + expect(relay.aggregateStats()).toEqual({ + schema: 'ftw.relay-stats/1', + observed_at: '2026-08-14T12:00:00.000Z', + relay: { + uptime_seconds: 0, + rooms: 0, + sockets: 0, + frames_routed: 0, + bytes_routed: 0, + }, + fleet: { + schema: 'ftw.fleet-stats/1', + meaning: 'reports, not unique boxes', + days: [], + }, + }) + expect(JSON.stringify(relay.aggregateStats())).not.toContain('epoch') + } finally { + await relay.stop() + } + }) +}) diff --git a/relay/src/stats-export.ts b/relay/src/stats-export.ts new file mode 100644 index 0000000..c104877 --- /dev/null +++ b/relay/src/stats-export.ts @@ -0,0 +1,107 @@ +import { createHmac } from 'node:crypto' +import type { RelayAggregateStats } from './server.ts' + +const DEFAULT_INTERVAL_MS = 5 * 60_000 +const DEFAULT_TIMEOUT_MS = 10_000 +const MAX_BODY_BYTES = 240 * 1024 + +export interface StatsExportOptions { + url: string + secret: string + snapshot: () => RelayAggregateStats + intervalMs?: number + timeoutMs?: number + now?: () => number + log?: (line: string) => void + post?: (url: string, init: RequestInit) => Promise<{ status: number }> +} + +export class RelayStatsExporter { + #opts: Required + #timer: ReturnType | null = null + #inFlight = false + #abort: AbortController | null = null + + constructor(opts: StatsExportOptions) { + const url = new URL(opts.url) + if (url.protocol !== 'https:' || url.username || url.password) { + throw new Error('relay stats export needs an HTTPS URL') + } + if (Buffer.byteLength(opts.secret) < 32) { + throw new Error('relay stats export secret is too short') + } + this.#opts = { + url: url.toString(), + secret: opts.secret, + snapshot: opts.snapshot, + intervalMs: opts.intervalMs ?? DEFAULT_INTERVAL_MS, + timeoutMs: opts.timeoutMs ?? DEFAULT_TIMEOUT_MS, + now: opts.now ?? (() => Date.now()), + log: opts.log ?? (() => {}), + post: opts.post ?? ((target, init) => fetch(target, init)), + } + } + + start(): void { + if (this.#timer) return + void this.sendOnce() + this.#timer = setInterval(() => void this.sendOnce(), this.#opts.intervalMs) + this.#timer.unref?.() + } + + stop(): void { + if (this.#timer) clearInterval(this.#timer) + this.#timer = null + this.#abort?.abort() + } + + async sendOnce(): Promise { + if (this.#inFlight) return false + this.#inFlight = true + const abort = new AbortController() + this.#abort = abort + const timeout = setTimeout(() => abort.abort(), this.#opts.timeoutMs) + timeout.unref?.() + + try { + const snapshot = this.#opts.snapshot() + let days = snapshot.fleet.days + let body = JSON.stringify(snapshot) + while (Buffer.byteLength(body) > MAX_BODY_BYTES && days.length > 1) { + days = days.slice(0, -1) + body = JSON.stringify({ ...snapshot, fleet: { ...snapshot.fleet, days } }) + } + if (Buffer.byteLength(body) > MAX_BODY_BYTES) { + this.#opts.log('relay: stats export too large') + return false + } + const timestamp = String(Math.floor(this.#opts.now() / 1000)) + const signature = createHmac('sha256', this.#opts.secret) + .update(`${timestamp}.${body}`) + .digest('hex') + const response = await this.#opts.post(this.#opts.url, { + method: 'POST', + body, + signal: abort.signal, + headers: { + 'content-type': 'application/json', + 'user-agent': 'ftw-relay/1', + 'x-ftw-timestamp': timestamp, + 'x-ftw-signature': `v1=${signature}`, + }, + }) + if (response.status < 200 || response.status >= 300) { + this.#opts.log('relay: stats export rejected') + return false + } + return true + } catch { + this.#opts.log('relay: stats export failed') + return false + } finally { + clearTimeout(timeout) + this.#abort = null + this.#inFlight = false + } + } +} diff --git a/stats/.dev.vars.example b/stats/.dev.vars.example new file mode 100644 index 0000000..4e592c8 --- /dev/null +++ b/stats/.dev.vars.example @@ -0,0 +1,6 @@ +GITHUB_TOKEN=github_pat_replace_me +CLOUDFLARE_ZONE_ID=replace_with_the_ftw_energy_zone_id +CLOUDFLARE_ANALYTICS_TOKEN=replace_with_a_token_that_can_read_the_ftw_energy_zone +RELAY_INGEST_SECRET=replace_with_at_least_32_random_bytes +ACCESS_TEAM_DOMAIN=https://your-team.cloudflareaccess.com +ACCESS_AUD=replace_with_the_access_application_aud diff --git a/stats/README.md b/stats/README.md new file mode 100644 index 0000000..ab8886e --- /dev/null +++ b/stats/README.md @@ -0,0 +1,146 @@ +# FTW project stats + +This Worker keeps project growth data in one small D1 database. It polls the +configured GitHub repositories, reads server-side traffic counts for +`app.ftw.energy`, polls public FTW Cloud daily reports and usage cells, keeps the retired relay archive, and serves a +public and a private dashboard at `stats.ftw.energy`. + +The public view follows the six active public product repositories: `ftw`, +`device-drivers`, `ftw-cloud`, `ftw-app`, `ftw-webapp` and `ftw-web`. Private +repositories, the archived `ftwdb` experiment and the solver demo are outside +these totals. Release labels name the newest published release among the +latest 100 releases, including beta releases. A new repository appears as +awaiting collection until its first snapshot arrives; missing data is not zero. + +The public view shows repository data, web app traffic and aggregate daily +reports. Fleet totals stay hidden until at least `PUBLIC_MIN_REPORTS` reports +exist in the chosen period. Each version, device integration and other +dimension label also stays hidden until its own count reaches that threshold. +A report is one daily check-in, not one user or one box. The private view may +show smaller aggregate counts, but it requires a valid Cloudflare Access JWT. + +## Relay retirement + +The separate relay is retired. Earlier reports remain as an archive and do +not measure current Cloud use. Core's anonymous daily reports now go to +Cloud; the Worker reads its thresholded public view at `/v1/fleet/public`. They still cannot count unique installations. + +The installation estimate divides reports from the last seven complete UTC +days by seven, then rounds to a whole installation. Core normally reports +once a day. Missing or privacy-withheld days hide the estimate; stale source +data is marked. Opt-outs, offline boxes, failed sends, restarts and test boxes +can change it. It estimates active reporting installations, not people or +all installations ever made. Cloud holds imported relay history, so the page +uses the Cloud view in place of the archive and never adds them together. The separate usage +policy below supplies installation, capacity and energy cells. Never merge +these two report types or treat check-ins as people. + +The signed legacy ingest route stays for compatibility. It accepts group +counters only, never routed frames, room handles, box ids, serials, IP +addresses, site names or raw household reports. + +## Set up + +Create the D1 database, copy its id into `stats/wrangler.jsonc`, then apply the +migration: + +```sh +npx wrangler d1 create ftw-project-stats +npx wrangler d1 migrations apply ftw-project-stats --config stats/wrangler.jsonc --remote +``` + +Set three Worker secrets: + +```sh +npx wrangler secret put GITHUB_TOKEN --config stats/wrangler.jsonc +npx wrangler secret put CLOUDFLARE_ANALYTICS_TOKEN --config stats/wrangler.jsonc +npx wrangler secret put RELAY_INGEST_SECRET --config stats/wrangler.jsonc +``` + +Set `CLOUDFLARE_ZONE_ID` as a Worker variable. Give the Cloudflare token +`Account > Account Analytics > Read` and make sure its resource scope includes +the `ftw.energy` zone. The hourly collection asks for Cloudflare's server-side +visits, requests and response bytes for `app.ftw.energy` over the last 7 complete +UTC days. It uses one aggregate query per day to fit the zone's one-day query +limit. It sends no +browser script and stores no request rows, IP addresses or visitor ids. + +The GitHub repository snapshot and site query run each hour. GitHub traffic +runs once a day because GitHub exposes it as a rolling daily window. Cloud usage collection also runs hourly. The retired relay no longer +supplies an active source. + +Set `ACCESS_TEAM_DOMAIN` and `ACCESS_AUD` as Worker variables. Protect +`stats.ftw.energy/admin*` and `stats.ftw.energy/api/admin*` with one Cloudflare +Access application. The Worker also checks the token signature, issuer and +audience, so a route mistake fails closed. + +Use a policy made for this app and include the GitHub organization `srcfl`. +Do not add GitHub as a separate login-method Include rule: Access joins Include +rules with OR, which would let any GitHub user through. + +Historical relay export configuration (retired; do not set up again): + +```sh +RELAY_STATS_EXPORT_URL=https://stats.ftw.energy/api/ingest/relay +RELAY_STATS_EXPORT_SECRET=replace_with_the_same_secret +``` + +The retired relay exporter used to retry every five minutes. It is no longer +a source for current activity. + +## Retention + +The hourly cleanup keeps only the data each dashboard window needs: + +- 90 days of GitHub snapshots, relay snapshots, fleet daily totals and + collector results; +- only the latest thresholded Cloud daily-report view, removed if over 90 days old; +- 30 UTC days of GitHub traffic, referrers and popular paths; +- the last 7 complete UTC days of site traffic. + +Relay ingest also applies the 90-day relay and fleet limits, so those two +tables stay bounded even if a scheduled collection fails. Rows outside these +windows are deleted, not only hidden from dashboard queries. + +## Metric meanings + +- Stars, forks, pull requests and issues come from GitHub. +- Views and clones use GitHub's rolling traffic data. GitHub only returns the + recent window, so the daily collector saves it for longer trends. +- Site visits are Cloudflare entry visits, not users or unique people. One visit + can include more than one page view. Cloudflare may sample these counts, so + use them as a trend. Requests and data transfer use the same server-side query. +- Unique visitor and cloner totals are per repository. Adding repositories can + count the same person more than once. +- Contributor identities are also per repository and include bots. The same + person can appear in more than one repository total. +- Fleet reports are daily reports, not users or unique installs. Public version + and device integration names show which broad labels appeared in those + reports. A device integration is a type, not a count of physical devices. +- The API keeps its legacy relay activity bands for compatibility. The public + page does not render them. The private view shows current room and socket + counts plus frame and byte deltas over the current relay process, capped to + the last 24 hours. These counts do not measure FTW Cloud. + +## Cloud usage + +The hourly collector reads only public group cells from +`https://cloud.ftw.energy/v1/stats`. D1 keeps those published snapshots for +90 days. It never receives a home id, stats id or per-home reading. Missing, +stale and privacy-withheld values remain distinct from zero. + +The paired Core and Cloud changes implement policy +`ftw-project-stats-2026-10-v2`. Statistics are included for all Cloud users +under updated Cloud terms. Local users receive a separate Yes/No question, +with Yes suggested and no upload before an answer. Earlier choices do not +accept the new scope. Cloud must also advertise the exact +policy after the C8 DPIA review. An HTTP 503 with `FTW-Statistics-State: disabled` +is an expected waiting state, not a collector failure. Until those changes are deployed +and boxes report, the page shows an awaiting-source state. The public view +requires at least 20 homes per cell and no home above 30 percent of a sum. + +Relay is retired. Earlier relay reports remain as an archive; they are +check-ins, never unique users or installations. The legacy signed ingest +route remains for compatibility and is not the Cloud usage source. GitHub +charts keep saved subtotals with repo coverage; star lines break when the +set changes, so adding repositories cannot look like organic growth. diff --git a/stats/migrations/0001_initial.sql b/stats/migrations/0001_initial.sql new file mode 100644 index 0000000..833244a --- /dev/null +++ b/stats/migrations/0001_initial.sql @@ -0,0 +1,94 @@ +CREATE TABLE github_snapshots ( + repo TEXT NOT NULL, + captured_hour TEXT NOT NULL, + captured_at TEXT NOT NULL, + stars INTEGER NOT NULL CHECK (stars >= 0), + forks INTEGER NOT NULL CHECK (forks >= 0), + watchers INTEGER NOT NULL CHECK (watchers >= 0), + open_prs INTEGER NOT NULL CHECK (open_prs >= 0), + draft_prs INTEGER NOT NULL CHECK (draft_prs >= 0), + dependency_prs INTEGER NOT NULL CHECK (dependency_prs >= 0), + open_issues INTEGER NOT NULL CHECK (open_issues >= 0), + merged_prs_30d INTEGER NOT NULL CHECK (merged_prs_30d >= 0), + closed_issues_30d INTEGER NOT NULL CHECK (closed_issues_30d >= 0), + contributors INTEGER NOT NULL CHECK (contributors >= 0), + pushed_at TEXT, + latest_release TEXT, + latest_release_at TEXT, + PRIMARY KEY (repo, captured_hour) +); + +CREATE INDEX github_snapshots_by_time + ON github_snapshots (captured_hour DESC); + +CREATE TABLE github_traffic_daily ( + repo TEXT NOT NULL, + date TEXT NOT NULL, + views INTEGER NOT NULL CHECK (views >= 0), + unique_visitors INTEGER NOT NULL CHECK (unique_visitors >= 0), + clones INTEGER NOT NULL CHECK (clones >= 0), + unique_cloners INTEGER NOT NULL CHECK (unique_cloners >= 0), + observed_at TEXT NOT NULL, + PRIMARY KEY (repo, date) +); + +CREATE TABLE github_referrers ( + repo TEXT NOT NULL, + date TEXT NOT NULL, + referrer TEXT NOT NULL, + visits INTEGER NOT NULL CHECK (visits >= 0), + unique_visitors INTEGER NOT NULL CHECK (unique_visitors >= 0), + PRIMARY KEY (repo, date, referrer) +); + +CREATE TABLE github_paths ( + repo TEXT NOT NULL, + date TEXT NOT NULL, + path TEXT NOT NULL, + title TEXT NOT NULL, + views INTEGER NOT NULL CHECK (views >= 0), + unique_visitors INTEGER NOT NULL CHECK (unique_visitors >= 0), + PRIMARY KEY (repo, date, path) +); + +CREATE TABLE site_traffic_daily ( + date TEXT PRIMARY KEY, + hostname TEXT NOT NULL, + requests INTEGER NOT NULL CHECK (requests >= 0), + visits INTEGER NOT NULL CHECK (visits >= 0), + response_bytes INTEGER NOT NULL CHECK (response_bytes >= 0), + sample_interval REAL NOT NULL CHECK (sample_interval >= 1), + observed_at TEXT NOT NULL +); + +CREATE TABLE relay_snapshots ( + observed_at TEXT PRIMARY KEY, + received_at TEXT NOT NULL, + uptime_seconds INTEGER NOT NULL CHECK (uptime_seconds >= 0), + rooms INTEGER NOT NULL CHECK (rooms >= 0), + sockets INTEGER NOT NULL CHECK (sockets >= 0), + frames_routed INTEGER NOT NULL CHECK (frames_routed >= 0), + bytes_routed INTEGER NOT NULL CHECK (bytes_routed >= 0) +); + +CREATE INDEX relay_snapshots_by_time + ON relay_snapshots (observed_at DESC); + +CREATE TABLE fleet_daily ( + date TEXT PRIMARY KEY, + reports INTEGER NOT NULL CHECK (reports >= 0), + dimensions_json TEXT NOT NULL, + observed_at TEXT NOT NULL +); + +CREATE TABLE collector_runs ( + source TEXT NOT NULL, + started_at TEXT NOT NULL, + finished_at TEXT NOT NULL, + ok INTEGER NOT NULL CHECK (ok IN (0, 1)), + detail TEXT NOT NULL, + PRIMARY KEY (source, started_at) +); + +CREATE INDEX collector_runs_latest + ON collector_runs (source, finished_at DESC); diff --git a/stats/migrations/0002_usage.sql b/stats/migrations/0002_usage.sql new file mode 100644 index 0000000..d353163 --- /dev/null +++ b/stats/migrations/0002_usage.sql @@ -0,0 +1,6 @@ +-- Only already-public group cells. No home ids or household values. +CREATE TABLE usage_snapshots ( + captured_hour TEXT PRIMARY KEY, + observed_at TEXT NOT NULL, + view_json TEXT NOT NULL +); diff --git a/stats/migrations/0003_cloud_fleet.sql b/stats/migrations/0003_cloud_fleet.sql new file mode 100644 index 0000000..42bc8ee --- /dev/null +++ b/stats/migrations/0003_cloud_fleet.sql @@ -0,0 +1,6 @@ +-- Latest public daily cells only; never household reports or ids. +CREATE TABLE cloud_fleet_view ( + id INTEGER PRIMARY KEY CHECK(id=1), + observed_at TEXT NOT NULL, + view_json TEXT NOT NULL +); diff --git a/stats/src/access.ts b/stats/src/access.ts new file mode 100644 index 0000000..a6a5dac --- /dev/null +++ b/stats/src/access.ts @@ -0,0 +1,41 @@ +import { createRemoteJWKSet, jwtVerify } from 'jose' +import type { AccessIdentity, AppEnv } from './types.ts' + +export async function accessIdentity(request: Request, env: AppEnv): Promise { + const audience = env.ACCESS_AUD + const teamDomain = validTeamDomain(env.ACCESS_TEAM_DOMAIN) + const token = request.headers.get('cf-access-jwt-assertion') + if (!audience || !teamDomain || !token) return null + + try { + const keySet = createRemoteJWKSet(new URL('/cdn-cgi/access/certs', teamDomain)) + const { payload } = await jwtVerify(token, keySet, { + algorithms: ['RS256'], + audience, + issuer: teamDomain.origin, + }) + if (payload['type'] !== 'app' || typeof payload['email'] !== 'string') return null + return { email: payload['email'] } + } catch { + return null + } +} + +function validTeamDomain(value: string | undefined): URL | null { + if (!value) return null + try { + const url = new URL(value) + if ( + url.protocol !== 'https:' || + !url.hostname.endsWith('.cloudflareaccess.com') || + url.pathname !== '/' || + url.search !== '' || + url.hash !== '' + ) { + return null + } + return url + } catch { + return null + } +} diff --git a/stats/src/collector.ts b/stats/src/collector.ts new file mode 100644 index 0000000..29f2993 --- /dev/null +++ b/stats/src/collector.ts @@ -0,0 +1,16 @@ +export async function saveCollectorRun( + db: D1Database, + source: string, + startedAt: string, + ok: boolean, + detail: string +): Promise { + await db + .prepare( + `INSERT OR REPLACE INTO collector_runs + (source, started_at, finished_at, ok, detail) + VALUES (?, ?, ?, ?, ?)` + ) + .bind(source, startedAt, new Date().toISOString(), ok ? 1 : 0, detail) + .run() +} diff --git a/stats/src/dashboard.ts b/stats/src/dashboard.ts new file mode 100644 index 0000000..08db618 --- /dev/null +++ b/stats/src/dashboard.ts @@ -0,0 +1,559 @@ +export interface DashboardDocument { + html: string + nonce: string +} + +export function dashboardDocument(privateView: boolean): DashboardDocument { + const nonce = randomNonce() + const endpoint = privateView ? '/api/admin' : '/api/public' + const mode = privateView ? 'PRIVATE' : 'PUBLIC' + const html = ` + + + + + + FTW project stats + + + +
+ + FTW/PROJECT STATS + +
+ GitHubEnergy${mode} + Loading +
+
+ +
+
+
+

OPEN SOURCE ENERGY

+

FTW development and energy.

+

Development across FTW Core, device drivers, Cloud, the native app, the web app and the website. Public project data, with clear sources.

+
+
+
+ + + +
+
+

OVERVIEW

Project at a glance

+

GitHub activity, a reporting-installation estimate and group energy totals.

+
+
+
Open source reach—— forks
+
Project work—Merged PRs in 30d
+
Web app visits—Last 7 complete days
+
Active installations (estimate)—Last 7 complete UTC days
+
Flexible power—Configured battery + charger limits · kW
+
+
+ +

Estimate pending. Total installations through all time are unknown.

+
+

ENERGY VIA FTW CLOUD

Capacity and recorded energy

Awaiting source
+
+
Battery capacity—Configured · kWh
+
Solar capacity—Configured · kWp
+
Solar generation—Recorded hours · kWh
+
Grid import / export—Recorded hours · kWh
+
Battery charge / discharge—Recorded hours · kWh
+
Vehicle charge / discharge—Recorded hours · kWh
+
+

The Cloud usage source is not connected yet. Values appear after privacy review, policy activation and updated owner terms.

+

Installation counts do not count people or all FTW boxes. Configured limits do not prove physical control. Energy sums the recorded hours in a 24-hour window; gaps reduce coverage. Withheld cells show no value.

+
+ +
+

DATA SOURCES

What this page covers

+

Six active public repositories. Private repositories and archived experiments are outside these totals.

+

The separate relay is retired. Anonymous daily reports now come through FTW Cloud. The updated Cloud terms include statistics for all Cloud users. Local FTW users can choose to contribute without remote access. Hourly collection awaits privacy review. App Store installs are not collected. Native app development appears below.

+

Missing usage values stay unknown. Public energy cells need at least 20 contributing installations. Historical relay reports remain in the archive below.

+
+ +
+
+

DETAIL

GitHub activity

+
+
+
GitHub stars—No baseline yet
+
Forks—No baseline yet
+
Open PRs—— drafts
+
Open issues—— closed in 30d
+
Merged PRs—Last 30 days
+
Repo views—14-day window
+
Clones—14-day window
+
Contributors—Per-repo identities, bots included
+
+
+ +
+
+

90 DAYS

Stars

+ +
+
+

30 DAYS

GitHub traffic

Per-repo totals
+ +
+
+ +
+
+

WEB APP

Server-side site traffic

+ Last 7 complete days +
+
+
Visits—
+
Requests—
+
Data transfer—
+
+ +

A visit is an entry visit, not one user or one unique person. No browser tracker is used.

+
+ +
+

REPOSITORIES

Work and reach

+
+ + + +
RepositoryStarsPRsIssues30d mergedPublished releaseLast push
+
+

The latest published release includes beta releases. New repositories build history from their first collection.

+
+ +
+
+

FLEET

Archived relay reports

+ — +
+

A report is one daily check-in, not one user or one unique box.

+

Historical check-ins from the retired relay.

+ +
+

Versions and device integrations describe aggregate reports, not physical device counts.

+
+ + + + +
+ +
+ FTW is local first. These numbers describe the project, not a household. + GitHub +
+ + + +` + return { html, nonce } +} + +function randomNonce(): string { + const bytes = crypto.getRandomValues(new Uint8Array(18)) + let binary = '' + for (const byte of bytes) binary += String.fromCharCode(byte) + return btoa(binary) +} + +const styles = ` +:root{--bg:#0b0c0b;--surface:#111311;--raised:#171917;--line:#2a2d2a;--fg:#f2f1ec;--dim:#9c9f9a;--accent:#ffab2e;--accent-soft:#3b2b13;--ok:#72d69b;--warn:#ffbf5f;font-family:Inter,ui-sans-serif,system-ui,-apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif;color:var(--fg);background:var(--bg);font-synthesis:none} +[hidden]{display:none!important} +*{box-sizing:border-box}body{margin:0;background:radial-gradient(circle at 80% -20%,#252116 0,transparent 34rem),var(--bg);min-height:100vh}a{color:inherit}.topbar{height:58px;border-bottom:1px solid var(--line);display:flex;align-items:center;justify-content:space-between;padding:0 clamp(18px,4vw,56px);position:sticky;top:0;background:color-mix(in srgb,var(--bg) 90%,transparent);backdrop-filter:blur(16px);z-index:5}.brand{display:flex;gap:10px;align-items:center;text-decoration:none;font:700 12px/1 ui-monospace,SFMono-Regular,Menlo,monospace;letter-spacing:.18em}.brand-mark{color:var(--accent)}.brand-slash{color:#5e615d}.top-actions{display:flex;align-items:center;gap:16px;font:11px/1 ui-monospace,SFMono-Regular,Menlo,monospace;letter-spacing:.08em}.mode{color:var(--accent);border:1px solid #59401c;padding:6px 8px;border-radius:2px}.muted{color:var(--dim)}main{width:min(1280px,calc(100% - 36px));margin:0 auto;padding:64px 0 72px}.hero{display:flex;justify-content:space-between;gap:48px;align-items:flex-end;margin-bottom:54px}.eyebrow{color:var(--accent);font:700 10px/1.2 ui-monospace,SFMono-Regular,Menlo,monospace;letter-spacing:.2em;margin:0 0 12px}h1{font-size:clamp(36px,5vw,72px);line-height:.98;letter-spacing:-.055em;max-width:800px;margin:0}h2{font-size:20px;letter-spacing:-.025em;margin:0}h3{font-size:12px;color:var(--dim);text-transform:uppercase;letter-spacing:.08em;margin:0 0 10px}.lede{color:var(--dim);font-size:17px;max-width:660px;line-height:1.5;margin:20px 0 0}.freshness{min-width:230px;display:grid;gap:8px}.freshness-item{display:grid;grid-template-columns:10px 1fr auto;align-items:center;gap:10px;font-size:12px;color:var(--dim)}.freshness-item strong{font-weight:500;color:var(--fg)}.dot{width:7px;height:7px;border-radius:50%;background:#555;display:inline-block}.dot.ok{background:var(--ok);box-shadow:0 0 12px color-mix(in srgb,var(--ok) 50%,transparent)}.dot.warn{background:var(--warn)}.dot.idle{background:#555}.notice{border:1px solid #674a20;background:#21190e;color:#ffd48f;padding:14px 16px;margin-bottom:24px}.section-head,.panel-head{display:flex;justify-content:space-between;align-items:flex-end;gap:20px}.section-head{margin:0 0 16px}.section-copy{color:var(--dim);font-size:12px;line-height:1.5;max-width:560px;margin:0;text-align:right}.overview-grid,.metric-grid{display:grid;border-top:1px solid var(--line);border-left:1px solid var(--line);margin-bottom:28px}.overview-grid{grid-template-columns:repeat(5,minmax(0,1fr))}.metric-grid{grid-template-columns:repeat(4,minmax(0,1fr));margin-bottom:18px}.metric-card{background:color-mix(in srgb,var(--surface) 92%,transparent);min-height:154px;padding:22px;border-right:1px solid var(--line);border-bottom:1px solid var(--line);display:flex;flex-direction:column}.overview-card{min-height:142px}.metric-card>span,.mini-grid span{font:11px/1.2 ui-monospace,SFMono-Regular,Menlo,monospace;color:var(--dim);text-transform:uppercase;letter-spacing:.08em}.metric-card strong{font-size:42px;line-height:1;margin:auto 0 12px;letter-spacing:-.05em}.metric-card.status-card strong{font-size:30px;letter-spacing:-.035em}.metric-card small{color:var(--dim);font-size:12px;line-height:1.35}.two-col{display:grid;grid-template-columns:1fr 1fr;gap:18px;margin:18px 0}.panel{background:color-mix(in srgb,var(--surface) 94%,transparent);border:1px solid var(--line);padding:22px;margin:18px 0}.two-col>.panel{margin:0}.chart-panel{min-height:330px}.chart{height:235px;margin-top:24px}.chart svg{display:block;width:100%;height:100%;overflow:visible}h2{scroll-margin-top:90px}.mini-grid.usage-grid{grid-template-columns:repeat(3,minmax(0,1fr))}.chart-panel .muted{font-size:11px}.chart-area{fill:color-mix(in srgb,var(--accent) 9%,transparent)}.chart-line{fill:none;stroke:var(--accent);stroke-width:3;stroke-linejoin:round;stroke-linecap:round}.bars{height:230px;display:flex;gap:4px;align-items:flex-end;margin-top:24px;border-bottom:1px solid var(--line);padding:0 2px}.bars.compact{height:150px}.bar-wrap{height:100%;flex:1;min-width:3px;display:flex;flex-direction:column;justify-content:flex-end;align-items:center;gap:8px}.bar{width:100%;max-width:28px;background:linear-gradient(to top,#88520d,var(--accent));border-radius:2px 2px 0 0;min-height:2px}.bar.withheld{background:#4a4c49}.bar.h0{height:2%}.bar.h1{height:5%}.bar.h2{height:10%}.bar.h3{height:15%}.bar.h4{height:20%}.bar.h5{height:25%}.bar.h6{height:30%}.bar.h7{height:35%}.bar.h8{height:40%}.bar.h9{height:45%}.bar.h10{height:50%}.bar.h11{height:55%}.bar.h12{height:60%}.bar.h13{height:65%}.bar.h14{height:70%}.bar.h15{height:75%}.bar.h16{height:80%}.bar.h17{height:85%}.bar.h18{height:90%}.bar.h19{height:95%}.bar.h20{height:100%}.bar-wrap span{font-size:8px;color:var(--dim);height:12px;white-space:nowrap}.table-wrap{overflow-x:auto;margin-top:20px}table{width:100%;border-collapse:collapse;font-size:13px}th{text-align:left;color:var(--dim);font:10px/1 ui-monospace,SFMono-Regular,Menlo,monospace;text-transform:uppercase;letter-spacing:.08em;padding:12px 14px;border-bottom:1px solid var(--line)}td{padding:15px 14px;border-bottom:1px solid #232523;white-space:nowrap}td:first-child{font-weight:650}td a{text-decoration-color:#51544f;text-underline-offset:3px}.repo-role{display:block;color:var(--dim);font-size:11px;font-weight:400;margin-top:4px}.numeric{font-variant-numeric:tabular-nums}.empty{color:var(--dim);padding:30px 4px;text-align:center}.panel-copy,.footnote{color:var(--dim);font-size:13px;line-height:1.5}.big-inline{font-size:28px;font-weight:700;letter-spacing:-.04em}.dimension-grid{display:grid;grid-template-columns:repeat(3,1fr);gap:20px;margin-top:26px}.dimension{border-top:1px solid var(--line);padding-top:14px}.chips{display:flex;gap:7px;flex-wrap:wrap}.chip{font:11px/1.2 ui-monospace,SFMono-Regular,Menlo,monospace;color:#d5d3cb;background:var(--raised);border:1px solid var(--line);padding:6px 8px;border-radius:2px}.mini-grid{display:grid;grid-template-columns:repeat(4,1fr);gap:1px;background:var(--line);margin-top:22px}.mini-grid>div{background:var(--raised);padding:18px;display:flex;flex-direction:column;gap:12px}.mini-grid strong{font-size:26px}.ranked-list,.status-list{list-style:none;padding:0;margin:18px 0 0}.ranked-list li{display:flex;justify-content:space-between;gap:18px;padding:12px 0;border-bottom:1px solid var(--line);font-size:13px}.status-list li{display:grid;grid-template-columns:10px 1fr auto;align-items:center;gap:10px;padding:12px 0;border-bottom:1px solid var(--line);font-size:12px}footer{border-top:1px solid var(--line);padding:24px clamp(18px,4vw,56px);display:flex;justify-content:space-between;gap:20px;color:var(--dim);font-size:12px}footer a{color:var(--fg);text-underline-offset:3px} +.mini-grid.site-grid{grid-template-columns:repeat(3,1fr)} +@media(max-width:1100px){.overview-grid{grid-template-columns:repeat(3,1fr)}} +@media(max-width:900px){.hero{align-items:flex-start;flex-direction:column}.freshness{width:100%;grid-template-columns:repeat(2,1fr)}.section-head{align-items:flex-start;flex-direction:column}.section-copy{text-align:left}.overview-grid,.metric-grid{grid-template-columns:repeat(2,1fr)}.two-col{grid-template-columns:1fr}.dimension-grid{grid-template-columns:repeat(2,1fr)}.mini-grid,.mini-grid.usage-grid{grid-template-columns:repeat(2,1fr)}.mini-grid.site-grid{grid-template-columns:repeat(3,1fr)}} +@media(max-width:560px){main{width:min(100% - 24px,1280px);padding-top:38px}.topbar{padding:0 12px}.top-actions #generated{display:none}.hero{margin-bottom:36px;gap:28px}h1{font-size:40px}.freshness{grid-template-columns:1fr}.metric-card{min-height:130px;padding:16px}.metric-card strong{font-size:34px}.panel{padding:16px}.dimension-grid{grid-template-columns:1fr}.mini-grid{grid-template-columns:1fr 1fr}.bars{gap:2px}footer{flex-direction:column;padding:20px 12px}} +@media(max-width:560px){.mini-grid.site-grid{grid-template-columns:1fr}} +@media(prefers-reduced-motion:reduce){*{scroll-behavior:auto!important}} +` diff --git a/stats/src/fleet.ts b/stats/src/fleet.ts new file mode 100644 index 0000000..8c5e4ea --- /dev/null +++ b/stats/src/fleet.ts @@ -0,0 +1,52 @@ +import { saveCollectorRun } from './collector.ts' +import type { AppEnv } from './types.ts' + +const dimensions = ['ftw_versions', 'channels', 'drivers', 'battery_kwh', 'price_zones', 'install_age'] as const +export interface PublicFleetDay { date: string; reports: number | null } +export interface PublicFleetFeed { schema: 'ftw.fleet-public/1'; minimum: number; meaning: string; observed_at: string; days: PublicFleetDay[] } +export function parseFleet(value: unknown, nowMs: number): PublicFleetFeed { + const v = value as Record + if (!v || typeof v !== 'object' || Array.isArray(v) || Object.keys(v).sort().join() !== ['days','meaning','minimum','observed_at','schema'].join() || v['schema'] !== 'ftw.fleet-public/1' || v['minimum'] !== 10 || v['meaning'] !== 'reports, not unique boxes') throw new Error('invalid public fleet') + const observed = Date.parse(String(v['observed_at'])) + if (!Number.isFinite(observed) || Math.abs(nowMs-observed)>2*3600_000 || !Array.isArray(v['days']) || v['days'].length>90) throw new Error('invalid fleet window') + const seen = new Set() + for (const item of v['days']) { + const d = item as Record, date = String(d['date']) + if (!d || Object.keys(d).sort().join() !== ['date','reports',...dimensions].sort().join() || !/^\d{4}-\d{2}-\d{2}$/.test(date) || !Number.isFinite(Date.parse(date)) || new Date(date).toISOString().slice(0,10)!==date || date>new Date(nowMs).toISOString().slice(0,10) || Date.parse(date)100000)) throw new Error('small fleet count') + for (const key of dimensions) { + const labels=d[key] as Record + if (!labels || typeof labels!=='object' || Array.isArray(labels) || Object.keys(labels).length>128) throw new Error('invalid fleet labels') + for (const [name,n] of Object.entries(labels)) if (reports===null || !name || name.length>128 || /[\x00-\x1f\x7f]/.test(name) || !Number.isSafeInteger(n) || (n as number)<10 || (n as number)>(reports as number)) throw new Error('small fleet label') + } + } + return v as unknown as PublicFleetFeed +} +export async function collectFleet(env: AppEnv, nowMs = Date.now()) { + const started = new Date(nowMs).toISOString() + try { + const r=await fetch('https://cloud.ftw.energy/v1/fleet/public',{redirect:'error',headers:{accept:'application/json'},signal:AbortSignal.timeout(15000)}) + if (!r.ok) {await r.body?.cancel();throw new Error('fleet unavailable')} + const reader=r.body?.getReader(); if (!reader) throw new Error('empty fleet') + const parts:Uint8Array[]=[];let size=0 + try {while(true){const {done,value}=await reader.read();if(done)break;size+=value.length;if(size>256*1024)throw new Error('large fleet');parts.push(value)}}finally{await reader.cancel()} + const bytes=new Uint8Array(size);let offset=0;for(const p of parts){bytes.set(p,offset);offset+=p.length} + const feed=parseFleet(JSON.parse(new TextDecoder().decode(bytes)),nowMs) + await env.DB.prepare('INSERT INTO cloud_fleet_view(id,observed_at,view_json) VALUES(1,?,?) ON CONFLICT(id) DO UPDATE SET observed_at=excluded.observed_at,view_json=excluded.view_json').bind(feed.observed_at,JSON.stringify(feed)).run() + await saveCollectorRun(env.DB,'cloud:fleet',started,true,'public daily cells') + } catch(e){await saveCollectorRun(env.DB,'cloud:fleet',started,false,'fleet source unavailable or invalid');throw e} +} + +// This estimates reporting active installations, not lifetime installs or people. +// Seven complete UTC days dampen the box's 18–30 hour send interval. +export function installationEstimate(days: PublicFleetDay[], nowMs: number, observedAt: string | null, source: string) { + const end=Date.UTC(new Date(nowMs).getUTCFullYear(),new Date(nowMs).getUTCMonth(),new Date(nowMs).getUTCDate()) + const dates=Array.from({length:7},(_,i)=>new Date(end-(7-i)*86400_000).toISOString().slice(0,10)) + const chosen=dates.map(date=>days.find(d=>d.date===date)) + if (chosen.some(d=>!d || typeof d.reports!=='number')) return {state:'insufficient',value:null,source,meaning:'reporting active installations, not all-time installs or people'} + const reports=chosen.reduce((n,d)=>n+d!.reports!,0) + return {state:observedAt && nowMs-Date.parse(observedAt)<=48*3600_000?'estimated':'stale',value:Math.round(reports/7),reports,days:7,window_start:dates[0],window_end:dates.at(-1),observed_at:observedAt,source, + meaning:'reporting active installations, not all-time installs or people',method:'Reports per complete UTC day, averaged over 7 days. Assumes one successful report per installation per day on average. Opt-outs, offline boxes and failed sends are absent; restarts and test boxes can bias the estimate.'} +} diff --git a/stats/src/github.ts b/stats/src/github.ts new file mode 100644 index 0000000..4e2fb8b --- /dev/null +++ b/stats/src/github.ts @@ -0,0 +1,406 @@ +import { saveCollectorRun } from './collector.ts' +import type { AppEnv, GitHubSnapshot } from './types.ts' + +const API_ROOT = 'https://api.github.com' +const API_VERSION = '2022-11-28' +const MAX_RESPONSE_BYTES = 2 * 1024 * 1024 +const MAX_PAGES = 10 +const NAME_RE = /^[A-Za-z0-9_.-]{1,100}$/ + +type JsonRecord = Record + +interface TrafficPoint { + timestamp: string + count: number + uniques: number +} + +interface ReferrerPoint { + referrer: string + count: number + uniques: number +} + +interface PathPoint { + path: string + title: string + count: number + uniques: number +} + +interface ReleasePoint { + tag: string + publishedAt: string | null +} + +export async function collectGitHub(env: AppEnv, daily: boolean, nowMs = Date.now()): Promise { + const token = env.GITHUB_TOKEN + if (!token) throw new Error('GITHUB_TOKEN is not configured') + + const owner = checkedName(env.GITHUB_OWNER, 'GitHub owner') + const repos = parseRepos(env.GITHUB_REPOS) + const startedAt = new Date(nowMs).toISOString() + let complete = 0 + + for (const repo of repos) { + try { + const snapshot = await collectSnapshot(owner, repo, token, nowMs) + await saveSnapshot(env.DB, snapshot) + complete += 1 + await saveCollectorRun(env.DB, `github:${repo}`, startedAt, true, 'repository snapshot') + } catch (error) { + console.error('stats: GitHub repository collection failed', { + repo, + kind: error instanceof Error ? error.name : 'unknown', + }) + await saveCollectorRun(env.DB, `github:${repo}`, startedAt, false, 'repository request failed') + } + + if (!daily) continue + try { + await collectTraffic(env.DB, owner, repo, token, nowMs) + await saveCollectorRun(env.DB, `github-traffic:${repo}`, startedAt, true, 'traffic window') + } catch (error) { + console.error('stats: GitHub traffic collection failed', { + repo, + kind: error instanceof Error ? error.name : 'unknown', + }) + await saveCollectorRun(env.DB, `github-traffic:${repo}`, startedAt, false, 'traffic request failed') + } + } + + if (complete === 0) throw new Error('all GitHub repository requests failed') +} + +export function parseRepos(value: string): string[] { + const repos = [...new Set(value.split(',').map((repo) => repo.trim()).filter(Boolean))] + if (repos.length === 0 || repos.length > 10) throw new Error('GITHUB_REPOS must name 1 to 10 repositories') + return repos.map((repo) => checkedName(repo, 'GitHub repository')) +} + +async function collectSnapshot( + owner: string, + repo: string, + token: string, + nowMs: number +): Promise { + const encodedRepo = `${encodeURIComponent(owner)}/${encodeURIComponent(repo)}` + const cutoff = new Date(nowMs - 30 * 24 * 60 * 60_000).toISOString().slice(0, 10) + const [rawRepo, pulls, issues, contributors, mergedPrs30d, closedIssues30d, release] = + await Promise.all([ + githubJson(`/repos/${encodedRepo}`, token), + githubPages(`/repos/${encodedRepo}/pulls?state=open`, token), + githubPages(`/repos/${encodedRepo}/issues?state=open`, token), + githubPages(`/repos/${encodedRepo}/contributors?anon=1`, token), + githubSearchTotal(`repo:${owner}/${repo} is:pr is:merged merged:>=${cutoff}`, token), + githubSearchTotal(`repo:${owner}/${repo} is:issue is:closed closed:>=${cutoff}`, token), + latestRelease(encodedRepo, token), + ]) + + const repoData = record(rawRepo, 'repository') + const openPulls = pulls.map((pull) => record(pull, 'pull request')) + const openIssues = issues.map((issue) => record(issue, 'issue')).filter((issue) => !('pull_request' in issue)) + const capturedAt = new Date(nowMs).toISOString() + + return { + repo, + capturedHour: `${capturedAt.slice(0, 13)}:00:00.000Z`, + capturedAt, + stars: nonNegativeInt(repoData['stargazers_count'], 'stargazers_count'), + forks: nonNegativeInt(repoData['forks_count'], 'forks_count'), + watchers: nonNegativeInt(repoData['subscribers_count'], 'subscribers_count'), + openPrs: openPulls.length, + draftPrs: openPulls.filter((pull) => pull['draft'] === true).length, + dependencyPrs: openPulls.filter(isDependencyPull).length, + openIssues: openIssues.length, + mergedPrs30d, + closedIssues30d, + contributors: contributors.length, + pushedAt: optionalIso(repoData['pushed_at'], 'pushed_at'), + latestRelease: release?.tag ?? null, + latestReleaseAt: release?.publishedAt ?? null, + } +} + +async function collectTraffic( + db: D1Database, + owner: string, + repo: string, + token: string, + nowMs: number +): Promise { + const encodedRepo = `${encodeURIComponent(owner)}/${encodeURIComponent(repo)}` + const [rawViews, rawClones, rawReferrers, rawPaths] = await Promise.all([ + githubJson(`/repos/${encodedRepo}/traffic/views?per=day`, token), + githubJson(`/repos/${encodedRepo}/traffic/clones?per=day`, token), + githubJson(`/repos/${encodedRepo}/traffic/popular/referrers`, token), + githubJson(`/repos/${encodedRepo}/traffic/popular/paths`, token), + ]) + + const views = trafficPoints(record(rawViews, 'views')['views'], 'views') + const clones = trafficPoints(record(rawClones, 'clones')['clones'], 'clones') + const referrers = referrerPoints(rawReferrers) + const paths = pathPoints(rawPaths) + const observedAt = new Date(nowMs).toISOString() + const byDate = new Map() + + for (const point of views) { + byDate.set(point.timestamp.slice(0, 10), { + views: point.count, + uniqueVisitors: point.uniques, + clones: 0, + uniqueCloners: 0, + }) + } + for (const point of clones) { + const date = point.timestamp.slice(0, 10) + const day = byDate.get(date) ?? { views: 0, uniqueVisitors: 0, clones: 0, uniqueCloners: 0 } + day.clones = point.count + day.uniqueCloners = point.uniques + byDate.set(date, day) + } + + const trafficWrites = [...byDate].map(([date, day]) => + db + .prepare( + `INSERT INTO github_traffic_daily + (repo, date, views, unique_visitors, clones, unique_cloners, observed_at) + VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(repo, date) DO UPDATE SET + views = excluded.views, + unique_visitors = excluded.unique_visitors, + clones = excluded.clones, + unique_cloners = excluded.unique_cloners, + observed_at = excluded.observed_at` + ) + .bind(repo, date, day.views, day.uniqueVisitors, day.clones, day.uniqueCloners, observedAt) + ) + if (trafficWrites.length > 0) await db.batch(trafficWrites) + + const date = observedAt.slice(0, 10) + const referrerWrites = [ + db.prepare('DELETE FROM github_referrers WHERE repo = ? AND date = ?').bind(repo, date), + ...referrers.map((point) => + db + .prepare( + `INSERT INTO github_referrers + (repo, date, referrer, visits, unique_visitors) + VALUES (?, ?, ?, ?, ?)` + ) + .bind(repo, date, point.referrer, point.count, point.uniques) + ), + ] + await db.batch(referrerWrites) + + const pathWrites = [ + db.prepare('DELETE FROM github_paths WHERE repo = ? AND date = ?').bind(repo, date), + ...paths.map((point) => + db + .prepare( + `INSERT INTO github_paths + (repo, date, path, title, views, unique_visitors) + VALUES (?, ?, ?, ?, ?, ?)` + ) + .bind(repo, date, point.path, point.title, point.count, point.uniques) + ), + ] + await db.batch(pathWrites) +} + +async function saveSnapshot(db: D1Database, snapshot: GitHubSnapshot): Promise { + await db + .prepare( + `INSERT INTO github_snapshots + (repo, captured_hour, captured_at, stars, forks, watchers, open_prs, draft_prs, + dependency_prs, open_issues, merged_prs_30d, closed_issues_30d, contributors, + pushed_at, latest_release, latest_release_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(repo, captured_hour) DO UPDATE SET + captured_at = excluded.captured_at, + stars = excluded.stars, + forks = excluded.forks, + watchers = excluded.watchers, + open_prs = excluded.open_prs, + draft_prs = excluded.draft_prs, + dependency_prs = excluded.dependency_prs, + open_issues = excluded.open_issues, + merged_prs_30d = excluded.merged_prs_30d, + closed_issues_30d = excluded.closed_issues_30d, + contributors = excluded.contributors, + pushed_at = excluded.pushed_at, + latest_release = excluded.latest_release, + latest_release_at = excluded.latest_release_at` + ) + .bind( + snapshot.repo, + snapshot.capturedHour, + snapshot.capturedAt, + snapshot.stars, + snapshot.forks, + snapshot.watchers, + snapshot.openPrs, + snapshot.draftPrs, + snapshot.dependencyPrs, + snapshot.openIssues, + snapshot.mergedPrs30d, + snapshot.closedIssues30d, + snapshot.contributors, + snapshot.pushedAt, + snapshot.latestRelease, + snapshot.latestReleaseAt + ) + .run() +} + +async function githubSearchTotal(query: string, token: string): Promise { + const raw = await githubJson(`/search/issues?q=${encodeURIComponent(query)}&per_page=1`, token) + return nonNegativeInt(record(raw, 'search result')['total_count'], 'total_count') +} + +async function latestRelease(encodedRepo: string, token: string): Promise { + // /releases/latest excludes prereleases and can keep an old stable tag on + // screen while the active native line ships beta releases. + const raw = await githubJson(`/repos/${encodedRepo}/releases?per_page=100`, token) + if (!Array.isArray(raw) || raw.length > 100) throw new TypeError('releases were not a bounded array') + const releases = raw.map((value) => record(value, 'release')) + .filter((release) => release['draft'] !== true && release['published_at']) + .sort((a, b) => String(b['published_at']).localeCompare(String(a['published_at']))) + const release = releases[0] + if (!release) return null + const tag = shortString(release['tag_name'], 'tag_name', 128) + return { tag, publishedAt: optionalIso(release['published_at'], 'published_at') } +} + +async function githubPages(path: string, token: string): Promise { + const rows: unknown[] = [] + for (let page = 1; page <= MAX_PAGES; page += 1) { + const url = new URL(path, API_ROOT) + url.searchParams.set('per_page', '100') + url.searchParams.set('page', String(page)) + const raw = await githubJson(`${url.pathname}${url.search}`, token) + if (!Array.isArray(raw)) throw new TypeError('GitHub page was not an array') + rows.push(...raw) + if (raw.length < 100) return rows + } + throw new RangeError('GitHub result exceeded the page limit') +} + +async function githubJson(path: string, token: string, allow404 = false): Promise { + const response = await fetch(new URL(path, API_ROOT), { + headers: { + accept: 'application/vnd.github+json', + authorization: `Bearer ${token}`, + 'user-agent': 'ftw-project-stats/1', + 'x-github-api-version': API_VERSION, + }, + }) + if (allow404 && response.status === 404) { + await response.body?.cancel() + return null + } + if (!response.ok) { + await response.body?.cancel() + throw new GitHubRequestError(response.status) + } + + const length = Number(response.headers.get('content-length') ?? 0) + if (Number.isFinite(length) && length > MAX_RESPONSE_BYTES) throw new RangeError('GitHub response was too large') + const text = await response.text() + if (new TextEncoder().encode(text).byteLength > MAX_RESPONSE_BYTES) { + throw new RangeError('GitHub response was too large') + } + return JSON.parse(text) as unknown +} + +class GitHubRequestError extends Error { + override name = 'GitHubRequestError' + + constructor(status: number) { + super(`GitHub returned HTTP ${status}`) + } +} + +function isDependencyPull(pull: JsonRecord): boolean { + const user = pull['user'] + if (!isRecord(user)) return false + const login = user['login'] + return typeof login === 'string' && login.toLowerCase().startsWith('dependabot') +} + +function trafficPoints(value: unknown, label: string): TrafficPoint[] { + if (!Array.isArray(value) || value.length > 31) throw new TypeError(`${label} was not a bounded array`) + return value.map((raw) => { + const point = record(raw, label) + return { + timestamp: isoString(point['timestamp'], 'timestamp'), + count: nonNegativeInt(point['count'], 'count'), + uniques: nonNegativeInt(point['uniques'], 'uniques'), + } + }) +} + +function referrerPoints(value: unknown): ReferrerPoint[] { + if (!Array.isArray(value) || value.length > 20) throw new TypeError('referrers was not a bounded array') + return value.map((raw) => { + const point = record(raw, 'referrer') + return { + referrer: shortString(point['referrer'], 'referrer', 256), + count: nonNegativeInt(point['count'], 'count'), + uniques: nonNegativeInt(point['uniques'], 'uniques'), + } + }) +} + +function pathPoints(value: unknown): PathPoint[] { + if (!Array.isArray(value) || value.length > 20) throw new TypeError('paths was not a bounded array') + return value.map((raw) => { + const point = record(raw, 'path') + return { + path: shortString(point['path'], 'path', 512), + title: + typeof point['title'] === 'string' && point['title'].length > 0 + ? shortString(point['title'], 'title', 512) + : shortString(point['path'], 'path', 512), + count: nonNegativeInt(point['count'], 'count'), + uniques: nonNegativeInt(point['uniques'], 'uniques'), + } + }) +} + +function checkedName(value: string, label: string): string { + if (!NAME_RE.test(value)) throw new Error(`${label} is invalid`) + return value +} + +function nonNegativeInt(value: unknown, label: string): number { + if (!Number.isSafeInteger(value) || (value as number) < 0) throw new TypeError(`${label} was not a count`) + return value as number +} + +function shortString(value: unknown, label: string, max: number): string { + if (typeof value !== 'string' || value.length === 0 || value.length > max) { + throw new TypeError(`${label} was not a short string`) + } + return value +} + +function optionalIso(value: unknown, label: string): string | null { + if (value === null) return null + return isoString(value, label) +} + +function isoString(value: unknown, label: string): string { + if (typeof value !== 'string' || !Number.isFinite(Date.parse(value))) { + throw new TypeError(`${label} was not a timestamp`) + } + return value +} + +function record(value: unknown, label: string): JsonRecord { + if (!isRecord(value)) throw new TypeError(`${label} was not an object`) + return value +} + +function isRecord(value: unknown): value is JsonRecord { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} diff --git a/stats/src/index.ts b/stats/src/index.ts new file mode 100644 index 0000000..71d4f89 --- /dev/null +++ b/stats/src/index.ts @@ -0,0 +1,140 @@ +import { accessIdentity } from './access.ts' +import { dashboardDocument } from './dashboard.ts' +import { collectFleet } from './fleet.ts' +import { collectUsage } from './usage.ts' +import { collectGitHub } from './github.ts' +import { ingestRelay } from './relay.ts' +import { pruneStoredData } from './retention.ts' +import { collectSiteTraffic, siteCollectorFailureCode } from './site.ts' +import { dashboardData } from './store.ts' +import type { AppEnv } from './types.ts' + +export default { + async fetch(request: Request, env: AppEnv): Promise { + const url = new URL(request.url) + try { + if (request.method === 'GET' && url.pathname === '/healthz') { + return new Response('ok\n', { + headers: { 'content-type': 'text/plain; charset=utf-8', 'cache-control': 'no-store' }, + }) + } + + if (request.method === 'POST' && url.pathname === '/api/ingest/relay') { + return withSecurityHeaders(await ingestRelay(request, env)) + } + + if (request.method === 'GET' && url.pathname === '/api/public') { + return withSecurityHeaders( + Response.json(await dashboardData(env, false), { + headers: { 'cache-control': 'public, max-age=60, stale-while-revalidate=300' }, + }) + ) + } + + if (request.method === 'GET' && url.pathname === '/api/admin') { + const identity = await accessIdentity(request, env) + if (!identity) return withSecurityHeaders(forbidden()) + return withSecurityHeaders( + Response.json(await dashboardData(env, true), { + headers: { 'cache-control': 'no-store' }, + }) + ) + } + + if (request.method === 'GET' && url.pathname === '/') { + return htmlResponse(false) + } + + if (request.method === 'GET' && (url.pathname === '/admin' || url.pathname === '/admin/')) { + const identity = await accessIdentity(request, env) + if (!identity) return withSecurityHeaders(forbidden()) + return htmlResponse(true) + } + + return withSecurityHeaders( + new Response('not found\n', { + status: 404, + headers: { 'content-type': 'text/plain; charset=utf-8', 'cache-control': 'no-store' }, + }) + ) + } catch (error) { + console.error('stats: request failed', { + path: url.pathname, + kind: error instanceof Error ? error.name : 'unknown', + }) + return withSecurityHeaders( + Response.json( + { error: 'stats are temporarily unavailable' }, + { status: 500, headers: { 'cache-control': 'no-store' } } + ) + ) + } + }, + + async scheduled(controller: ScheduledController, env: AppEnv): Promise { + const daily = controller.cron === '17 0 * * *' + const failures: string[] = [] + try { + await collectGitHub(env, daily, controller.scheduledTime) + } catch (error) { + failures.push('github') + console.error('stats: scheduled GitHub collection failed', { + kind: error instanceof Error ? error.name : 'unknown', + }) + } + try { + await collectSiteTraffic(env, controller.scheduledTime) + } catch (error) { + failures.push('site') + console.error('stats: scheduled site collection failed', { + kind: error instanceof Error ? error.name : 'unknown', + code: siteCollectorFailureCode(error), + }) + } + try { await collectUsage(env) } catch { + failures.push('cloud:usage'); console.error('stats: Cloud usage collection unavailable') + } + try {await collectFleet(env)} catch {failures.push('cloud:fleet')} + await pruneStoredData(env.DB, controller.scheduledTime) + if (failures.length > 0) throw new Error('one or more scheduled collectors failed') + }, +} satisfies ExportedHandler + +function htmlResponse(privateView: boolean): Response { + const document = dashboardDocument(privateView) + return withSecurityHeaders( + new Response(document.html, { + headers: { + 'content-type': 'text/html; charset=utf-8', + 'cache-control': privateView ? 'no-store' : 'public, max-age=300', + 'content-security-policy': [ + "default-src 'none'", + `script-src 'nonce-${document.nonce}'`, + `style-src 'nonce-${document.nonce}'`, + "connect-src 'self'", + "img-src 'self' data:", + "base-uri 'none'", + "form-action 'none'", + "frame-ancestors 'none'", + ].join('; '), + }, + }) + ) +} + +function forbidden(): Response { + return Response.json( + { error: 'Cloudflare Access is required' }, + { status: 403, headers: { 'cache-control': 'no-store' } } + ) +} + +function withSecurityHeaders(response: Response): Response { + const headers = new Headers(response.headers) + headers.set('referrer-policy', 'no-referrer') + headers.set('x-content-type-options', 'nosniff') + headers.set('x-frame-options', 'DENY') + headers.set('permissions-policy', 'camera=(), microphone=(), geolocation=()') + headers.set('cross-origin-resource-policy', 'same-origin') + return new Response(response.body, { status: response.status, statusText: response.statusText, headers }) +} diff --git a/stats/src/relay.ts b/stats/src/relay.ts new file mode 100644 index 0000000..03ec9ff --- /dev/null +++ b/stats/src/relay.ts @@ -0,0 +1,323 @@ +import type { AppEnv, FleetDay, FleetDimensions, RelayIngestBody } from './types.ts' +import { retentionCutoffs } from './retention.ts' + +const MAX_BODY_BYTES = 256 * 1024 +const MAX_CLOCK_SKEW_SECONDS = 5 * 60 +const MAX_OBSERVATION_AGE_MS = 15 * 60_000 +const MAX_FLEET_DAYS = 14 +const MAX_LABELS = 128 +const MAX_LABEL_LENGTH = 128 +const MAX_COUNT = 100_000 +const DAY_RE = /^\d{4}-\d{2}-\d{2}$/ +const encoder = new TextEncoder() + +const BODY_KEYS = ['fleet', 'observed_at', 'relay', 'schema'] as const +const RELAY_KEYS = ['bytes_routed', 'frames_routed', 'rooms', 'sockets', 'uptime_seconds'] as const +const FLEET_KEYS = ['days', 'meaning', 'schema'] as const +const DAY_KEYS = [ + 'battery_kwh', + 'channels', + 'date', + 'drivers', + 'ftw_versions', + 'install_age', + 'price_zones', + 'reports', +] as const + +export async function ingestRelay( + request: Request, + env: AppEnv, + nowMs = Date.now() +): Promise { + const secret = env.RELAY_INGEST_SECRET + if (!secret || encoder.encode(secret).byteLength < 32) { + return jsonError(503, 'relay ingest is not configured') + } + + try { + const bodyText = await readLimitedText(request, MAX_BODY_BYTES) + await verifySignature(request, bodyText, secret, nowMs) + const body = relayBody(JSON.parse(bodyText) as unknown, nowMs) + await saveRelay(env.DB, body, nowMs) + return new Response(null, { status: 204 }) + } catch (error) { + if (error instanceof IngestError) return jsonError(error.status, error.message) + console.error('stats: relay ingest failed', { + kind: error instanceof Error ? error.name : 'unknown', + }) + return jsonError(400, 'invalid relay stats') + } +} + +export function relayBody(value: unknown, nowMs: number): RelayIngestBody { + const body = exactRecord(value, BODY_KEYS, 'body') + if (body['schema'] !== 'ftw.relay-stats/1') throw new IngestError(400, 'unknown schema') + + const observedAt = exactIso(body['observed_at'], 'observed_at') + const observedMs = Date.parse(observedAt) + if (Math.abs(nowMs - observedMs) > MAX_OBSERVATION_AGE_MS) { + throw new IngestError(400, 'stale observation') + } + + const relay = exactRecord(body['relay'], RELAY_KEYS, 'relay') + const fleet = exactRecord(body['fleet'], FLEET_KEYS, 'fleet') + if (fleet['schema'] !== 'ftw.fleet-stats/1' || fleet['meaning'] !== 'reports, not unique boxes') { + throw new IngestError(400, 'invalid fleet summary') + } + if (!Array.isArray(fleet['days']) || fleet['days'].length > MAX_FLEET_DAYS) { + throw new IngestError(400, 'too many fleet days') + } + + const days = fleet['days'].map((day) => fleetDay(day, nowMs)) + if (new Set(days.map((day) => day.date)).size !== days.length) { + throw new IngestError(400, 'duplicate fleet day') + } + + return { + schema: 'ftw.relay-stats/1', + observed_at: observedAt, + relay: { + uptime_seconds: count(relay['uptime_seconds'], 10 * 365 * 24 * 60 * 60, 'uptime_seconds'), + rooms: count(relay['rooms'], MAX_COUNT, 'rooms'), + sockets: count(relay['sockets'], MAX_COUNT, 'sockets'), + frames_routed: count(relay['frames_routed'], Number.MAX_SAFE_INTEGER, 'frames_routed'), + bytes_routed: count(relay['bytes_routed'], Number.MAX_SAFE_INTEGER, 'bytes_routed'), + }, + fleet: { + schema: 'ftw.fleet-stats/1', + meaning: 'reports, not unique boxes', + days, + }, + } +} + +async function saveRelay(db: D1Database, body: RelayIngestBody, nowMs: number): Promise { + const receivedAt = new Date(nowMs).toISOString() + const cutoff = retentionCutoffs(nowMs) + const statements = [ + db + .prepare( + `INSERT INTO relay_snapshots + (observed_at, received_at, uptime_seconds, rooms, sockets, frames_routed, bytes_routed) + VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(observed_at) DO UPDATE SET + received_at = excluded.received_at, + uptime_seconds = excluded.uptime_seconds, + rooms = excluded.rooms, + sockets = excluded.sockets, + frames_routed = excluded.frames_routed, + bytes_routed = excluded.bytes_routed` + ) + .bind( + body.observed_at, + receivedAt, + body.relay.uptime_seconds, + body.relay.rooms, + body.relay.sockets, + body.relay.frames_routed, + body.relay.bytes_routed + ), + ...body.fleet.days.map((day) => + db + .prepare( + `INSERT INTO fleet_daily (date, reports, dimensions_json, observed_at) + VALUES (?, ?, ?, ?) + ON CONFLICT(date) DO UPDATE SET + reports = excluded.reports, + dimensions_json = excluded.dimensions_json, + observed_at = excluded.observed_at + WHERE excluded.observed_at >= fleet_daily.observed_at` + ) + .bind(day.date, day.reports, JSON.stringify(dimensions(day)), body.observed_at) + ), + db + .prepare('DELETE FROM relay_snapshots WHERE observed_at < ?') + .bind(cutoff.relaySnapshots), + db.prepare('DELETE FROM fleet_daily WHERE date < ?').bind(cutoff.fleetDaily), + ] + await db.batch(statements) +} + +async function verifySignature( + request: Request, + body: string, + secret: string, + nowMs: number +): Promise { + const rawTimestamp = request.headers.get('x-ftw-timestamp') + const rawSignature = request.headers.get('x-ftw-signature') + if (!rawTimestamp || !/^\d{10}$/.test(rawTimestamp) || !rawSignature) { + throw new IngestError(401, 'missing signature') + } + const timestamp = Number(rawTimestamp) + if (Math.abs(Math.floor(nowMs / 1000) - timestamp) > MAX_CLOCK_SKEW_SECONDS) { + throw new IngestError(401, 'expired signature') + } + const match = /^v1=([0-9a-f]{64})$/i.exec(rawSignature) + if (!match?.[1]) throw new IngestError(401, 'invalid signature') + + const key = await crypto.subtle.importKey( + 'raw', + encoder.encode(secret), + { name: 'HMAC', hash: 'SHA-256' }, + false, + ['verify'] + ) + const valid = await crypto.subtle.verify( + 'HMAC', + key, + hexBytes(match[1]), + encoder.encode(`${rawTimestamp}.${body}`) + ) + if (!valid) throw new IngestError(401, 'invalid signature') +} + +async function readLimitedText(request: Request, limit: number): Promise { + const declared = Number(request.headers.get('content-length') ?? 0) + if (Number.isFinite(declared) && declared > limit) throw new IngestError(413, 'body too large') + if (!request.body) return '' + + const reader = request.body.getReader() + const decoder = new TextDecoder('utf-8', { fatal: true }) + let size = 0 + let text = '' + try { + while (true) { + const chunk = await reader.read() + if (chunk.done) break + size += chunk.value.byteLength + if (size > limit) throw new IngestError(413, 'body too large') + text += decoder.decode(chunk.value, { stream: true }) + } + text += decoder.decode() + return text + } catch (error) { + await reader.cancel().catch(() => {}) + throw error + } +} + +function fleetDay(value: unknown, nowMs: number): FleetDay { + const day = exactRecord(value, DAY_KEYS, 'fleet day') + const date = day['date'] + if (typeof date !== 'string' || !validDay(date)) throw new IngestError(400, 'invalid fleet date') + const dayMs = Date.parse(`${date}T00:00:00.000Z`) + if (dayMs > nowMs + 24 * 60 * 60_000 || dayMs < nowMs - 100 * 24 * 60 * 60_000) { + throw new IngestError(400, 'fleet date outside retention') + } + + const reports = count(day['reports'], MAX_COUNT, 'reports') + const result: FleetDay = { + date, + reports, + ftw_versions: counts(day['ftw_versions']), + channels: counts(day['channels']), + drivers: counts(day['drivers']), + battery_kwh: counts(day['battery_kwh']), + price_zones: counts(day['price_zones']), + install_age: counts(day['install_age']), + } + for (const key of ['ftw_versions', 'channels', 'battery_kwh', 'price_zones', 'install_age'] as const) { + if (sum(result[key]) !== reports) throw new IngestError(400, `${key} does not match reports`) + } + const driverReports = sum(result.drivers) + if (driverReports < reports || driverReports > reports * 16) { + throw new IngestError(400, 'drivers does not match reports') + } + return result +} + +function dimensions(day: FleetDay): FleetDimensions { + return { + ftw_versions: day.ftw_versions, + channels: day.channels, + drivers: day.drivers, + battery_kwh: day.battery_kwh, + price_zones: day.price_zones, + install_age: day.install_age, + } +} + +function counts(value: unknown): Record { + if (!isRecord(value)) throw new IngestError(400, 'dimension is not an object') + const entries = Object.entries(value) + if (entries.length > MAX_LABELS) throw new IngestError(400, 'too many labels') + const result: Record = Object.create(null) as Record + for (const [label, rawCount] of entries) { + if (label.length === 0 || label.length > MAX_LABEL_LENGTH || /[\u0000-\u001f]/.test(label)) { + throw new IngestError(400, 'invalid label') + } + result[label] = count(rawCount, MAX_COUNT, 'label count') + } + return result +} + +function count(value: unknown, max: number, label: string): number { + if (!Number.isSafeInteger(value) || (value as number) < 0 || (value as number) > max) { + throw new IngestError(400, `${label} is not a count`) + } + return value as number +} + +function sum(values: Record): number { + return Object.values(values).reduce((total, value) => total + value, 0) +} + +function exactIso(value: unknown, label: string): string { + if (typeof value !== 'string' || !Number.isFinite(Date.parse(value))) { + throw new IngestError(400, `${label} is not a timestamp`) + } + const normalized = new Date(value).toISOString() + if (normalized !== value) throw new IngestError(400, `${label} is not canonical`) + return value +} + +function validDay(value: string): boolean { + if (!DAY_RE.test(value)) return false + return new Date(`${value}T00:00:00.000Z`).toISOString().slice(0, 10) === value +} + +function exactRecord( + value: unknown, + keys: T, + label: string +): Record { + if (!isRecord(value)) throw new IngestError(400, `${label} is not an object`) + const actual = Object.keys(value).sort() + const expected = [...keys].sort() + if (actual.length !== expected.length || actual.some((key, index) => key !== expected[index])) { + throw new IngestError(400, `${label} fields do not match schema`) + } + return value as Record +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} + +function hexBytes(value: string): Uint8Array { + const bytes = new Uint8Array(new ArrayBuffer(value.length / 2)) + for (let index = 0; index < bytes.length; index += 1) { + bytes[index] = Number.parseInt(value.slice(index * 2, index * 2 + 2), 16) + } + return bytes +} + +function jsonError(status: number, message: string): Response { + return Response.json( + { error: message }, + { status, headers: { 'cache-control': 'no-store' } } + ) +} + +class IngestError extends Error { + override name = 'IngestError' + + constructor( + readonly status: number, + message: string + ) { + super(message) + } +} diff --git a/stats/src/retention.ts b/stats/src/retention.ts new file mode 100644 index 0000000..29d2542 --- /dev/null +++ b/stats/src/retention.ts @@ -0,0 +1,60 @@ +const DAY_MS = 24 * 60 * 60_000 + +export const RETENTION_DAYS = { + githubSnapshots: 90, + githubTraffic: 30, + githubDiscovery: 30, + siteTrafficComplete: 7, + relaySnapshots: 90, + fleetDaily: 90, + collectorRuns: 90, +} as const + +export interface RetentionCutoffs { + githubSnapshots: string + githubTraffic: string + githubDiscovery: string + siteTraffic: string + relaySnapshots: string + fleetDaily: string + collectorRuns: string +} + +export function retentionCutoffs(nowMs: number): RetentionCutoffs { + return { + githubSnapshots: rollingCutoff(nowMs, RETENTION_DAYS.githubSnapshots), + githubTraffic: utcDayCutoff(nowMs, RETENTION_DAYS.githubTraffic, true), + githubDiscovery: utcDayCutoff(nowMs, RETENTION_DAYS.githubDiscovery, true), + siteTraffic: utcDayCutoff(nowMs, RETENTION_DAYS.siteTrafficComplete, false), + relaySnapshots: rollingCutoff(nowMs, RETENTION_DAYS.relaySnapshots), + fleetDaily: utcDayCutoff(nowMs, RETENTION_DAYS.fleetDaily, true), + collectorRuns: rollingCutoff(nowMs, RETENTION_DAYS.collectorRuns), + } +} + +export async function pruneStoredData(db: D1Database, nowMs: number): Promise { + const cutoff = retentionCutoffs(nowMs) + await db.batch([ + db.prepare('DELETE FROM github_snapshots WHERE captured_hour < ?').bind(cutoff.githubSnapshots), + db.prepare('DELETE FROM github_traffic_daily WHERE date < ?').bind(cutoff.githubTraffic), + db.prepare('DELETE FROM github_referrers WHERE date < ?').bind(cutoff.githubDiscovery), + db.prepare('DELETE FROM github_paths WHERE date < ?').bind(cutoff.githubDiscovery), + db.prepare('DELETE FROM site_traffic_daily WHERE date < ?').bind(cutoff.siteTraffic), + db.prepare('DELETE FROM relay_snapshots WHERE observed_at < ?').bind(cutoff.relaySnapshots), + db.prepare('DELETE FROM fleet_daily WHERE date < ?').bind(cutoff.fleetDaily), + db.prepare('DELETE FROM cloud_fleet_view WHERE observed_at < ?').bind(cutoff.githubSnapshots), + db.prepare('DELETE FROM usage_snapshots WHERE captured_hour < ?').bind(cutoff.githubSnapshots), + db.prepare('DELETE FROM collector_runs WHERE finished_at < ?').bind(cutoff.collectorRuns), + ]) +} + +function rollingCutoff(nowMs: number, days: number): string { + return new Date(nowMs - days * DAY_MS).toISOString() +} + +export function utcDayCutoff(nowMs: number, days: number, includeToday: boolean): string { + const today = new Date(nowMs) + const todayMs = Date.UTC(today.getUTCFullYear(), today.getUTCMonth(), today.getUTCDate()) + const offset = includeToday ? days - 1 : days + return new Date(todayMs - offset * DAY_MS).toISOString().slice(0, 10) +} diff --git a/stats/src/site.ts b/stats/src/site.ts new file mode 100644 index 0000000..9ef99d8 --- /dev/null +++ b/stats/src/site.ts @@ -0,0 +1,301 @@ +import { saveCollectorRun } from './collector.ts' +import type { AppEnv } from './types.ts' + +const API_URL = 'https://api.cloudflare.com/client/v4/graphql' +const MAX_RESPONSE_BYTES = 256 * 1024 +const HISTORY_DAYS = 7 +const MAX_HOURLY_GROUPS_PER_DAY = 24 +const ZONE_ID_RE = /^[a-f0-9]{32}$/i +const HOSTNAME_RE = /^(?=.{1,253}$)(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/i +const HOUR_RE = /^\d{4}-\d{2}-\d{2}T\d{2}:00:00Z$/ + +type JsonRecord = Record +type GraphQLErrorCode = 'graphql-access-denied' | 'graphql-query-limit' | 'graphql-error' + +interface SiteDay { + date: string + requests: number + visits: number + responseBytes: number + sampleInterval: number +} + +export async function collectSiteTraffic(env: AppEnv, nowMs = Date.now()): Promise { + const startedAt = new Date(nowMs).toISOString() + try { + const zoneId = checkedZoneId(env.CLOUDFLARE_ZONE_ID) + const token = checkedToken(env.CLOUDFLARE_ANALYTICS_TOKEN) + const hostname = checkedHostname(env.SITE_HOSTNAME) + const days = completeDays(nowMs) + const rows = await Promise.all( + days.map(async (date) => { + const start = `${date}T00:00:00Z` + const end = new Date(Date.parse(start) + 24 * 60 * 60_000).toISOString() + const response = await fetch(API_URL, { + method: 'POST', + headers: { + accept: 'application/json', + authorization: `Bearer ${token}`, + 'content-type': 'application/json', + }, + body: JSON.stringify({ + query: analyticsQuery(), + variables: { zoneTag: zoneId, hostname, start, end }, + }), + signal: AbortSignal.timeout(15_000), + }) + if (!response.ok) { + await response.body?.cancel() + throw new CloudflareRequestError(response.status) + } + + const text = await boundedResponseText(response) + return parseAnalyticsResponse(JSON.parse(text) as unknown, [date])[0]! + }) + ) + const observedAt = new Date(nowMs).toISOString() + await env.DB.batch( + rows.map((row) => + env.DB.prepare( + `INSERT INTO site_traffic_daily + (date, hostname, requests, visits, response_bytes, sample_interval, observed_at) + VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(date) DO UPDATE SET + hostname = excluded.hostname, + requests = excluded.requests, + visits = excluded.visits, + response_bytes = excluded.response_bytes, + sample_interval = excluded.sample_interval, + observed_at = excluded.observed_at` + ).bind( + row.date, + hostname, + row.requests, + row.visits, + row.responseBytes, + row.sampleInterval, + observedAt + ) + ) + ) + await saveCollectorRun(env.DB, 'site:ftw.energy', startedAt, true, '7-day traffic window') + } catch (error) { + await saveCollectorRun(env.DB, 'site:ftw.energy', startedAt, false, siteCollectorFailureCode(error)) + throw error + } +} + +function completeDays(nowMs: number): string[] { + const today = new Date(nowMs) + today.setUTCHours(0, 0, 0, 0) + return Array.from({ length: HISTORY_DAYS }, (_, index) => { + const offset = HISTORY_DAYS - index + return new Date(today.getTime() - offset * 24 * 60 * 60_000).toISOString().slice(0, 10) + }) +} + +function analyticsQuery(): string { + return `query SiteTraffic($zoneTag: string, $hostname: string, $start: Time, $end: Time) { + viewer { + zones(filter: { zoneTag: $zoneTag }) { + traffic: httpRequestsAdaptiveGroups( + limit: 10000 + orderBy: [datetimeHour_ASC] + filter: { + datetime_geq: $start + datetime_lt: $end + clientRequestHTTPHost: $hostname + requestSource: "eyeball" + } + ) { + count + avg { sampleInterval } + sum { visits edgeResponseBytes } + dimensions { datetimeHour } + } + } + } + }` +} + +function parseAnalyticsResponse(value: unknown, days: string[]): SiteDay[] { + const root = record(value, 'Cloudflare response') + const errors = root['errors'] + if (Array.isArray(errors) && errors.length > 0) { + throw new CloudflareGraphQLError(graphQLErrorCode(errors)) + } + const data = record(root['data'], 'Cloudflare data') + const viewer = record(data['viewer'], 'Cloudflare viewer') + const zones = viewer['zones'] + if (!Array.isArray(zones) || zones.length !== 1) throw new TypeError('Cloudflare zone was not unique') + const zone = record(zones[0], 'Cloudflare zone') + const groups = zone['traffic'] + if (!Array.isArray(groups) || groups.length > days.length * MAX_HOURLY_GROUPS_PER_DAY) { + throw new TypeError('Cloudflare traffic window was not bounded') + } + + const byDate = new Map( + days.map((date) => [date, { date, requests: 0, visits: 0, responseBytes: 0, sampleInterval: 1 }]) + ) + const seenHours = new Set() + for (const raw of groups) { + const group = record(raw, 'Cloudflare traffic group') + const dimensions = record(group['dimensions'], 'Cloudflare traffic dimensions') + const hour = checkedHour(dimensions['datetimeHour']) + const day = byDate.get(hour.slice(0, 10)) + if (!day || seenHours.has(hour)) throw new TypeError('Cloudflare traffic hour was outside the window') + seenHours.add(hour) + const sum = record(group['sum'], 'Cloudflare traffic sum') + const avg = record(group['avg'], 'Cloudflare traffic average') + day.requests = safeAdd(day.requests, nonNegativeInt(group['count'], 'request count')) + day.visits = safeAdd(day.visits, nonNegativeInt(sum['visits'], 'visit count')) + day.responseBytes = safeAdd( + day.responseBytes, + nonNegativeInt(sum['edgeResponseBytes'], 'response bytes') + ) + day.sampleInterval = Math.max( + day.sampleInterval, + positiveNumber(avg['sampleInterval'], 'sample interval') + ) + } + + return days.map((date) => { + const day = byDate.get(date) + if (!day) throw new TypeError('Cloudflare traffic day was missing') + return day + }) +} + +async function boundedResponseText(response: Response): Promise { + const length = Number(response.headers.get('content-length') ?? 0) + if (Number.isFinite(length) && length > MAX_RESPONSE_BYTES) { + await response.body?.cancel() + throw new RangeError('Cloudflare response was too large') + } + if (!response.body) return '' + + const reader = response.body.getReader() + const chunks: Uint8Array[] = [] + let total = 0 + while (true) { + const { done, value } = await reader.read() + if (done) break + total += value.byteLength + if (total > MAX_RESPONSE_BYTES) { + await reader.cancel() + throw new RangeError('Cloudflare response was too large') + } + chunks.push(value) + } + const bytes = new Uint8Array(total) + let offset = 0 + for (const chunk of chunks) { + bytes.set(chunk, offset) + offset += chunk.byteLength + } + return new TextDecoder().decode(bytes) +} + +export function siteCollectorFailureCode(error: unknown): string { + if (error instanceof CloudflareRequestError) return `http-${error.status}` + if (error instanceof CloudflareGraphQLError) return error.code + if (error instanceof DOMException && (error.name === 'TimeoutError' || error.name === 'AbortError')) { + return 'request-timeout' + } + if (error instanceof RangeError) return 'response-too-large' + if (error instanceof SyntaxError) return 'invalid-json' + if (error instanceof TypeError) return 'invalid-response' + if ( + error instanceof Error && + (error.message.includes('is not configured') || error.message.includes('is invalid')) + ) { + return 'configuration-error' + } + return 'unexpected-error' +} + +function graphQLErrorCode(errors: unknown[]): GraphQLErrorCode { + const message = errors + .map((error) => { + if (typeof error !== 'object' || error === null || Array.isArray(error)) return '' + const value = (error as JsonRecord)['message'] + return typeof value === 'string' ? value.toLowerCase() : '' + }) + .join(' ') + if (/(auth|permission|access denied|not authorized|forbidden)/.test(message)) { + return 'graphql-access-denied' + } + if ( + /(limit|complex|resource|budget|timeout|too many|time range|wider than|spans|older than)/.test( + message + ) + ) { + return 'graphql-query-limit' + } + return 'graphql-error' +} + +function checkedHour(value: unknown): string { + if (typeof value !== 'string' || !HOUR_RE.test(value)) { + throw new TypeError('datetimeHour was not an hour') + } + return value +} + +function checkedZoneId(value: string | undefined): string { + if (!value || !ZONE_ID_RE.test(value)) throw new Error('CLOUDFLARE_ZONE_ID is not configured') + return value +} + +function checkedToken(value: string | undefined): string { + if (!value || value.length < 20 || value.length > 512) { + throw new Error('CLOUDFLARE_ANALYTICS_TOKEN is not configured') + } + return value +} + +function checkedHostname(value: string): string { + if (!HOSTNAME_RE.test(value)) throw new Error('SITE_HOSTNAME is invalid') + return value.toLowerCase() +} + +function nonNegativeInt(value: unknown, label: string): number { + if (!Number.isSafeInteger(value) || (value as number) < 0) throw new TypeError(`${label} was not a count`) + return value as number +} + +function safeAdd(left: number, right: number): number { + const total = left + right + if (!Number.isSafeInteger(total)) throw new TypeError('Cloudflare traffic total was too large') + return total +} + +function positiveNumber(value: unknown, label: string): number { + if (typeof value !== 'number' || !Number.isFinite(value) || value < 1) { + throw new TypeError(`${label} was not positive`) + } + return value +} + +function record(value: unknown, label: string): JsonRecord { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new TypeError(`${label} was not an object`) + } + return value as JsonRecord +} + +class CloudflareRequestError extends Error { + override name = 'CloudflareRequestError' + + constructor(readonly status: number) { + super(`Cloudflare returned HTTP ${status}`) + } +} + +class CloudflareGraphQLError extends Error { + override name = 'CloudflareGraphQLError' + + constructor(readonly code: GraphQLErrorCode) { + super('Cloudflare returned GraphQL errors') + } +} diff --git a/stats/src/store.ts b/stats/src/store.ts new file mode 100644 index 0000000..f1e502d --- /dev/null +++ b/stats/src/store.ts @@ -0,0 +1,623 @@ +import type { AppEnv, FleetDay, FleetDimensions } from './types.ts' +import { installationEstimate, type PublicFleetFeed } from './fleet.ts' +import { retentionCutoffs, utcDayCutoff } from './retention.ts' +import { parseRepos } from './github.ts' + +interface GitHubRow { + repo: string + captured_hour: string + captured_at: string + stars: number + forks: number + watchers: number + open_prs: number + draft_prs: number + dependency_prs: number + open_issues: number + merged_prs_30d: number + closed_issues_30d: number + contributors: number + pushed_at: string | null + latest_release: string | null + latest_release_at: string | null +} + +interface GitHubHistoryRow { + repo: string + date: string + stars: number + forks: number +} + +interface TrafficRow { + repo: string + date: string + views: number + unique_visitors: number + clones: number + unique_cloners: number + observed_at: string +} + +interface SiteTrafficRow { + date: string + hostname: string + requests: number + visits: number + response_bytes: number + sample_interval: number + observed_at: string +} + +interface RelayRow { + observed_at: string + uptime_seconds: number + rooms: number + sockets: number + frames_routed: number + bytes_routed: number +} + +interface FleetRow { + date: string + reports: number + dimensions_json: string + observed_at: string +} + +interface ReferrerRow { + repo: string + date: string + referrer: string + visits: number + unique_visitors: number +} + +interface PathRow { + repo: string + date: string + path: string + title: string + views: number + unique_visitors: number +} + +interface CollectorRow { + source: string + started_at: string + finished_at: string + ok: number + detail: string +} + +interface StoredFleetDay extends FleetDay { + observed_at: string +} + +const RELAY_REPORTING_MAX_AGE_MS = 15 * 60_000 + +export async function dashboardData( + env: AppEnv, + privateView: boolean, + nowMs = Date.now() +): Promise> { + const retention = retentionCutoffs(nowMs) + const githubCutoff = retention.githubSnapshots + const trafficCutoff = retention.githubTraffic + const siteTrafficCutoff = retention.siteTraffic + const relayCutoff = new Date(nowMs - 24 * 60 * 60_000).toISOString() + const fleetCutoff = retention.fleetDaily + + const [latest, history, traffic, siteTraffic, relay, fleet, referrers, paths, collectors] = + await Promise.all([ + env.DB.prepare( + `WITH ranked AS ( + SELECT *, ROW_NUMBER() OVER (PARTITION BY repo ORDER BY captured_hour DESC) AS rank + FROM github_snapshots + ) + SELECT repo, captured_hour, captured_at, stars, forks, watchers, open_prs, + draft_prs, dependency_prs, open_issues, merged_prs_30d, + closed_issues_30d, contributors, pushed_at, latest_release, + latest_release_at + FROM ranked WHERE rank = 1` + ).all(), + env.DB.prepare( + `WITH ranked AS ( + SELECT repo, substr(captured_hour, 1, 10) AS date, stars, forks, + ROW_NUMBER() OVER ( + PARTITION BY repo, substr(captured_hour, 1, 10) + ORDER BY captured_hour DESC + ) AS rank + FROM github_snapshots + WHERE captured_hour >= ? + ) + SELECT repo, date, stars, forks FROM ranked WHERE rank = 1 + ORDER BY date ASC, repo ASC` + ) + .bind(githubCutoff) + .all(), + env.DB.prepare( + `SELECT repo, date, views, unique_visitors, clones, unique_cloners, observed_at + FROM github_traffic_daily WHERE date >= ? ORDER BY date ASC, repo ASC` + ) + .bind(trafficCutoff) + .all(), + env.DB.prepare( + `SELECT date, hostname, requests, visits, response_bytes, sample_interval, observed_at + FROM site_traffic_daily WHERE date >= ? AND hostname = ? ORDER BY date ASC` + ) + .bind(siteTrafficCutoff, env.SITE_HOSTNAME) + .all(), + env.DB.prepare( + `SELECT observed_at, uptime_seconds, rooms, sockets, frames_routed, bytes_routed + FROM relay_snapshots WHERE observed_at >= ? ORDER BY observed_at DESC LIMIT 300` + ) + .bind(relayCutoff) + .all(), + env.DB.prepare( + `SELECT date, reports, dimensions_json, observed_at + FROM fleet_daily WHERE date >= ? ORDER BY date ASC` + ) + .bind(fleetCutoff) + .all(), + env.DB.prepare( + `WITH latest AS (SELECT repo, MAX(date) AS date FROM github_referrers GROUP BY repo) + SELECT r.repo, r.date, r.referrer, r.visits, r.unique_visitors + FROM github_referrers r + JOIN latest l ON l.repo = r.repo AND l.date = r.date + ORDER BY r.visits DESC` + ).all(), + env.DB.prepare( + `WITH latest AS (SELECT repo, MAX(date) AS date FROM github_paths GROUP BY repo) + SELECT p.repo, p.date, p.path, p.title, p.views, p.unique_visitors + FROM github_paths p + JOIN latest l ON l.repo = p.repo AND l.date = p.date + ORDER BY p.views DESC` + ).all(), + env.DB.prepare( + `WITH ranked AS ( + SELECT *, ROW_NUMBER() OVER (PARTITION BY source ORDER BY finished_at DESC) AS rank + FROM collector_runs + ) + SELECT source, started_at, finished_at, ok, detail + FROM ranked WHERE rank = 1 ORDER BY source` + ).all(), + ]) + + const usage = await env.DB.prepare('SELECT view_json, observed_at FROM usage_snapshots ORDER BY captured_hour DESC LIMIT 1').first<{ view_json: string; observed_at: string }>() + const usagePaused = collectors.results.some(row => row.source === 'cloud:usage' && row.ok === 1 && row.detail === 'awaiting policy approval') + const usageState = usagePaused ? 'paused' : usage ? 'connected' : 'pending' + const cloudFleet = await env.DB.prepare('SELECT observed_at, view_json FROM cloud_fleet_view WHERE id=1').first<{ observed_at: string; view_json: string }>() + const feed = cloudFleet ? JSON.parse(cloudFleet.view_json) as PublicFleetFeed : null + const repoOrder = parseRepos(env.GITHUB_REPOS) + const repositories = [...latest.results] + .filter((row) => repoOrder.includes(row.repo)) + .sort((a, b) => repoOrder.indexOf(a.repo) - repoOrder.indexOf(b.repo)) + .map(githubRepository) + const githubHistory = history.results.filter((row) => repoOrder.includes(row.repo)).map((row) => ({ + repo: row.repo, + date: row.date, + stars: row.stars, + forks: row.forks, + })) + const trafficDays = traffic.results.filter((row) => repoOrder.includes(row.repo)).map((row) => ({ + repo: row.repo, + date: row.date, + views: row.views, + unique_visitors: row.unique_visitors, + clones: row.clones, + unique_cloners: row.unique_cloners, + })) + const fleetDays = fleet.results.map(storedFleetDay) + const minimum = publicMinimum(env.PUBLIC_MIN_REPORTS) + const github = { + repositories, + totals: githubTotals(repositories, githubHistory, trafficDays, nowMs), + history: githubHistory, + traffic: trafficDays, + traffic_note: 'Unique counts are per repository and may count one person more than once.', + } + const fleetView = privateView + ? { + state: fleetDays.length > 0 ? 'visible' : 'empty', + meaning: 'daily reports, not users or unique boxes', + days: fleetDays, + } + : publicFleet(fleetDays, minimum, nowMs) + const freshness: Record = { + github: latestTimestamp(latest.results.map((row) => row.captured_at)), + github_traffic: latestTimestamp(traffic.results.map((row) => row.observed_at)), + site: latestTimestamp(siteTraffic.results.map((row) => row.observed_at)), + fleet: latestTimestamp(fleet.results.map((row) => row.observed_at)), + relay: relay.results[0]?.observed_at ?? null, + } + + const result: Record = { + schema: 'ftw.project-dashboard/1', + mode: privateView ? 'private' : 'public', + generated_at: new Date(nowMs).toISOString(), + coverage: { + repositories: repoOrder, + pending_repositories: repoOrder.filter((repo) => !repositories.some((row) => row['repo'] === repo)), + cloud: { hostname: 'cloud.ftw.energy', state: usageState }, + native_app: { state: 'not_collected', meaning: 'App Store installs and app sessions are not collected' }, + fleet: { source: feed ? 'cloud.ftw.energy' : 'relay.ftw.energy', state: feed ? 'connected' : 'retired' }, + }, + freshness: { ...freshness, cloud_fleet: cloudFleet?.observed_at ?? null, usage: usage?.observed_at ?? null }, + installation_estimate: installationEstimate(feed?.days ?? (fleetView['days'] as {date:string;reports:number|null}[] ?? []), nowMs, cloudFleet?.observed_at ?? freshness['fleet'] ?? null, feed ? 'cloud.ftw.energy' : 'retired relay archive'), + fleet_current: feed, + usage: { ...(usage ? JSON.parse(usage.view_json) as Record : { minimum: 20 }), state: usageState }, + github, + site: siteSummary(siteTraffic.results, env.SITE_HOSTNAME, nowMs), + fleet: fleetView, + relay_status: relayStatus(relay.results, nowMs), + relay_activity: publicRelayActivity(relay.results, nowMs), + } + + if (privateView) { + result['relay'] = relaySummary(relay.results) + result['discovery'] = { + referrers: referrers.results, + paths: paths.results, + } + result['collectors'] = collectors.results.map((row) => ({ + source: row.source, + started_at: row.started_at, + finished_at: row.finished_at, + ok: row.ok === 1, + detail: row.detail, + })) + } + + return result +} + +function siteSummary(rows: SiteTrafficRow[], hostname: string, nowMs: number): Record { + const cutoff = new Date(nowMs - 7 * 24 * 60 * 60_000).toISOString().slice(0, 10) + const recent = rows.filter((row) => row.date >= cutoff) + const days = recent.map((row) => ({ + date: row.date, + requests: row.requests, + visits: row.visits, + response_bytes: row.response_bytes, + sample_interval: row.sample_interval, + })) + return { + state: days.length > 0 ? 'visible' : 'empty', + hostname, + meaning: 'Cloudflare visits, not users or unique people', + sampled: recent.some((row) => row.sample_interval > 1), + totals: + days.length > 0 + ? { + visits_7d: recent.reduce((sum, row) => sum + row.visits, 0), + requests_7d: recent.reduce((sum, row) => sum + row.requests, 0), + response_bytes_7d: recent.reduce((sum, row) => sum + row.response_bytes, 0), + } + : { visits_7d: null, requests_7d: null, response_bytes_7d: null }, + days, + note: 'Visits are entry visits, not unique people. Cloudflare may sample these server-side counts.', + } +} + +export function publicFleet(days: StoredFleetDay[], minimum: number, nowMs: number): Record { + const cutoff = utcDayCutoff(nowMs, 30, true) + const recent = days.filter((day) => day.date >= cutoff) + const reports = recent.reduce((total, day) => total + day.reports, 0) + if (reports < minimum) { + return { + state: reports === 0 ? 'empty' : 'withheld', + meaning: 'daily reports, not users or unique boxes', + minimum, + reports_30d: null, + days: [], + dimensions: {}, + observed: { ftw_versions: [], drivers: [] }, + } + } + + const aggregate = emptyDimensions() + for (const day of recent) addDimensions(aggregate, day) + const safeDimensions: FleetDimensions = { + ftw_versions: visibleCounts(aggregate.ftw_versions, minimum), + channels: visibleCounts(aggregate.channels, minimum), + drivers: visibleCounts(aggregate.drivers, minimum), + battery_kwh: visibleCounts(aggregate.battery_kwh, minimum), + price_zones: visibleCounts(aggregate.price_zones, minimum), + install_age: visibleCounts(aggregate.install_age, minimum), + } + return { + state: 'visible', + meaning: 'daily reports, not users or unique boxes', + minimum, + reports_30d: reports, + days: recent.map((day) => ({ date: day.date, reports: day.reports >= minimum ? day.reports : null })), + dimensions: safeDimensions, + observed: { + ftw_versions: visibleLabels(safeDimensions.ftw_versions), + drivers: visibleLabels(safeDimensions.drivers), + }, + } +} + +function githubRepository(row: GitHubRow): Record { + return { + repo: row.repo, + captured_at: row.captured_at, + stars: row.stars, + forks: row.forks, + watchers: row.watchers, + open_prs: row.open_prs, + draft_prs: row.draft_prs, + dependency_prs: row.dependency_prs, + open_issues: row.open_issues, + merged_prs_30d: row.merged_prs_30d, + closed_issues_30d: row.closed_issues_30d, + contributors: row.contributors, + pushed_at: row.pushed_at, + latest_release: row.latest_release, + latest_release_at: row.latest_release_at, + } +} + +function githubTotals( + repositories: Record[], + history: GitHubHistoryRow[], + traffic: Array<{ + date: string + views: number + unique_visitors: number + clones: number + unique_cloners: number + }>, + nowMs: number +): Record { + const total = (key: string): number => + repositories.reduce((sum, repo) => sum + (typeof repo[key] === 'number' ? (repo[key] as number) : 0), 0) + const trafficCutoff = new Date(nowMs - 14 * 24 * 60 * 60_000).toISOString().slice(0, 10) + const recentTraffic = traffic.filter((day) => day.date >= trafficCutoff) + return { + stars: total('stars'), + stars_7d: changeForRepositories(history, repositories, 'stars', nowMs, 7), + forks: total('forks'), + forks_30d: changeForRepositories(history, repositories, 'forks', nowMs, 30), + watchers: total('watchers'), + open_prs: total('open_prs'), + draft_prs: total('draft_prs'), + dependency_prs: total('dependency_prs'), + open_issues: total('open_issues'), + merged_prs_30d: total('merged_prs_30d'), + closed_issues_30d: total('closed_issues_30d'), + contributor_identities: total('contributors'), + views_14d: recentTraffic.reduce((sum, day) => sum + day.views, 0), + unique_visitors_14d: recentTraffic.reduce((sum, day) => sum + day.unique_visitors, 0), + clones_14d: recentTraffic.reduce((sum, day) => sum + day.clones, 0), + unique_cloners_14d: recentTraffic.reduce((sum, day) => sum + day.unique_cloners, 0), + } +} + +function changeForRepositories( + history: GitHubHistoryRow[], + repositories: Record[], + key: 'stars' | 'forks', + nowMs: number, + days: number +): number | null { + const target = new Date(nowMs - days * 24 * 60 * 60_000).toISOString().slice(0, 10) + let change = 0 + for (const repository of repositories) { + const repo = repository['repo'] + const current = repository[key] + if (typeof repo !== 'string' || typeof current !== 'number') return null + const baseline = history.filter((point) => point.repo === repo && point.date <= target).at(-1) + if (!baseline) return null + change += current - baseline[key] + } + return change +} + +function currentRelayWindow(rows: RelayRow[]): { + latest: RelayRow + baseline: RelayRow + frames: number + bytes: number +} | null { + const latest = rows[0] + if (!latest) return null + let baseline = latest + let previous = latest + for (const row of rows.slice(1)) { + if ( + row.uptime_seconds > previous.uptime_seconds || + row.frames_routed > previous.frames_routed || + row.bytes_routed > previous.bytes_routed + ) { + break + } + baseline = row + previous = row + } + return { + latest, + baseline, + frames: Math.max(0, latest.frames_routed - baseline.frames_routed), + bytes: Math.max(0, latest.bytes_routed - baseline.bytes_routed), + } +} + +function relaySummary(rows: RelayRow[]): Record { + const current = currentRelayWindow(rows) + if (!current) return { state: 'empty', latest: null, window: null, series: [] } + return { + state: 'visible', + latest: current.latest, + window: { + from: current.baseline.observed_at, + to: current.latest.observed_at, + frames: current.frames, + bytes: current.bytes, + }, + series: [...rows].reverse().map((row) => ({ + observed_at: row.observed_at, + rooms: row.rooms, + sockets: row.sockets, + })), + } +} + +function publicRelayActivity(rows: RelayRow[], nowMs: number): Record { + const current = currentRelayWindow(rows) + const status = relayStatus(rows, nowMs) + if (!current) { + return { + ...status, + meaning: 'coarse aggregate process activity; not users or unique boxes', + window: null, + rooms_band: null, + sockets_band: null, + frames_band: null, + bytes_band: null, + } + } + return { + ...status, + meaning: 'coarse aggregate process activity; not users or unique boxes', + window: { + from: current.baseline.observed_at, + to: current.latest.observed_at, + }, + rooms_band: smallCountBand(current.latest.rooms), + sockets_band: smallCountBand(current.latest.sockets), + frames_band: eventCountBand(current.frames), + bytes_band: byteCountBand(current.bytes), + } +} + +function relayStatus(rows: RelayRow[], nowMs: number): Record { + const observedAt = rows[0]?.observed_at ?? null + if (!observedAt) { + return { + state: 'empty', + observed_at: null, + meaning: 'export heartbeat only; no relay load or user counts', + } + } + + const observedMs = Date.parse(observedAt) + const reporting = Number.isFinite(observedMs) && Math.max(0, nowMs - observedMs) <= RELAY_REPORTING_MAX_AGE_MS + return { + state: reporting ? 'reporting' : 'delayed', + observed_at: observedAt, + meaning: 'export heartbeat only; no relay load or user counts', + } +} + +function storedFleetDay(row: FleetRow): StoredFleetDay { + return { + date: row.date, + reports: row.reports, + ...readDimensions(row.dimensions_json), + observed_at: row.observed_at, + } +} + +function readDimensions(json: string): FleetDimensions { + try { + const raw = JSON.parse(json) as Record + return { + ftw_versions: numberRecord(raw['ftw_versions']), + channels: numberRecord(raw['channels']), + drivers: numberRecord(raw['drivers']), + battery_kwh: numberRecord(raw['battery_kwh']), + price_zones: numberRecord(raw['price_zones']), + install_age: numberRecord(raw['install_age']), + } + } catch { + return emptyDimensions() + } +} + +function numberRecord(value: unknown): Record { + if (typeof value !== 'object' || value === null || Array.isArray(value)) return {} + return Object.fromEntries( + Object.entries(value).filter((entry): entry is [string, number] => Number.isSafeInteger(entry[1]) && entry[1] >= 0) + ) +} + +function emptyDimensions(): FleetDimensions { + return { + ftw_versions: {}, + channels: {}, + drivers: {}, + battery_kwh: {}, + price_zones: {}, + install_age: {}, + } +} + +function addDimensions(target: FleetDimensions, source: FleetDimensions): void { + for (const key of Object.keys(target) as Array) { + for (const [label, count] of Object.entries(source[key])) { + const values = target[key] + const previous = Object.prototype.hasOwnProperty.call(values, label) ? values[label]! : 0 + // A driver label is data, including names such as "constructor" or + // "__proto__". Define an own value so Object's inherited keys and legacy + // prototype setter cannot change or discard the aggregate. + Object.defineProperty(values, label, { + value: previous + count, + enumerable: true, + configurable: true, + writable: true, + }) + } + } +} + +function visibleCounts(values: Record, minimum: number): Record { + return Object.fromEntries( + Object.entries(values) + .filter(([, count]) => count >= minimum) + .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])) + ) +} + +function visibleLabels(values: Record): string[] { + return Object.keys(values) + .sort((a, b) => a.localeCompare(b)) + .slice(0, 32) +} + +function smallCountBand(value: number): string { + if (value === 0) return '0' + if (value < 10) return '<10' + if (value < 100) return '10–99' + if (value < 1000) return '100–999' + return '1k+' +} + +function eventCountBand(value: number): string { + if (value === 0) return '0' + if (value < 1000) return '<1k' + if (value < 10_000) return '1k–9.9k' + if (value < 100_000) return '10k–99k' + if (value < 1_000_000) return '100k–999k' + return '1m+' +} + +function byteCountBand(value: number): string { + if (value === 0) return '0 B' + if (value < 1_000_000) return '<1 MB' + if (value < 10_000_000) return '1–9 MB' + if (value < 100_000_000) return '10–99 MB' + if (value < 1_000_000_000) return '100–999 MB' + return '1 GB+' +} + +function publicMinimum(value: string): number { + const parsed = Number(value) + return Number.isSafeInteger(parsed) && parsed >= 10 && parsed <= 1000 ? parsed : 10 +} + +function latestTimestamp(values: Array): string | null { + return values.filter((value): value is string => value !== null).sort().at(-1) ?? null +} diff --git a/stats/src/types.ts b/stats/src/types.ts new file mode 100644 index 0000000..200c6b6 --- /dev/null +++ b/stats/src/types.ts @@ -0,0 +1,67 @@ +export interface AppEnv { + DB: D1Database + GITHUB_OWNER: string + GITHUB_REPOS: string + SITE_HOSTNAME: string + PUBLIC_MIN_REPORTS: string + GITHUB_TOKEN?: string + CLOUDFLARE_ZONE_ID?: string + CLOUDFLARE_ANALYTICS_TOKEN?: string + RELAY_INGEST_SECRET?: string + ACCESS_TEAM_DOMAIN?: string + ACCESS_AUD?: string +} + +export interface GitHubSnapshot { + repo: string + capturedHour: string + capturedAt: string + stars: number + forks: number + watchers: number + openPrs: number + draftPrs: number + dependencyPrs: number + openIssues: number + mergedPrs30d: number + closedIssues30d: number + contributors: number + pushedAt: string | null + latestRelease: string | null + latestReleaseAt: string | null +} + +export interface FleetDimensions { + ftw_versions: Record + channels: Record + drivers: Record + battery_kwh: Record + price_zones: Record + install_age: Record +} + +export interface FleetDay extends FleetDimensions { + date: string + reports: number +} + +export interface RelayIngestBody { + schema: 'ftw.relay-stats/1' + observed_at: string + relay: { + uptime_seconds: number + rooms: number + sockets: number + frames_routed: number + bytes_routed: number + } + fleet: { + schema: 'ftw.fleet-stats/1' + meaning: 'reports, not unique boxes' + days: FleetDay[] + } +} + +export interface AccessIdentity { + email: string +} diff --git a/stats/src/usage.ts b/stats/src/usage.ts new file mode 100644 index 0000000..7a868b9 --- /dev/null +++ b/stats/src/usage.ts @@ -0,0 +1,57 @@ +import { saveCollectorRun } from './collector.ts' +import type { AppEnv } from './types.ts' + +export const usageKeys = ['battery_capacity_wh', 'flexible_power_w', 'pv_rated_w', 'grid_import_wh', 'grid_export_wh', 'pv_generation_wh', 'battery_charge_wh', 'battery_discharge_wh', 'vehicle_charge_wh', 'vehicle_discharge_wh'] as const +interface Cell { state: 'visible' | 'withheld' | 'dominated' | 'empty'; value: number | null; homes: number | null } +export interface UsageView { + schema: 'ftw.project-usage/1'; observed_at: string; window_start: string; window_end: string; minimum: number + installations: Cell; metrics: Record +} +function record(v: unknown): Record { + if (!v || typeof v !== 'object' || Array.isArray(v)) throw new Error('invalid usage object') + return v as Record +} +function exact(v: Record, keys: readonly string[]) { + if (Object.keys(v).length !== keys.length || Object.keys(v).some(k => !keys.includes(k))) throw new Error('unexpected usage fields') +} +function cell(v: unknown, installation = false): Cell { + const c = record(v) + exact(c, ['state', 'value', 'homes']) + const integer = (n: unknown) => typeof n === 'number' && Number.isSafeInteger(n) && n >= 0 + if (c['state'] === 'empty' && installation && c['value'] === 0 && c['homes'] === null) return c as unknown as Cell + if (c['state'] === 'withheld' && c['value'] === null && c['homes'] === null) return c as unknown as Cell + if (!integer(c['homes']) || (c['homes'] as number) < 20) throw new Error('small usage cell') + if (c['state'] === 'dominated' && !installation && c['value'] === null) return c as unknown as Cell + if (c['state'] !== 'visible' || !integer(c['value']) || (installation && c['value'] !== c['homes'])) throw new Error('invalid usage cell') + return c as unknown as Cell +} +export function parseUsage(value: unknown, nowMs: number): UsageView { + const v = record(value) + exact(v, ['schema', 'observed_at', 'window_start', 'window_end', 'minimum', 'installations', 'metrics']) + const time = (k: string) => typeof v[k] === 'string' ? Date.parse(v[k] as string) : NaN + const observed = time('observed_at'), start = time('window_start'), end = time('window_end') + if (v['schema'] !== 'ftw.project-usage/1' || v['minimum'] !== 20 || !Number.isFinite(observed) || observed > nowMs + 60_000 || nowMs - observed > 2 * 3600_000 || end % 3600_000 !== 0 || end - start !== 24 * 3600_000 || end > observed || observed - end >= 3600_000) throw new Error('invalid usage window') + cell(v['installations'], true) + const metrics = record(v['metrics']) + exact(metrics, usageKeys) + for (const c of Object.values(metrics)) cell(c) + return v as unknown as UsageView +} +export async function collectUsage(env: AppEnv, nowMs = Date.now()): Promise { + const started = new Date(nowMs).toISOString() + try { + const response = await fetch('https://cloud.ftw.energy/v1/stats', { headers: { accept: 'application/json' }, redirect: 'error', signal: AbortSignal.timeout(15_000) }) + if (response.status === 503 && response.headers.get('FTW-Statistics-State') === 'disabled') {await response.body?.cancel(); await saveCollectorRun(env.DB,'cloud:usage',started,true,'awaiting policy approval');return} + if (!response.ok) { await response.body?.cancel(); throw new Error('usage source unavailable') } + const reader = response.body?.getReader() + if (!reader) throw new Error('empty usage body') + const chunks: Uint8Array[] = []; let size = 0 + try { while (true) { const { done, value } = await reader.read(); if (done) break; size += value.length; if (size > 32 * 1024) throw new Error('large usage body'); chunks.push(value) } } + finally { await reader.cancel() } + const bytes = new Uint8Array(size); let offset = 0 + for (const chunk of chunks) { bytes.set(chunk, offset); offset += chunk.length } + const view = parseUsage(JSON.parse(new TextDecoder().decode(bytes)), nowMs) + await env.DB.prepare('INSERT INTO usage_snapshots(captured_hour,observed_at,view_json) VALUES(?,?,?) ON CONFLICT(captured_hour) DO UPDATE SET observed_at=excluded.observed_at,view_json=excluded.view_json').bind(started.slice(0, 13) + ':00:00.000Z', view.observed_at, JSON.stringify(view)).run() + await saveCollectorRun(env.DB, 'cloud:usage', started, true, 'public group cells') + } catch (error) { await saveCollectorRun(env.DB, 'cloud:usage', started, false, 'usage source unavailable or invalid'); throw error } +} diff --git a/stats/test/worker.test.ts b/stats/test/worker.test.ts new file mode 100644 index 0000000..beda2ee --- /dev/null +++ b/stats/test/worker.test.ts @@ -0,0 +1,752 @@ +import { applyD1Migrations, env as cloudflareEnv, reset, type D1Migration } from 'cloudflare:test' +import { exportJWK, generateKeyPair, SignJWT } from 'jose' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import worker from '../src/index.ts' +import { collectUsage, parseUsage, usageKeys } from '../src/usage.ts' +import { installationEstimate, parseFleet, collectFleet } from '../src/fleet.ts' +import { collectGitHub } from '../src/github.ts' +import { ingestRelay } from '../src/relay.ts' +import { pruneStoredData, retentionCutoffs, utcDayCutoff } from '../src/retention.ts' +import { collectSiteTraffic } from '../src/site.ts' +import { dashboardData, publicFleet } from '../src/store.ts' +import type { AppEnv, RelayIngestBody } from '../src/types.ts' + +interface TestEnv extends AppEnv { + TEST_MIGRATIONS: D1Migration[] +} + +const env = cloudflareEnv as unknown as TestEnv +const secret = '0123456789abcdef0123456789abcdef' + +beforeEach(async () => { + await reset() + await applyD1Migrations(env.DB, env.TEST_MIGRATIONS) +}) + +afterEach(() => { + vi.unstubAllGlobals() +}) + +describe('project stats Worker', () => { + it('serves a strict public shell and an empty source-backed response', async () => { + const page = await worker.fetch(new Request('https://stats.ftw.energy/'), env) + expect(page.status).toBe(200) + expect(page.headers.get('content-security-policy')).toContain("default-src 'none'") + expect(page.headers.get('x-frame-options')).toBe('DENY') + const html = await page.text() + expect(html).toContain('FTW development and energy.') + expect(html).toContain('Project at a glance') + expect(html).toContain('What this page covers') + expect(html).toContain('Daily counts stay hidden while the public sample is small.') + + const response = await worker.fetch(new Request('https://stats.ftw.energy/api/public'), env) + const data = await response.json>() + expect(response.status).toBe(200) + expect(data.mode).toBe('public') + expect(data.github.repositories).toEqual([]) + expect(data.fleet).toMatchObject({ + state: 'empty', + reports_30d: null, + minimum: 10, + observed: { ftw_versions: [], drivers: [] }, + }) + expect(data.relay_status).toEqual({ + state: 'empty', + observed_at: null, + meaning: 'export heartbeat only; no relay load or user counts', + }) + expect(data.relay_activity).toMatchObject({ + state: 'empty', + rooms_band: null, + sockets_band: null, + frames_band: null, + bytes_band: null, + }) + }) + + it('fails the private view closed without a verified Access token', async () => { + const api = await worker.fetch(new Request('https://stats.ftw.energy/api/admin'), env) + const page = await worker.fetch(new Request('https://stats.ftw.energy/admin'), env) + expect(api.status).toBe(403) + expect(page.status).toBe(403) + }) + + it('names pending repositories and excludes removed repositories from current totals and history', async () => { + const now = new Date().toISOString() + await env.DB.batch(['ftw', 'ftwdb'].flatMap((repo) => [ + env.DB.prepare(`INSERT INTO github_snapshots + (repo, captured_hour, captured_at, stars, forks, watchers, open_prs, draft_prs, + dependency_prs, open_issues, merged_prs_30d, closed_issues_30d, contributors) + VALUES (?, ?, ?, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0)`).bind(repo, now, now), + env.DB.prepare(`INSERT INTO github_traffic_daily + (repo, date, views, unique_visitors, clones, unique_cloners, observed_at) + VALUES (?, ?, 9, 0, 0, 0, ?)`).bind(repo, now.slice(0, 10), now), + ])) + const data = await dashboardData({ ...env, GITHUB_REPOS: 'ftw,ftw-cloud,ftw-app' }, false) as Record + expect(data.coverage.repositories).toEqual(['ftw', 'ftw-cloud', 'ftw-app']) + expect(data.coverage.pending_repositories).toEqual(['ftw-cloud', 'ftw-app']) + expect(data.coverage.cloud.state).toBe('pending') + expect(data.coverage.fleet.source).toBe('relay.ftw.energy') + expect(data.github.totals.stars).toBe(7) + expect(data.github.totals.views_14d).toBe(9) + expect(JSON.stringify(data.github)).not.toContain('ftwdb') + }) + + it('opens the private API only after checking the Access signature, issuer and audience', async () => { + const { privateKey, publicKey } = await generateKeyPair('RS256', { extractable: true }) + const publicJwk = await exportJWK(publicKey) + const token = await new SignJWT({ type: 'app', email: 'operator@ftw.energy' }) + .setProtectedHeader({ alg: 'RS256', kid: 'test-key' }) + .setIssuer('https://test.cloudflareaccess.com') + .setAudience('test-audience') + .setIssuedAt() + .setExpirationTime('5m') + .sign(privateKey) + vi.stubGlobal('fetch', async (input: RequestInfo | URL) => { + const url = new URL(typeof input === 'string' ? input : input instanceof URL ? input : input.url) + expect(url.href).toBe('https://test.cloudflareaccess.com/cdn-cgi/access/certs') + return githubResponse({ keys: [{ ...publicJwk, alg: 'RS256', use: 'sig', kid: 'test-key' }] }) + }) + + const response = await worker.fetch( + new Request('https://stats.ftw.energy/api/admin', { + headers: { 'cf-access-jwt-assertion': token }, + }), + env + ) + const data = await response.json>() + expect(response.status).toBe(200) + expect(data['mode']).toBe('private') + expect(data).toHaveProperty('relay') + expect(data).toHaveProperty('collectors') + }) + + it('accepts a signed aggregate relay body and withholds a small public fleet', async () => { + const now = new Date() + const earlier = new Date(now.getTime() - 60_000) + const earlierPayload = relayPayload(earlier, 1) + await sendRelay(earlierPayload, earlier) + const payload = relayPayload(now, 1) + payload.relay.uptime_seconds = 360 + payload.relay.frames_routed = 1540 + payload.relay.bytes_routed = 2_005_000 + const response = await sendRelay(payload, now) + expect(response.status).toBe(204) + + const stored = await env.DB.prepare( + 'SELECT rooms, sockets FROM relay_snapshots ORDER BY observed_at DESC LIMIT 1' + ).first<{ rooms: number; sockets: number }>() + expect(stored).toEqual({ rooms: 2, sockets: 3 }) + + const publicResponse = await worker.fetch(new Request('https://stats.ftw.energy/api/public'), env) + const data = await publicResponse.json>() + expect(data.fleet).toMatchObject({ state: 'withheld', reports_30d: null, minimum: 10 }) + expect(data.fleet.observed).toEqual({ ftw_versions: [], drivers: [] }) + expect(data.fleet.dimensions).toEqual({}) + expect(JSON.stringify(data.fleet)).not.toContain('v1.16.1-beta.22') + expect(JSON.stringify(data.fleet)).not.toContain('easee_cloud') + expect(JSON.stringify(data.fleet)).not.toContain('SE3') + expect(JSON.stringify(data.fleet)).not.toContain('5-15') + expect(JSON.stringify(data.fleet)).not.toContain('0-1m') + expect(data.relay_status).toMatchObject({ state: 'reporting', observed_at: now.toISOString() }) + expect(data.relay_activity).toMatchObject({ + state: 'reporting', + observed_at: now.toISOString(), + rooms_band: '<10', + sockets_band: '<10', + frames_band: '1k–9.9k', + bytes_band: '1–9 MB', + }) + for (const field of ['rooms', 'sockets', 'frames_routed', 'bytes_routed', 'latest', 'series']) { + expect(JSON.stringify(data.relay_activity)).not.toContain('"' + field + '"') + } + expect(data.freshness.relay).toBe(now.toISOString()) + expect(data.freshness.fleet).toBe(now.toISOString()) + }) + + it('shows only version and driver labels whose own count reaches ten', async () => { + const now = new Date() + const payload = relayPayload(now, 20) + payload.fleet.days[0]!.channels = { beta: 9, unknown: 11 } + payload.fleet.days[0]!.ftw_versions = { 'common-version': 10, 'rare-version': 9, 'other-version': 1 } + payload.fleet.days[0]!.drivers = { easee_cloud: 10, solis: 9, modbus: 1 } + await sendRelay(payload, now) + + const response = await worker.fetch(new Request('https://stats.ftw.energy/api/public'), env) + const data = await response.json>() + expect(data.fleet.reports_30d).toBe(20) + expect(data.fleet.days[0].reports).toBe(20) + expect(data.fleet.dimensions.channels).toEqual({ unknown: 11 }) + expect(data.fleet.dimensions.ftw_versions).toEqual({ 'common-version': 10 }) + expect(data.fleet.dimensions.drivers).toEqual({ easee_cloud: 10 }) + expect(data.fleet.observed).toEqual({ + ftw_versions: ['common-version'], + drivers: ['easee_cloud'], + }) + for (const label of ['rare-version', 'other-version', 'solis', 'modbus']) { + expect(JSON.stringify(data.fleet)).not.toContain(label) + } + }) + + it('counts driver labels that match inherited Object keys', async () => { + const now = new Date() + const payload = relayPayload(now, 20) + payload.fleet.days[0]!.drivers = JSON.parse('{"__proto__":10,"constructor":10}') as Record + await sendRelay(payload, now) + + const response = await worker.fetch(new Request('https://stats.ftw.energy/api/public'), env) + const data = await response.json>() + + expect(Object.prototype.hasOwnProperty.call(data.fleet.dimensions.drivers, '__proto__')).toBe(true) + expect(Object.prototype.hasOwnProperty.call(data.fleet.dimensions.drivers, 'constructor')).toBe(true) + expect(data.fleet.dimensions.drivers['__proto__']).toBe(10) + expect(data.fleet.dimensions.drivers['constructor']).toBe(10) + expect(data.fleet.observed.drivers).toEqual(['__proto__', 'constructor']) + }) + + it('uses exactly 30 UTC dates in the public fleet window at midday', () => { + const nowMs = Date.parse('2026-08-16T12:00:00.000Z') + expect(utcDayCutoff(nowMs, 30, true)).toBe('2026-07-18') + const days = Array.from({ length: 31 }, (_, index) => { + const date = new Date(Date.UTC(2026, 6, 17 + index)).toISOString().slice(0, 10) + return { + date, + reports: 1, + ftw_versions: {}, + channels: {}, + drivers: {}, + battery_kwh: {}, + price_zones: {}, + install_age: {}, + observed_at: `${date}T12:00:00.000Z`, + } + }) + + const fleet = publicFleet(days, 1, nowMs) as Record + expect(fleet.reports_30d).toBe(30) + expect(fleet.days).toHaveLength(30) + expect(fleet.days[0].date).toBe('2026-07-18') + expect(fleet.days.at(-1).date).toBe('2026-08-16') + }) + + it('rejects a changed body after it has been signed', async () => { + const now = new Date() + const payload = relayPayload(now, 1) + const body = JSON.stringify(payload) + const timestamp = String(Math.floor(now.getTime() / 1000)) + const signature = await sign(timestamp, body) + const changed = body.replace('"rooms":2', '"rooms":3') + const response = await worker.fetch( + new Request('https://stats.ftw.energy/api/ingest/relay', { + method: 'POST', + body: changed, + headers: { + 'content-type': 'application/json', + 'x-ftw-timestamp': timestamp, + 'x-ftw-signature': `v1=${signature}`, + }, + }), + env + ) + expect(response.status).toBe(401) + expect(await env.DB.prepare('SELECT COUNT(*) AS count FROM relay_snapshots').first('count')).toBe(0) + }) + + it('does not let a replayed older fleet snapshot replace newer daily totals', async () => { + const now = new Date() + const earlier = new Date(now.getTime() - 60_000) + const olderPayload = relayPayload(earlier, 1) + const newerPayload = relayPayload(now, 2) + + expect((await ingestRelay(await relayRequest(olderPayload, earlier), env, earlier.getTime())).status).toBe(204) + expect((await ingestRelay(await relayRequest(newerPayload, now), env, now.getTime())).status).toBe(204) + expect((await ingestRelay(await relayRequest(olderPayload, earlier), env, now.getTime())).status).toBe(204) + + const stored = await env.DB.prepare('SELECT reports, observed_at FROM fleet_daily') + .first<{ reports: number; observed_at: string }>() + expect(stored).toEqual({ reports: 2, observed_at: now.toISOString() }) + }) + + it('rejects expired signatures, stale observations and oversized bodies', async () => { + const now = new Date() + const expired = new Date(now.getTime() - 301_000) + const stale = new Date(now.getTime() - 15 * 60_000 - 1) + + expect( + (await ingestRelay(await relayRequest(relayPayload(now, 1), expired), env, now.getTime())).status + ).toBe(401) + expect( + (await ingestRelay(await relayRequest(relayPayload(stale, 1), now), env, now.getTime())).status + ).toBe(400) + expect( + ( + await ingestRelay( + new Request('https://stats.ftw.energy/api/ingest/relay', { + method: 'POST', + body: 'x'.repeat(256 * 1024 + 1), + }), + env, + now.getTime() + ) + ).status + ).toBe(413) + }) + + it('deletes fleet aggregates outside the 90 UTC-day window during ingest', async () => { + const now = new Date('2026-08-16T12:00:00.000Z') + await env.DB.prepare( + `INSERT INTO fleet_daily (date, reports, dimensions_json, observed_at) + VALUES (?, ?, ?, ?)` + ) + .bind('2026-05-18', 1, '{}', '2026-05-18T12:00:00.000Z') + .run() + + expect((await ingestRelay(await relayRequest(relayPayload(now, 1), now), env, now.getTime())).status).toBe(204) + expect(await env.DB.prepare('SELECT COUNT(*) AS count FROM fleet_daily').first('count')).toBe(1) + expect(await env.DB.prepare('SELECT date FROM fleet_daily').first('date')).toBe('2026-08-16') + }) + + it('deletes every stored series outside its declared retention window', async () => { + const nowMs = Date.parse('2026-08-16T12:00:00.000Z') + expect(retentionCutoffs(nowMs)).toEqual({ + githubSnapshots: '2026-05-18T12:00:00.000Z', + githubTraffic: '2026-07-18', + githubDiscovery: '2026-07-18', + siteTraffic: '2026-08-09', + relaySnapshots: '2026-05-18T12:00:00.000Z', + fleetDaily: '2026-05-19', + collectorRuns: '2026-05-18T12:00:00.000Z', + }) + + await env.DB.batch([ + ...['2026-05-17T12:00:00.000Z', '2026-05-18T12:00:00.000Z'].map((captured) => + env.DB + .prepare( + `INSERT INTO github_snapshots + (repo, captured_hour, captured_at, stars, forks, watchers, open_prs, draft_prs, + dependency_prs, open_issues, merged_prs_30d, closed_issues_30d, contributors) + VALUES ('ftw', ?, ?, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)` + ) + .bind(captured, captured) + ), + ...['2026-07-17', '2026-07-18'].map((date) => + env.DB + .prepare( + `INSERT INTO github_traffic_daily + (repo, date, views, unique_visitors, clones, unique_cloners, observed_at) + VALUES ('ftw', ?, 0, 0, 0, 0, ?)` + ) + .bind(date, `${date}T12:00:00.000Z`) + ), + ...['2026-07-17', '2026-07-18'].map((date) => + env.DB + .prepare( + `INSERT INTO github_referrers (repo, date, referrer, visits, unique_visitors) + VALUES ('ftw', ?, 'github.com', 0, 0)` + ) + .bind(date) + ), + ...['2026-07-17', '2026-07-18'].map((date) => + env.DB + .prepare( + `INSERT INTO github_paths (repo, date, path, title, views, unique_visitors) + VALUES ('ftw', ?, '/', 'FTW', 0, 0)` + ) + .bind(date) + ), + ...['2026-08-08', '2026-08-09'].map((date) => + env.DB + .prepare( + `INSERT INTO site_traffic_daily + (date, hostname, requests, visits, response_bytes, sample_interval, observed_at) + VALUES (?, 'app.ftw.energy', 0, 0, 0, 1, ?)` + ) + .bind(date, `${date}T12:00:00.000Z`) + ), + ...['2026-05-17T12:00:00.000Z', '2026-05-18T12:00:00.000Z'].map((observed) => + env.DB + .prepare( + `INSERT INTO relay_snapshots + (observed_at, received_at, uptime_seconds, rooms, sockets, frames_routed, bytes_routed) + VALUES (?, ?, 0, 0, 0, 0, 0)` + ) + .bind(observed, observed) + ), + ...['2026-05-18', '2026-05-19'].map((date) => + env.DB + .prepare( + `INSERT INTO fleet_daily (date, reports, dimensions_json, observed_at) + VALUES (?, 0, '{}', ?)` + ) + .bind(date, `${date}T12:00:00.000Z`) + ), + ...['2026-05-17T12:00:00.000Z', '2026-05-18T12:00:00.000Z'].map((finished) => + env.DB + .prepare( + `INSERT INTO collector_runs (source, started_at, finished_at, ok, detail) + VALUES (?, ?, ?, 1, 'test')` + ) + .bind(`source:${finished}`, finished, finished) + ), + ]) + + await pruneStoredData(env.DB, nowMs) + + for (const table of [ + 'github_snapshots', + 'github_traffic_daily', + 'github_referrers', + 'github_paths', + 'site_traffic_daily', + 'relay_snapshots', + 'fleet_daily', + 'collector_runs', + ]) { + expect(await env.DB.prepare(`SELECT COUNT(*) AS count FROM ${table}`).first('count')).toBe(1) + } + }) + + it('rejects fields that could turn an aggregate into an identity log', async () => { + const now = new Date() + const payload = relayPayload(now, 1) as RelayIngestBody & { gateway_id?: string } + payload.gateway_id = 'box-7' + const response = await sendRelay(payload, now) + expect(response.status).toBe(400) + }) + + it('collects repository work and reach from bounded GitHub responses', async () => { + const calls: string[] = [] + vi.stubGlobal('fetch', async (input: RequestInfo | URL, init?: RequestInit) => { + const url = new URL(typeof input === 'string' ? input : input instanceof URL ? input : input.url) + calls.push(url.pathname + url.search) + const headers = new Headers(init?.headers) + expect(headers.get('authorization')).toBe('Bearer github-test-token') + + if (url.pathname === '/repos/srcfl/ftw') { + return githubResponse({ + stargazers_count: 14, + forks_count: 9, + subscribers_count: 2, + pushed_at: '2026-08-14T11:00:00Z', + }) + } + if (url.pathname === '/repos/srcfl/ftw/pulls') { + return githubResponse([ + { draft: true, user: { login: 'fredde', type: 'User' } }, + { draft: false, user: { login: 'dependabot[bot]', type: 'Bot' } }, + ]) + } + if (url.pathname === '/repos/srcfl/ftw/issues') { + return githubResponse([{ number: 1 }, { number: 2, pull_request: {} }]) + } + if (url.pathname === '/repos/srcfl/ftw/contributors') { + return githubResponse([{ login: 'fredde' }, { login: 'dependabot[bot]' }]) + } + if (url.pathname === '/search/issues') { + return githubResponse({ total_count: url.searchParams.get('q')?.includes('is:merged') ? 8 : 5 }) + } + if (url.pathname === '/repos/srcfl/ftw/releases') { + return githubResponse([ + { tag_name: 'v2.3.2', published_at: '2026-08-01T18:00:00Z', draft: false, prerelease: false }, + { tag_name: 'v1.16.1-beta.22', published_at: '2026-08-13T18:00:00Z', draft: false, prerelease: true }, + { tag_name: 'unpublished', published_at: '2026-08-14T18:00:00Z', draft: true }, + ]) + } + return new Response(null, { status: 404 }) + }) + + await collectGitHub(env, false, Date.UTC(2026, 7, 14, 12)) + const row = await env.DB.prepare('SELECT * FROM github_snapshots WHERE repo = ?') + .bind('ftw') + .first>() + expect(row).toMatchObject({ + stars: 14, + forks: 9, + watchers: 2, + open_prs: 2, + draft_prs: 1, + dependency_prs: 1, + open_issues: 1, + merged_prs_30d: 8, + closed_issues_30d: 5, + contributors: 2, + latest_release: 'v1.16.1-beta.22', + }) + expect(calls).toHaveLength(7) + }) + + it('collects 7 complete days of server-side site traffic without visitor ids', async () => { + const now = new Date() + now.setUTCHours(12, 0, 0, 0) + const nowMs = now.getTime() + const todayMs = Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate()) + const firstDay = new Date(todayMs - 7 * 86400000).toISOString().slice(0, 10) + const lastDay = new Date(todayMs - 86400000).toISOString().slice(0, 10) + const trafficWindows: Array<{ start: string; end: string }> = [] + vi.stubGlobal('fetch', async (input: RequestInfo | URL, init?: RequestInit) => { + expect(String(input)).toBe('https://api.cloudflare.com/client/v4/graphql') + const headers = new Headers(init?.headers) + expect(headers.get('authorization')).toBe('Bearer cloudflare-analytics-test-token') + const body = JSON.parse(String(init?.body)) as { query: string; variables: Record } + expect(body.variables.zoneTag).toBe('0123456789abcdef0123456789abcdef') + expect(body.variables.hostname).toBe('app.ftw.energy') + const start = body.variables.start! + const end = body.variables.end! + expect(Date.parse(end) - Date.parse(start)).toBe(86400000) + trafficWindows.push({ start, end }) + expect(body.query.match(/httpRequestsAdaptiveGroups/g) ?? []).toHaveLength(1) + expect(body.query).toContain('orderBy: [datetimeHour_ASC]') + expect(body.query).toContain('clientRequestHTTPHost: $hostname') + expect(body.query).toContain('dimensions { datetimeHour }') + const index = Math.floor( + (Date.parse(start) - Date.parse(`${firstDay}T00:00:00Z`)) / 86400000 + ) + const date = start.slice(0, 10) + const traffic = [ + { + count: 100 + index, + avg: { sampleInterval: index === 6 ? 5 : 1 }, + sum: { visits: index + 1, edgeResponseBytes: 1000 * (index + 1) }, + dimensions: { datetimeHour: `${date}T12:00:00Z` }, + }, + ] + return githubResponse({ data: { viewer: { zones: [{ traffic }] } }, errors: null }) + }) + + await collectSiteTraffic(env, nowMs) + + expect(trafficWindows).toHaveLength(7) + const starts = trafficWindows.map((window) => window.start).sort() + expect(starts[0]).toBe(`${firstDay}T00:00:00Z`) + expect(starts.at(-1)).toBe(`${lastDay}T00:00:00Z`) + + const rows = await env.DB.prepare( + 'SELECT date, requests, visits, response_bytes, sample_interval FROM site_traffic_daily ORDER BY date' + ).all>() + expect(rows.results).toHaveLength(7) + expect(rows.results[0]).toMatchObject({ date: firstDay, requests: 100, visits: 1 }) + expect(rows.results.at(-1)).toMatchObject({ + date: lastDay, + requests: 106, + visits: 7, + response_bytes: 7000, + sample_interval: 5, + }) + + const response = await worker.fetch(new Request('https://stats.ftw.energy/api/public'), env) + const data = await response.json>() + expect(data.site).toMatchObject({ + state: 'visible', + meaning: 'Cloudflare visits, not users or unique people', + sampled: true, + totals: { visits_7d: 28, requests_7d: 721, response_bytes_7d: 28000 }, + }) + expect(JSON.stringify(data.site)).not.toContain('visitor_id') + }) + + it('stores a safe cause when Cloudflare rejects an expensive GraphQL query', async () => { + vi.stubGlobal('fetch', async () => + githubResponse({ + data: null, + errors: [ + { + message: + 'zone 0123456789abcdef0123456789abcdef cannot request a time range wider than 1d but your query time range spans 2w', + }, + ], + }) + ) + + await expect(collectSiteTraffic(env, Date.UTC(2026, 7, 16, 12))).rejects.toMatchObject({ + name: 'CloudflareGraphQLError', + }) + + const run = await env.DB.prepare( + 'SELECT ok, detail FROM collector_runs WHERE source = ? ORDER BY finished_at DESC LIMIT 1' + ) + .bind('site:ftw.energy') + .first<{ ok: number; detail: string }>() + expect(run).toEqual({ ok: 0, detail: 'graphql-query-limit' }) + expect(JSON.stringify(run)).not.toContain('0123456789abcdef0123456789abcdef') + }) + + it('retries site traffic on the hourly schedule', async () => { + let siteCalls = 0 + vi.stubGlobal('fetch', async (input: RequestInfo | URL) => { + const url = new URL(typeof input === 'string' ? input : input instanceof URL ? input : input.url) + if (url.hostname === 'cloud.ftw.energy' && url.pathname === '/v1/fleet/public') return Response.json(fleetView(Date.now())) + if (url.hostname === 'cloud.ftw.energy') return Response.json(usageView(Date.now())) + if (url.hostname === 'api.cloudflare.com') { + siteCalls += 1 + return githubResponse({ data: { viewer: { zones: [{ traffic: [] }] } }, errors: null }) + } + if (url.pathname === '/repos/srcfl/ftw') { + return githubResponse({ + stargazers_count: 0, + forks_count: 0, + subscribers_count: 0, + pushed_at: null, + }) + } + if ( + url.pathname === '/repos/srcfl/ftw/pulls' || + url.pathname === '/repos/srcfl/ftw/issues' || + url.pathname === '/repos/srcfl/ftw/contributors' + ) { + return githubResponse([]) + } + if (url.pathname === '/search/issues') return githubResponse({ total_count: 0 }) + if (url.pathname === '/repos/srcfl/ftw/releases') return githubResponse([]) + return new Response(null, { status: 404 }) + }) + + await worker.scheduled( + { cron: '3 * * * *', scheduledTime: Date.UTC(2026, 7, 16, 9, 3), noRetry() {} }, + env + ) + + expect(siteCalls).toBe(7) + expect(await env.DB.prepare('SELECT COUNT(*) AS count FROM site_traffic_daily').first('count')).toBe(7) + const run = await env.DB.prepare( + 'SELECT ok, detail FROM collector_runs WHERE source = ? ORDER BY finished_at DESC LIMIT 1' + ) + .bind('site:ftw.energy') + .first<{ ok: number; detail: string }>() + expect(run).toEqual({ ok: 1, detail: '7-day traffic window' }) + }) +}) + +function relayPayload(now: Date, reports: number): RelayIngestBody { + return { + schema: 'ftw.relay-stats/1', + observed_at: now.toISOString(), + relay: { + uptime_seconds: 300, + rooms: 2, + sockets: 3, + frames_routed: 40, + bytes_routed: 5000, + }, + fleet: { + schema: 'ftw.fleet-stats/1', + meaning: 'reports, not unique boxes', + days: [ + { + date: now.toISOString().slice(0, 10), + reports, + ftw_versions: { 'v1.16.1-beta.22': reports }, + channels: { beta: reports }, + drivers: { easee_cloud: reports }, + battery_kwh: { '5-15': reports }, + price_zones: { SE3: reports }, + install_age: { '0-1m': reports }, + }, + ], + }, + } +} + +async function sendRelay(payload: unknown, now: Date): Promise { + return worker.fetch(await relayRequest(payload, now), env) +} + +async function relayRequest(payload: unknown, signedAt: Date): Promise { + const body = JSON.stringify(payload) + const timestamp = String(Math.floor(signedAt.getTime() / 1000)) + const signature = await sign(timestamp, body) + return new Request('https://stats.ftw.energy/api/ingest/relay', { + method: 'POST', + body, + headers: { + 'content-type': 'application/json', + 'x-ftw-timestamp': timestamp, + 'x-ftw-signature': `v1=${signature}`, + }, + }) +} + +async function sign(timestamp: string, body: string): Promise { + const key = await crypto.subtle.importKey( + 'raw', + new TextEncoder().encode(secret), + { name: 'HMAC', hash: 'SHA-256' }, + false, + ['sign'] + ) + const signature = await crypto.subtle.sign( + 'HMAC', + key, + new TextEncoder().encode(`${timestamp}.${body}`) + ) + return [...new Uint8Array(signature)].map((byte) => byte.toString(16).padStart(2, '0')).join('') +} + +function githubResponse(value: unknown): Response { + return new Response(JSON.stringify(value), { + headers: { 'content-type': 'application/json; charset=utf-8' }, + }) +} + +function usageView(now: number) { + const end = Math.floor(now / 3600_000) * 3600_000 + return { schema: 'ftw.project-usage/1', observed_at: new Date(now).toISOString(), window_start: new Date(end-24*3600_000).toISOString(), window_end: new Date(end).toISOString(), minimum: 20, + installations: {state:'visible',value:20,homes:20}, metrics:Object.fromEntries(usageKeys.map(k=>[k,{state:'visible',value:20000,homes:20}])) } +} +describe('Cloud usage cells', () => { + it('shows policy review as a pause without inventing zero usage', async () => { + const now = Date.now() + vi.stubGlobal('fetch', vi.fn().mockResolvedValue(new Response(null, {status:503,headers:{'FTW-Statistics-State':'disabled'}}))) + await collectUsage(env, now) + const data = await dashboardData(env, false, now) + expect(data['usage']).toMatchObject({state:'paused'}) + expect(data['coverage']).toMatchObject({cloud:{state:'paused'}}) + expect(await env.DB.prepare('SELECT COUNT(*) AS n FROM usage_snapshots').first('n')).toBe(0) + }) + it('collects only validated public cells and exposes the saved source', async () => { + const now=Date.UTC(2026,9,17,12,3), view=usageView(now) + vi.stubGlobal('fetch',vi.fn().mockResolvedValue(Response.json(view))) + await collectUsage(env,now) + const data=await dashboardData(env,false,now) + expect(data['usage']).toMatchObject({state:'connected',installations:{value:20}}) + expect(JSON.stringify(data['usage'])).not.toContain('statsId') + }) + it('rejects small, stale or extra household fields without saving them', async () => { + const now=Date.UTC(2026,9,17,12,3) + const small=usageView(now);small.metrics['pv_rated_w']!.homes=19 + expect(()=>parseUsage(small,now)).toThrow() + expect(()=>parseUsage({...usageView(now),home:'secret'},now)).toThrow() + expect(()=>parseUsage(usageView(now-3*3600_000),now)).toThrow() + vi.stubGlobal('fetch',vi.fn().mockResolvedValue(Response.json({...usageView(now),home:'secret'}))) + await expect(collectUsage(env,now)).rejects.toThrow() + expect(await env.DB.prepare('SELECT COUNT(*) AS n FROM usage_snapshots').first('n')).toBe(0) + }) +}) + +function fleetView(now: number) { + const end=Math.floor(now/86400_000)*86400_000 + return {schema:'ftw.fleet-public/1',meaning:'reports, not unique boxes',minimum:10,observed_at:new Date(now).toISOString(),days:[22,20,23,22,22,26,21].map((reports,i)=>({date:new Date(end-(7-i)*86400_000).toISOString().slice(0,10),reports,ftw_versions:{},channels:{},drivers:{},battery_kwh:{},price_zones:{},install_age:{}}))} +} +describe('installation estimates',()=>{ + it('uses seven complete days and never adds overlapping sources or today',()=>{ + const now=Date.UTC(2026,9,17,13);const v=fleetView(now) + expect(installationEstimate(v.days,now,v.observed_at,'cloud')).toMatchObject({state:'estimated',value:22,reports:156,window_start:'2026-10-10',window_end:'2026-10-16'}) + expect(installationEstimate(v.days.slice(1),now,v.observed_at,'cloud')).toMatchObject({state:'insufficient',value:null}) + }) + it('rejects small public cells and extra household fields before storage',()=>{ + const now=Date.UTC(2026,9,17,13);const v=fleetView(now) + expect(parseFleet(v,now)).toEqual(v) + v.days[0]!.drivers={privateDriver:1} as {} + expect(()=>parseFleet(v,now)).toThrow() + expect(()=>parseFleet({...fleetView(now),home:'secret'},now)).toThrow() + }) + it('collects the safe Cloud feed independently of hourly energy approval',async()=>{ + const now=Date.UTC(2026,9,17,13) + vi.stubGlobal('fetch',vi.fn().mockResolvedValue(Response.json(fleetView(now)))) + await collectFleet(env,now) + const data=await dashboardData(env,false,now) + expect(data['installation_estimate']).toMatchObject({value:22,source:'cloud.ftw.energy'}) + expect(data['coverage']).toMatchObject({fleet:{state:'connected'}}) + vi.stubGlobal('fetch',vi.fn().mockResolvedValue(new Response(null,{status:503,headers:{'FTW-Statistics-State':'disabled'}}))) + await expect(collectUsage(env,now)).resolves.toBeUndefined() + }) +}) diff --git a/stats/tsconfig.json b/stats/tsconfig.json new file mode 100644 index 0000000..cdb99b0 --- /dev/null +++ b/stats/tsconfig.json @@ -0,0 +1,17 @@ +{ + "extends": "../tsconfig.json", + "compilerOptions": { + "types": [ + "@cloudflare/workers-types", + "@cloudflare/vitest-pool-workers", + "@cloudflare/vitest-pool-workers/types" + ], + "baseUrl": "." + }, + "include": [ + "src/**/*.ts", + "test/**/*.ts", + "worker-configuration.d.ts", + "vitest.config.ts" + ] +} diff --git a/stats/vitest.config.ts b/stats/vitest.config.ts new file mode 100644 index 0000000..1140131 --- /dev/null +++ b/stats/vitest.config.ts @@ -0,0 +1,30 @@ +import { fileURLToPath } from 'node:url' +import { cloudflareTest, readD1Migrations } from '@cloudflare/vitest-pool-workers' +import { defineConfig } from 'vitest/config' + +const configPath = fileURLToPath(new URL('./wrangler.jsonc', import.meta.url)) +const migrationsPath = fileURLToPath(new URL('./migrations', import.meta.url)) +const migrations = await readD1Migrations(migrationsPath) + +export default defineConfig({ + plugins: [ + cloudflareTest({ + wrangler: { configPath }, + miniflare: { + bindings: { + GITHUB_REPOS: 'ftw', + GITHUB_TOKEN: 'github-test-token', + CLOUDFLARE_ZONE_ID: '0123456789abcdef0123456789abcdef', + CLOUDFLARE_ANALYTICS_TOKEN: 'cloudflare-analytics-test-token', + RELAY_INGEST_SECRET: '0123456789abcdef0123456789abcdef', + ACCESS_TEAM_DOMAIN: 'https://test.cloudflareaccess.com', + ACCESS_AUD: 'test-audience', + TEST_MIGRATIONS: migrations, + }, + }, + }), + ], + test: { + include: ['stats/test/**/*.test.ts'], + }, +}) diff --git a/stats/worker-configuration.d.ts b/stats/worker-configuration.d.ts new file mode 100644 index 0000000..d39d811 --- /dev/null +++ b/stats/worker-configuration.d.ts @@ -0,0 +1,25 @@ +/* eslint-disable */ +// Generated by Wrangler by running `wrangler types --config=stats/wrangler.jsonc --include-runtime=false stats/worker-configuration.d.ts` (hash: 5c71bd7b19b04bb9824ffb139cc2964c) +interface __BaseEnv_Env { + DB: D1Database; + GITHUB_OWNER: "srcfl"; + GITHUB_REPOS: "ftw,device-drivers,ftw-cloud,ftw-app,ftw-webapp,ftw-web"; + SITE_HOSTNAME: "app.ftw.energy"; + PUBLIC_MIN_REPORTS: "10"; + CLOUDFLARE_ZONE_ID: "b8e2cf4b19abbeb535cca652750916c9"; + ACCESS_TEAM_DOMAIN: "https://sourceful.cloudflareaccess.com"; + ACCESS_AUD: "980a936eae5215f4484780b68996a4985f435a0e19cd8d11c13225b3f74844cd"; +} +declare namespace Cloudflare { + interface GlobalProps { + mainModule: typeof import("./src/index"); + } + interface Env extends __BaseEnv_Env {} +} +interface Env extends __BaseEnv_Env {} +type StringifyValues> = { + [Binding in keyof EnvType]: EnvType[Binding] extends string ? EnvType[Binding] : string; +}; +declare namespace NodeJS { + interface ProcessEnv extends StringifyValues> {} +} diff --git a/stats/wrangler.jsonc b/stats/wrangler.jsonc new file mode 100644 index 0000000..8578d97 --- /dev/null +++ b/stats/wrangler.jsonc @@ -0,0 +1,41 @@ +{ + "$schema": "../node_modules/wrangler/config-schema.json", + "name": "ftw-project-stats", + "main": "src/index.ts", + "compatibility_date": "2026-08-14", + "compatibility_flags": ["nodejs_compat"], + "workers_dev": false, + "routes": [ + { + "pattern": "stats.ftw.energy", + "custom_domain": true + } + ], + "vars": { + "GITHUB_OWNER": "srcfl", + "GITHUB_REPOS": "ftw,device-drivers,ftw-cloud,ftw-app,ftw-webapp,ftw-web", + "SITE_HOSTNAME": "app.ftw.energy", + "PUBLIC_MIN_REPORTS": "10", + "CLOUDFLARE_ZONE_ID": "b8e2cf4b19abbeb535cca652750916c9", + "ACCESS_TEAM_DOMAIN": "https://sourceful.cloudflareaccess.com", + "ACCESS_AUD": "980a936eae5215f4484780b68996a4985f435a0e19cd8d11c13225b3f74844cd" + }, + "d1_databases": [ + { + "binding": "DB", + "database_name": "ftw-project-stats", + "database_id": "0c4e65ba-58a8-4ae4-8874-83d130a2f516", + "migrations_dir": "migrations" + } + ], + "triggers": { + "crons": ["3 * * * *", "17 0 * * *"] + }, + "observability": { + "enabled": true, + "logs": { + "enabled": true, + "invocation_logs": true + } + } +} diff --git a/tests/stats-dashboard.test.ts b/tests/stats-dashboard.test.ts new file mode 100644 index 0000000..a995b63 --- /dev/null +++ b/tests/stats-dashboard.test.ts @@ -0,0 +1,80 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { dashboardDocument } from '../stats/src/dashboard.ts' + +afterEach(() => { + vi.unstubAllGlobals() + document.documentElement.innerHTML = '' +}) + +async function render(privateView = false, extra: Record = {}) { + const now = Date.now() + const hoursAgo = (hours: number) => new Date(now - hours * 3600_000).toISOString() + const data = { + mode: privateView ? 'private' : 'public', + generated_at: hoursAgo(0), + coverage: { repositories: ['ftw', 'device-drivers', 'ftw-cloud', 'ftw-app', 'ftw-webapp', 'ftw-web'], pending_repositories: ['ftw-cloud', 'ftw-app', 'ftw-web'] }, + freshness: { github: hoursAgo(6), github_traffic: hoursAgo(6), site: hoursAgo(0), fleet: hoursAgo(1), relay: hoursAgo(0) }, + github: { repositories: [{ repo: 'ftw', latest_release: 'v0.144.1-beta.1' }, { repo: 'ftw-cloud' }], totals: { stars: 21 }, history: [{repo:'ftw',date:'2026-10-15',stars:20},{repo:'ftw',date:'2026-10-16',stars:21},{repo:'ftw',date:'2026-10-17',stars:21},{repo:'ftw-cloud',date:'2026-10-17',stars:1}], traffic:[{repo:'ftw',date:'2026-10-16',views:9}] }, + site: { hostname: 'app.ftw.energy', totals: { visits_7d: 227 } }, + fleet: { state: 'withheld', minimum: 10, reports_30d: null, days: [] }, + relay_activity: { state: 'reporting', rooms_band: '10–99' }, + relay: { state: 'visible', latest: { rooms: 17, sockets: 20, observed_at: hoursAgo(0) } }, + } + Object.assign(data, extra) + vi.stubGlobal('fetch', vi.fn().mockResolvedValue(Response.json(data))) + const html = dashboardDocument(privateView).html + document.documentElement.innerHTML = html + const script = document.querySelector('script')!.textContent! + new Function(script)() + await vi.waitFor(() => expect(document.getElementById('stars')!.textContent).toBe('21')) +} + +describe('stats dashboard coverage', () => { + it('distinguishes a seven-day installation estimate from paused energy collection', async () => { + await render(false, { + installation_estimate: {state:'estimated',value:22,reports:156,days:7,window_start:'2026-10-10',window_end:'2026-10-16',source:'cloud.ftw.energy'}, + usage: {state:'paused'}, + }) + expect(document.getElementById('overview-installations')!.textContent).toBe('≈ 22') + expect(document.getElementById('installation-period')!.textContent).toContain('2026-10-10 to 2026-10-16') + expect(document.getElementById('installation-method')!.textContent).toContain('not people or all-time installs') + expect(document.getElementById('usage-age')!.textContent).toBe('Waiting for review') + expect(document.getElementById('usage-note')!.textContent).toContain('off pending privacy review') + expect(document.getElementById('usage-power')!.textContent).toBe('—') + }) + + it('shows the whole configured stack, names missing sources and keeps relay load out of the public view', async () => { + await render() + const rows = document.querySelectorAll('#repositories tr') + expect(rows).toHaveLength(6) + expect(document.querySelector('#repositories a[href="https://github.com/srcfl/ftw-app"]')).not.toBeNull() + expect(document.querySelector('#repositories a[href="https://github.com/srcfl/ftw-cloud"]')).not.toBeNull() + expect(rows[3]!.textContent).toContain('Awaiting first collection') + expect(rows[0]!.textContent).toContain('v0.144.1-beta.1') + expect(document.getElementById('site-hostname')!.textContent).toBe('app.ftw.energy') + expect(document.getElementById('relay-section')!.hidden).toBe(true) + expect(document.getElementById('coverage-title')!.parentElement!.parentElement!.parentElement!.textContent).toContain('App Store installs are not collected') + expect(document.getElementById('overview-installations')!.textContent).toBe('—') + expect(document.querySelectorAll('#star-chart circle')).toHaveLength(3) + expect(document.querySelectorAll('#star-chart polyline')).toHaveLength(2) + expect(document.querySelector('#traffic-chart .bar-wrap')!.getAttribute('title')).toContain('1/2 repos') + expect(document.getElementById('fleet-source')!.textContent).toContain('retired relay') + }) + + it('grades the daily GitHub traffic feed separately from hourly sources', async () => { + await render() + const items = [...document.querySelectorAll('#freshness .freshness-item')] + expect(items.find((item) => item.textContent!.startsWith('Repo traffic'))!.querySelector('.dot.ok')).not.toBeNull() + expect(items.find((item) => item.textContent!.startsWith('GitHub'))!.querySelector('.dot.warn')).not.toBeNull() + expect(items.find((item) => item.textContent!.startsWith('Energy reports'))!.querySelector('.dot.idle')).not.toBeNull() + expect(items.some(item => item.textContent!.includes('Legacy'))).toBe(false) + }) + + it('keeps the old relay counters available as private migration diagnostics', async () => { + await render(true) + expect(document.getElementById('relay-section')!.hidden).toBe(false) + expect(document.getElementById('relay-title')!.textContent).toBe('Legacy relay load') + expect(document.getElementById('relay-rooms')!.textContent).toBe('17') + expect(document.getElementById('relay-note')!.textContent).toContain('do not measure FTW Cloud') + }) +})