diff --git a/.config/nextest.toml b/.config/nextest.toml index 14596e0d880..8f8d2436b2a 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -24,10 +24,17 @@ fail-fast = false # wrapper retained one database per concurrently running test process. test-threads = 8 +[test-groups.postgres-cluster-global] +# These tests inspect cluster-wide activity or create least-privilege sessions, +# so database-per-test isolation alone cannot make them independent. +max-threads = 1 + [[profile.postgres-ci.overrides]] filter = "test(/cluster_global_/)" -# Activity scans also observe ordinary tests connecting to the same cluster. -# Reserve every worker so those tests and their wrappers cannot overlap a scan. +test-group = "postgres-cluster-global" +# The group only serializes these tests with each other. They read unfiltered, +# cluster-wide `pg_stat_activity`, so any overlapping test's session can mask +# activity; claim every worker so they run alone. threads-required = "num-test-threads" [[profile.postgres-ci.scripts]] @@ -36,3 +43,13 @@ threads-required = "num-test-threads" filter = "(test(/postgres_tests::/) or binary(/^postgres_/)) and not test(/(^|::)external_infra[^:]*::/)" setup = "postgres-template" run-wrapper = "postgres-isolation" + +[profile.flake] +# The advisory pull-request flake lane (scripts/flake-check.mjs rust) repeats +# a selection and must see every failed repetition, so no retries; the JUnit +# report gives it per-test statuses. +retries = 0 +fail-fast = false + +[profile.flake.junit] +path = "junit.xml" diff --git a/.env.example b/.env.example index f79eb23f734..3c8992a7256 100644 --- a/.env.example +++ b/.env.example @@ -52,25 +52,30 @@ REDIS_URL=redis://localhost:6379 # Emergency kill switch for all automatic partition DDL, including catch-all # advancement; audits remain enabled. # BUZZ_PARTITION_MANAGER_CREATE_ENABLED=true -# Opt in to replacing an empty partition catch-all with six months of dedicated -# monthlies, at startup and on every audit interval (default false). Has no -# effect while the kill switch above is off. -# BUZZ_PARTITION_MANAGER_ADVANCE_ENABLED=false +# Replace an empty partition catch-all with six months of dedicated monthlies, +# at startup and on every audit interval (default true). Has no effect while the +# kill switch above is off. +# BUZZ_PARTITION_MANAGER_ADVANCE_ENABLED=true # Writer-session Postgres timeouts for buzz-db-backed pools and the relay audit -# pool, all in milliseconds; 0 disables. The separately deployed push gateway -# owns its own database and session policy and does not consume these knobs. +# pool, all in milliseconds; 0 disables the lock and idle timeouts. The +# separately deployed push gateway owns its own database and session policy +# and does not consume these knobs. # lock_timeout: fail a statement that waits this long on any lock instead of # parking behind a wedged holder (default 5000). # BUZZ_DB_LOCK_TIMEOUT_MS=5000 # idle_in_transaction_session_timeout: reap sessions idle inside an open # transaction — bounds how long a wedged client can hold locks (default 60000). # BUZZ_DB_IDLE_TXN_TIMEOUT_MS=60000 -# statement_timeout: cap any single statement's runtime. Off by default — -# startup migrations/backfills legitimately run long statements. Warning: a -# pathologically low value (e.g. 1) also times out connection setup and can -# prevent any DB connection from establishing. -# BUZZ_DB_STATEMENT_TIMEOUT_MS=0 +# statement_timeout: cap any single statement's runtime (default 120000). A +# session cannot disable it: 0 or anything above 300000 installs 300000 (5 +# minutes) and logs a warning at startup. Migrations, schema destruction, and +# community-deletion inventory and purge transactions lift it for themselves. +# Generic event reads (REQ history, /query, /count) and read-replica sessions +# are further capped at 20s. Warning: a pathologically low value (e.g. 1) also +# times out connection setup and can prevent any DB connection from +# establishing. +# BUZZ_DB_STATEMENT_TIMEOUT_MS=120000 # ----------------------------------------------------------------------------- # Typesense (search) @@ -95,6 +100,8 @@ RELAY_URL=ws://localhost:3000 # NIP-PL mobile push is an explicit deployment opt-in. A gateway URL alone # never enables it. When enabled, the exact HTTP(S) delivery URL is required. +# Rollback requires replacing every enabled writer/worker process. Inspect and +# explicitly decide retained queue disposition before re-enabling (see docs/push-gateway-deployment.md). BUZZ_PUSH_ENABLED=false BUZZ_PUSH_GATEWAY_DELIVERY_URL=https://push.buzz.xyz/v1/deliveries/apns @@ -148,10 +155,20 @@ BUZZ_PUSH_GATEWAY_DELIVERY_URL=https://push.buzz.xyz/v1/deliveries/apns # Deployment-global operator-listener routes. Each configured Nostr identity # may register target pubkeys and receives mention notifications at its URL. +# URLs may use HTTP or HTTPS. Use HTTP only for operator-controlled internal +# connections: NIP-98 signs requests but does not encrypt them, and +# notifications carry full message content. # Format: pubkey:deliveryUrl;pubkey:deliveryUrl # BUZZ_OPERATOR_LISTENER_TIMEOUT_MS=5000 # BUZZ_OPERATOR_LISTENERS=<64-char hex pubkey>:https://listener.example/mentions +# Optional endpoint that receives each new per-community storage snapshot +# written by `buzz-admin storage-snapshot`, in NIP-98 relay-signed POST batches +# shaped like kgoose's storage-usage record request. One relay pod checks for a +# new snapshot every 5 minutes. Receivers must treat a repeated community and +# measured_at as already recorded. +# BUZZ_STORAGE_USAGE_URL=https://kgoose.example/cash-app/goose/v1/buzz/storage-usage/record + # Optional relay-owned KLIPY key. When set, NIP-11 advertises GIF search and # authenticated desktop clients use this relay as the metadata/search proxy. # Keep the real value in your deployment's secret manager; never commit it. @@ -223,7 +240,80 @@ BUZZ_S3_ADDRESSING_STYLE=path RUST_LOG=buzz_relay=debug,buzz_datastore=info,buzz_db=debug,buzz_auth=debug,buzz_pubsub=debug,tower_http=debug # Optional OpenTelemetry-only target filter. This is deliberately independent # from RUST_LOG so log verbosity changes cannot break trace parentage. -# BUZZ_OTEL_FILTER=buzz_relay=info,buzz_datastore=info +# BUZZ_OTEL_FILTER=buzz_relay=info,buzz_relay::access=off,buzz_datastore=info +# +# Access log: the relay writes JSON lines for security-relevant actions under +# the `buzz_relay::access` target, with an `event` field of `http`, `auth`, +# `req`, `count`, `sync`, or `close`. `http` is one line per HTTP request, written +# by one layer when the response is ready (not proof the body was delivered): +# `route` is the matched route template (never the raw path), with the response +# `status` and `time_to_response_ms` on every line. It carries the login +# (`method`, `pubkey`, `pubkey_verified`) and `outcome` (`success`, `denied`, +# `failed`, `challenged`, or `undecided` when the route recorded no decision; a +# refusal is final). `/query` and `/count` add the filter summary and +# `result_count`; `/events` adds `accepted`, `kind`, `event_id` and a fixed +# `reject_reason` code; invite and admin attachment routes add their own fields +# (`invite_id`, `expires_at`, `max_uses`, `joined`, `feedback_id`, +# `attachment_sha256`). Invite codes, tokens and Blossom read hashes are never +# logged. An attachment read with admin auth disabled logs +# `method=network_trusted` and no key. A request dropped before its response +# logs `request=aborted` with no `status`. +# Every matched route is logged except `/` (non-upgrade), `/info`, +# `/.well-known/nostr.json`, `/health`, `/_liveness` and `/_readiness`. +# Requests matching no route log only at status 400 and above, as +# `route=""`, except that the SPA and static files the relay +# serves when enabled (web assets, invite landing pages, git web pages, the +# admin bundle) log nothing at any status, including 404. Successful WebSocket +# upgrades log no `http` line. +# `auth` is one line per WebSocket login, written when it succeeds or is +# refused; every root WebSocket `auth` line carries the TCP `peer_addr`. +# `req` / `count` / `sync` is one line per WebSocket read (a NIP-AR sync +# subscription is a `sync` read), with the same filter summary +# (search text is never logged, only `search: true`). A `sync` line is written +# when its initial set completes, not when live delivery ends: its summary is +# the registered live filter (kinds and channels, never artifact predicates) +# and `events_queued` counts the initial set. `buzz_ws_reads_total` counts +# parsed reads by `type` (`req`, `count`, `sync`). The summary lists up to +# 10 kinds, channel ids, authors and `p` values each; `kind_count`, +# `channel_count`, `author_count` and `p_count` appear only when the list is +# `*` or shows fewer values than the filters held. `close` is the +# connection's one line: duration, totals, the login's `pubkey`, `method` and +# `pubkey_verified`, and `reads_suppressed` / `auth_suppressed` for repeated +# refusals past the first 16. It is written however the connection ends, with +# `refused` (`not_admitted`, `connection_limit`) when the relay refused it +# before serving it. Refusals that arrive after `close` has been written log +# their own lines. +# Successful lines and git's credential-less first request +# (`outcome=challenged`) are INFO; refusals, error statuses and aborted +# requests are WARN. A successful `/query` or `/count` that returned nothing +# (`result_count=0`) is DEBUG, since polling clients make most of them. A +# WebSocket read is WARN only when it ends in `error`; +# refused and `aborted` reads stay INFO, since `aborted` means the read ended +# before completing: the client closed the connection or sent CLOSE, a same-ID +# REQ replaced it, or the session was cancelled, all of which are routine. Turn it down with +# `buzz_relay::access=warn` (keeps refusals, errors and aborted HTTP requests, +# and of WebSocket reads only `error` outcomes) or `buzz_relay::access=off` in +# RUST_LOG. +# It is off in the OTel export by default so lines are not shipped twice. +# Removed or renamed: `NIP-42 auth successful/failed`, `WebSocket connection +# established/closed` and the `event=connect` line (folded into `close`), `HTTP +# bridge request` (folded into the bridge's `http` line), `Event ingested via +# pipeline` (the WebSocket `Event ingested` line and the `/events` line carry +# its fields), `relay invite minted`, `relay member added via invite` and `admin +# feedback attachment read` (folded into the route's `http` line), the `addr` +# field, now `peer_addr`, and the JSON `span` field (`spans` already ends with +# the current span; a line logged under an explicit parent span loses that +# parent's fields). Git subprocess stderr moved from WARN to DEBUG because hook +# output can contain message text. + +# Request header holding the real client IP, logged as `client_ip` on `http`, +# `auth`, `close` and connection-limit lines next to `peer_addr` (optional, +# case-insensitive; unset or empty turns it off). A list value logs its +# rightmost entry; a value that isn't an IP address is not logged. Set it only +# when every request reaches the relay through a proxy that overwrites this +# header (for example `cf-connecting-ip` behind Cloudflare or `x-real-ip` +# behind nginx); otherwise clients can forge it. +# BUZZ_CLIENT_IP_HEADER=x-real-ip # OTLP tracing endpoint (optional — leave unset to disable) # OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317 @@ -367,3 +457,28 @@ RUST_LOG=buzz_relay=debug,buzz_datastore=info,buzz_db=debug,buzz_auth=debug,buzz # BUZZ_TERMS_OF_SERVICE_MARKDOWN="# Terms of Service\n\nFull terms here." # BUZZ_PRIVACY_POLICY_MARKDOWN="# Privacy Policy\n\nFull policy here." # BUZZ_AGE_ATTESTATION_REQUIRED=true + +# ── NIP-FI database community source ───────────────────────────────────────── +# Where NIP-FI communities come from: "env" (BUZZ_NIP_FI_COMMUNITIES, default) +# or "database" (bindings created by provisioning; needs +# BUZZ_NIP_FI_PROVISION_BINDING and must not be combined with +# BUZZ_NIP_FI_COMMUNITIES). See docs/operator-nip-fi-bindings.md. +# BUZZ_NIP_FI_COMMUNITY_SOURCE=env +# Most community-issuer rows one snapshot may hold (1..=10000000). +# BUZZ_NIP_FI_SNAPSHOT_MAX_ROWS=1000000 +# Snapshot load timeout in ms; must be under the 30s refresh interval. +# BUZZ_NIP_FI_SNAPSHOT_REFRESH_TIMEOUT_MS=10000 +# Hosts whose "not found" is cached for 5s (1..=1000000). +# BUZZ_NIP_FI_RESOLVER_NEGATIVE_CACHE_MAX=100000 +# Concurrent live lookups; clamped below the database pool size. +# BUZZ_NIP_FI_RESOLVER_MAX_INFLIGHT=8 + +# Optional absolute legacy-v1 invite cutoff, UTC RFC3339 whole seconds. +# Unset preserves natural token expiry; never changes v2 invitations. +# BUZZ_INVITE_V1_INVALID_AFTER=2026-12-01T00:00:00Z + +# Optional NIP-FI historical verified profiles (disabled by default): issuer JSON +# entries may set public_profile=true, public_issuer_label="Example Corp", and +# verified_at_refresh_seconds=3600 (integer 0..86400). Requires a stable relay +# key and NIP-FI enforcement. Profiles are historical records and are never +# retracted. diff --git a/.github/scripts/codex-security-review.test.js b/.github/scripts/codex-security-review.test.js index 7ae3f852c3d..3d491af6c76 100644 --- a/.github/scripts/codex-security-review.test.js +++ b/.github/scripts/codex-security-review.test.js @@ -203,7 +203,7 @@ const NO_FINDINGS_REVIEW = { async function postReview(state, review = NO_FINDINGS_REVIEW) { const environment = { - CODEX_MODEL: "gpt-5.6-sol", + CODEX_MODEL: "gpt-6.1-sol", GITHUB_REPOSITORY: "block/buzz", GITHUB_RUN_ID: "1234", GITHUB_SERVER_URL: "https://github.com", diff --git a/.github/workflows/_ci-clients.yml b/.github/workflows/_ci-clients.yml index 310ab676b75..eeb5cd509b2 100644 --- a/.github/workflows/_ci-clients.yml +++ b/.github/workflows/_ci-clients.yml @@ -57,11 +57,30 @@ jobs: path: ${{ steps.pnpm-cache.outputs.STORE_PATH }} key: pnpm-${{ runner.os }}-${{ hashFiles('**/pnpm-lock.yaml') }} + # Mobile lanes run in parallel; the required check reads the combined + # matrix result. Test shards split files, so each compiles only its own. mobile: - name: Mobile + name: Mobile (${{ matrix.name }}) runs-on: ubuntu-latest timeout-minutes: 30 if: inputs.lane == 'required' && (github.event_name == 'push' || inputs.mobile) + strategy: + fail-fast: false + matrix: + include: + - name: Checks + lane: checks + - name: Tests 1/3 + lane: test + shard: 0 + - name: Tests 2/3 + lane: test + shard: 1 + - name: Tests 3/3 + lane: test + shard: 2 + - name: Android + lane: android permissions: contents: read steps: @@ -107,47 +126,152 @@ jobs: path: ~/.pub-cache key: pub-${{ runner.os }}-${{ hashFiles('mobile/pubspec.lock') }} - name: Format check + if: matrix.lane == 'checks' run: cd mobile && dart format --output=none --set-exit-if-changed . - name: Analyze + if: matrix.lane == 'checks' run: cd mobile && flutter analyze - name: Test - run: cd mobile && flutter test --dart-define=BUZZ_PUSH_GATEWAY_URL=https://push.example + if: matrix.lane == 'test' + run: /bin/bash scripts/mobile-test-shard.sh ${{ matrix.shard }} 3 --dart-define=BUZZ_PUSH_GATEWAY_URL=https://push.example + - name: Test native secp256k1 package + if: matrix.lane == 'checks' + run: cd mobile/packages/buzz_secp256k1 && dart pub get && dart test + - name: Test configured authentication domains + if: matrix.lane == 'checks' + run: >- + cd mobile && flutter test + --dart-define=BUZZ_PUSH_GATEWAY_URL=https://push.example + --dart-define=BUZZ_AUTH_DOMAINS=buzz.xyz,builderlab.xyz,builderlabstaging.xyz + test/shared/auth/community_entry_requirements_test.dart + test/features/agents/agent_creation_link_journey_test.dart + test/features/invites/invite_join_provider_test.dart + test/features/onboarding/onboarding_entry_routes_test.dart + test/shared/hosted/hosted_environment_test.dart + test/shared/hosted/self_hosted_entry_test.dart + test/features/agents/agents_signed_out_prompt_test.dart + test/features/agents/agents_restore_prompt_test.dart + test/shared/hosted/relay_auth_isolation_test.dart + test/features/onboarding/hosted_app_gate_environment_test.dart + test/features/onboarding/hosted_journeys_test.dart + test/features/onboarding/onboarding_sign_out_test.dart + test/features/onboarding/hosted_communities_test.dart + test/features/pairing/pairing_hosted_handoff_test.dart + test/shared/agents/remote_agents_test.dart + test/features/agents - name: Test mobile without push configuration + if: matrix.lane == 'checks' run: cd mobile && flutter test test/shared/push/push_unconfigured_build_test.dart - name: Test mobile recipe argument forwarding + if: matrix.lane == 'checks' run: /bin/bash scripts/test-mobile-gateway-recipes.sh + # ubuntu-latest (26.04) defaults to JDK 25, which Gradle 8.14 does not + # support. Pin the JDK 17 the 24.04 image used. + - name: Set up JDK 17 + if: matrix.lane == 'android' + uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad # v6.1.0 + with: + distribution: temurin + java-version: "17" + - name: Restore Gradle cache + if: matrix.lane == 'android' + uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + key: gradle-${{ runner.os }}-${{ hashFiles('mobile/android/**/*.gradle*', 'mobile/android/gradle/wrapper/gradle-wrapper.properties', 'mobile/pubspec.lock') }} + restore-keys: gradle-${{ runner.os }}- + # The Hermit package cache above holds only Flutter, so the build's first + # just and cmake calls download them. + - name: Fetch Hermit tools + if: matrix.lane == 'android' + run: | + scripts/ci-hermit-retry.sh ./bin/just --version + scripts/ci-hermit-retry.sh ./bin/cmake --version + # SDK component and plugin downloads intermittently fail on hosted + # runners; retry once only for those, so real build failures still fail. - name: Build Android debug APK - run: just mobile-build-android + if: matrix.lane == 'android' + run: | + set -o pipefail + log="$RUNNER_TEMP/android-build.log" + if ! just mobile-build-android 2>&1 | tee "$log"; then + grep -qE 'InstallFailedException|Could not (resolve|GET|download)' "$log" + just mobile-build-android + fi + - name: Test Android JVM units + if: matrix.lane == 'android' + run: cd mobile/android && ./gradlew :app:testDebugUnitTest --console=plain + - name: Save Gradle cache + if: matrix.lane == 'android' && github.event_name == 'push' + uses: actions/cache/save@caa296126883cff596d87d8935842f9db880ef25 # v5 + continue-on-error: true + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + key: gradle-${{ runner.os }}-${{ hashFiles('mobile/android/**/*.gradle*', 'mobile/android/gradle/wrapper/gradle-wrapper.properties', 'mobile/pubspec.lock') }} + # The unsigned release build needs nothing from the native lane, so it runs + # beside it instead of after the simulator tests. mobile-swift: - name: Mobile Swift + name: Mobile Swift (${{ matrix.name }}) runs-on: macos-latest - timeout-minutes: 30 + timeout-minutes: 45 if: inputs.lane == 'mobile-swift' && inputs.mobile + strategy: + fail-fast: false + matrix: + include: + - name: Native + lane: native + - name: Release + lane: release steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - uses: cashapp/activate-hermit@cea9af7913204a965fd488637a8d1811bba2e616 # v1 - name: Install Flutter dependencies run: cd mobile && flutter pub get - - name: Build - run: swift build --package-path mobile/ios/BuzzPushKit - - name: Build release - run: swift build -c release --package-path mobile/ios/BuzzPushKit + # `swift test` builds BuzzPushKit; the Release lane covers its release build. - name: Test + if: matrix.lane == 'native' run: swift test --package-path mobile/ios/BuzzPushKit - name: Test native pagination geometry + if: matrix.lane == 'native' run: /bin/bash scripts/test-mobile-pagination.sh - - name: Build complete iOS simulator app - run: cd mobile && flutter build ios --simulator --debug --no-pub - - name: Test native pagination accessibility + - name: Generate iOS build configuration + if: matrix.lane == 'native' + run: cd mobile && flutter build ios --simulator --debug --no-pub --config-only + # Build the complete simulator app and RunnerTests once, in the + # background, while the simulator boots and runs the native-control + # scripts. The build also guards Apple Silicon simulator linking (#7646). + - name: Test native controls and Runner unit tests + if: matrix.lane == 'native' run: | - device=$(xcrun simctl create "Pagination accessibility" "iPhone 16") + device=$(xcrun simctl create "Native tests" "iPhone 16") trap 'xcrun simctl shutdown "$device"; xcrun simctl delete "$device"' EXIT + xcode=(xcodebuild -workspace mobile/ios/Runner.xcworkspace -scheme Runner + -configuration Debug -destination "platform=iOS Simulator,id=$device" + CODE_SIGNING_ALLOWED=NO) + build_log="$RUNNER_TEMP/runner-build.log" + "${xcode[@]}" build-for-testing > "$build_log" 2>&1 & + build=$! xcrun simctl boot "$device" xcrun simctl bootstatus "$device" -b /bin/bash scripts/test-mobile-pagination-accessibility.sh "$device" + /bin/bash scripts/test-mobile-carousel.sh "$device" /bin/bash scripts/test-mobile-member-picker.sh "$device" + /bin/bash scripts/test-mobile-me-agent-accessibility.sh "$device" + if ! wait "$build"; then + cat "$build_log" + exit 1 + fi + tail -n 5 "$build_log" + "${xcode[@]}" -parallel-testing-enabled NO -only-testing:RunnerTests \ + test-without-building - name: Build complete unsigned iOS release + if: matrix.lane == 'release' run: | /bin/bash scripts/test-mobile-gateway-recipes.sh cd mobile && flutter build ios --release --no-codesign --no-pub diff --git a/.github/workflows/_ci-desktop-macos.yml b/.github/workflows/_ci-desktop-macos.yml index d19ffa72ebe..b54a1b3fd6a 100644 --- a/.github/workflows/_ci-desktop-macos.yml +++ b/.github/workflows/_ci-desktop-macos.yml @@ -31,6 +31,8 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - uses: cashapp/activate-hermit@cea9af7913204a965fd488637a8d1811bba2e616 # v1 + - name: Fetch Hermit tools + run: scripts/ci-hermit-retry.sh ./bin/just --version - name: Verify macOS icon representations run: scripts/macos-icon.sh --check - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 diff --git a/.github/workflows/_ci-desktop.yml b/.github/workflows/_ci-desktop.yml index a0b5e7e6283..bcdb3b63d96 100644 --- a/.github/workflows/_ci-desktop.yml +++ b/.github/workflows/_ci-desktop.yml @@ -52,6 +52,8 @@ jobs: run: just desktop-check - name: Desktop unit tests run: just desktop-test + - name: Flake check selection tests + run: node --test scripts/flake-check.test.mjs - name: Desktop build run: just desktop-build - name: Save pnpm store cache @@ -67,7 +69,7 @@ jobs: desktop-tauri: name: Desktop Tauri (${{ matrix.lane }}) runs-on: ubuntu-latest - # A lane needs up to ~15m. ci-apt-retry.sh bounds the apt step at ~34m worst + # A lane needs up to ~15m. ci-apt-retry.sh bounds the apt steps at ~34m worst # case (update ~8.5m + install ~25m); 55m fits both, so a degraded mirror # fails with the wrapper's error instead of the job timeout. timeout-minutes: 55 @@ -91,6 +93,22 @@ jobs: run: just desktop-tauri-test-compiled-flags permissions: contents: read + env: + # Resolved and installed by the steps below; the .deb cache key hashes + # what apt resolves for this list. + TAURI_APT_PACKAGES: >- + build-essential + curl + file + libasound2-dev + libayatana-appindicator3-dev + libgtk-3-dev + librsvg2-dev + libssl-dev + libwebkit2gtk-4.1-dev + libxdo-dev + patchelf + wget steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: @@ -102,24 +120,70 @@ jobs: workspaces: desktop/src-tauri shared-key: ${{ matrix.cache }} save-if: ${{ matrix.save-cache && github.event_name != 'pull_request' }} + # A degraded Ubuntu mirror serves .debs at ~50 KB/s, and the ~62 MB + # Tauri download (one 25.5 MB libwebkit2gtk .deb) then outlasts every + # ci-apt-retry.sh attempt. Cache the .debs in the Actions cache so the + # mirror only serves `apt-get update` and packages Ubuntu has updated + # since main last saved. + - name: Resolve Tauri apt archives + id: tauri-debs + run: | + set -euo pipefail + CI_APT_ATTEMPT_SECONDS=90 scripts/ci-apt-retry.sh sudo apt-get update + read -ra packages <<< "$TAURI_APT_PACKAGES" + # Name, size and SHA256 (from the signed index) of each .deb the + # install will fetch. The mirror URI is left out so the key doesn't + # depend on which mirror answered. Runs before the restore: + # --print-uris omits files that are already in + # /var/cache/apt/archives. + sudo apt-get install --print-uris -qq -y --no-install-recommends \ + -o Acquire::ForceHash=SHA256 "${packages[@]}" | + awk '{print $2, $3, $4}' | sort > "$RUNNER_TEMP/tauri-debs.txt" + . /etc/os-release + prefix="tauri-debs-${ID}${VERSION_ID}-" + echo "prefix=$prefix" >> "$GITHUB_OUTPUT" + echo "key=$prefix$(sha256sum "$RUNNER_TEMP/tauri-debs.txt" | cut -c1-16)" >> "$GITHUB_OUTPUT" + - name: Restore Tauri apt archives + id: tauri-debs-cache + uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5 + with: + path: ~/.cache/tauri-debs + key: ${{ steps.tauri-debs.outputs.key }} + restore-keys: ${{ steps.tauri-debs.outputs.prefix }} - name: Install Tauri dependencies (Linux) + id: tauri-debs-install env: DEBIAN_FRONTEND: noninteractive run: | - CI_APT_ATTEMPT_SECONDS=90 scripts/ci-apt-retry.sh sudo apt-get update - scripts/ci-apt-retry.sh sudo apt-get install -y --no-install-recommends \ - build-essential \ - curl \ - file \ - libasound2-dev \ - libayatana-appindicator3-dev \ - libgtk-3-dev \ - librsvg2-dev \ - libssl-dev \ - libwebkit2gtk-4.1-dev \ - libxdo-dev \ - patchelf \ - wget + set -euo pipefail + debs=~/.cache/tauri-debs + mkdir -p "$debs" + # apt reuses an archived .deb of the expected size without checking + # its hash, so seed only restored files whose SHA256 matches the + # signed index. apt downloads and verifies the rest. sha256sum exits + # non-zero when a file fails or none is restored; neither is fatal. + awk '{sub(/^SHA256:/, "", $3); print $3 " " $1}' "$RUNNER_TEMP/tauri-debs.txt" > "$RUNNER_TEMP/tauri-debs.sha256" + (cd "$debs" && { sha256sum --check --ignore-missing "$RUNNER_TEMP/tauri-debs.sha256" || true; }) | + sed -n 's/: OK$//p' | (cd "$debs" && xargs -r sudo cp -t /var/cache/apt/archives/) + read -ra packages <<< "$TAURI_APT_PACKAGES" + scripts/ci-apt-retry.sh sudo apt-get install -y --no-install-recommends "${packages[@]}" + # Keep exactly the resolved files, so a restore-keys fallback can't + # pile up superseded versions. The packages are installed by now, so + # a failure here only warns and skips the save. + if rm -rf "$debs" && mkdir -p "$debs" && + awk '{print "/var/cache/apt/archives/" $1}' "$RUNNER_TEMP/tauri-debs.txt" | xargs -r cp -t "$debs"; then + echo "rebuilt=true" >> "$GITHUB_OUTPUT" + else + echo "::warning::Could not rebuild the Tauri apt archive cache; not saving it." + fi + # Like the other caches here, only pushes save; PR runs restore. All + # three lanes resolve the same key, so one lane saves it. + - name: Save Tauri apt archives + if: matrix.lane == 'core' && github.event_name == 'push' && steps.tauri-debs-cache.outputs.cache-hit != 'true' && steps.tauri-debs-install.outputs.rebuilt == 'true' + uses: actions/cache/save@caa296126883cff596d87d8935842f9db880ef25 # v5 + with: + path: ~/.cache/tauri-debs + key: ${{ steps.tauri-debs.outputs.key }} - name: Desktop Tauri ${{ matrix.lane }} run: ${{ matrix.run }} env: diff --git a/.github/workflows/_ci-flake.yml b/.github/workflows/_ci-flake.yml new file mode 100644 index 00000000000..ff527ddb7d1 --- /dev/null +++ b/.github/workflows/_ci-flake.yml @@ -0,0 +1,330 @@ +name: CI / Flake Check +on: + workflow_call: + +env: + CARGO_TERM_COLOR: always + BUZZ_TEST_POSTGRES_PASSWORD: buzz_dev + PLAYWRIGHT_BROWSERS_PATH: ${{ github.workspace }}/.cache/ms-playwright + +# Advisory flake check: repeat the tests a pull request changes, ten times +# each with retries off, so a race inside a test surfaces before it lands on +# main. The lanes skip when no test changed, so they stay out of the required +# checks in ci.yml (whose gates reject skipped lanes); their own statuses and +# step summaries carry the result. Selections reach the lanes through env +# vars, never interpolated into run scripts; flake-check.mjs admits only plain +# path and identifier characters. +jobs: + flake-select: + name: Flake check selection + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + outputs: + rust: ${{ steps.select.outputs.rust }} + tauri: ${{ steps.select.outputs.tauri }} + node: ${{ steps.select.outputs.node }} + playwright: ${{ steps.select.outputs.playwright }} + playwright_shards: ${{ steps.select.outputs.playwright_shards }} + playwright_shard_total: ${{ steps.select.outputs.playwright_shard_total }} + dart: ${{ steps.select.outputs.dart }} + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + # The merge commit's first parent is the base branch tip, so this + # diff is exactly the pull request's change without a full clone. + fetch-depth: 2 + - uses: cashapp/activate-hermit@cea9af7913204a965fd488637a8d1811bba2e616 # v1 + # Only a changed spec needs desktop dependencies: Playwright lists its + # smoke journeys and the desktop app's TypeScript parses it. Pull + # requests without one skip the install. + - name: Check for changed Playwright specs + id: specs + run: | + if git diff --quiet --diff-filter=AM HEAD^1 HEAD -- 'desktop/tests/e2e/*.spec.ts'; then + echo "changed=false" >> "$GITHUB_OUTPUT" + else + echo "changed=true" >> "$GITHUB_OUTPUT" + fi + - name: Get pnpm store directory + if: steps.specs.outputs.changed == 'true' + id: pnpm-cache + run: echo "STORE_PATH=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT" + - name: Restore pnpm store cache + if: steps.specs.outputs.changed == 'true' + uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5 + with: + path: ${{ steps.pnpm-cache.outputs.STORE_PATH }} + key: pnpm-${{ runner.os }}-${{ hashFiles('**/pnpm-lock.yaml') }} + restore-keys: pnpm-${{ runner.os }}- + - name: Install desktop dependencies + if: steps.specs.outputs.changed == 'true' + run: just desktop-install-ci + - name: Map changed hunks to tests + id: select + run: node scripts/flake-check.mjs select HEAD^1 HEAD + + flake-rust: + name: Flake check (Rust) + needs: [flake-select] + if: needs.flake-select.outputs.rust != '' + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: cashapp/activate-hermit@cea9af7913204a965fd488637a8d1811bba2e616 # v1 + - uses: rui314/setup-mold@339e96b4ff1f3bd865856ba4cbbe8818a7acb038 # v1 + # Restores the Unit Tests lane's cache (_ci-rust.yml) read-only; this + # lane never saves one of its own. The restore only hits while the cache + # key matches: the same shared-key, and rust-cache's environment hash — + # the CARGO*/RUST*/CC* environment variables, the lockfiles and the rustc + # version — must stay identical to the Unit Tests job's, so any env this + # job sets must be set there too (scripts/test-rust-cache-contract.sh + # checks the key). + - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 + with: + shared-key: unit-tests-sherpa-cache-v1 + save-if: false + - name: Install cargo-nextest + uses: taiki-e/install-action@0fd46367812ee04360509b4169d9f659d6892bb2 # v2.79.15 + with: + tool: cargo-nextest@0.9.136 + # nextest has no repeat flag; the script runs the selection once per + # iteration under the `flake` profile (.config/nextest.toml). + - name: Repeat changed Rust tests + env: + SELECTED: ${{ needs.flake-select.outputs.rust }} + run: node scripts/flake-check.mjs rust --iterations=10 $SELECTED + - name: Rust iteration reports + if: ${{ !cancelled() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: flake-rust + path: test-results/flake + if-no-files-found: ignore + retention-days: 7 + + flake-tauri: + name: Flake check (Desktop Tauri) + needs: [flake-select] + if: needs.flake-select.outputs.tauri != '' + runs-on: ubuntu-latest + # A Desktop Tauri lane's bound (_ci-desktop.yml: ci-apt-retry.sh bounds + # the apt step at ~34m worst case and a cold build needs ~15m), plus a + # second cold build: a lib selection also builds the mesh-llm graph. + timeout-minutes: 70 + permissions: + contents: read + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: cashapp/activate-hermit@cea9af7913204a965fd488637a8d1811bba2e616 # v1 + - uses: rui314/setup-mold@339e96b4ff1f3bd865856ba4cbbe8818a7acb038 # v1 + # Restores the Desktop Tauri core lane's cache (_ci-desktop.yml) + # read-only, under the same constraints as flake-rust's restore above: + # the workspace, shared-key and environment hash must stay identical to + # that job's. + - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 + with: + workspaces: desktop/src-tauri + shared-key: desktop-core + save-if: false + - name: Install Tauri dependencies (Linux) + env: + DEBIAN_FRONTEND: noninteractive + run: | + CI_APT_ATTEMPT_SECONDS=90 scripts/ci-apt-retry.sh sudo apt-get update + scripts/ci-apt-retry.sh sudo apt-get install -y --no-install-recommends \ + build-essential \ + curl \ + file \ + libasound2-dev \ + libayatana-appindicator3-dev \ + libgtk-3-dev \ + librsvg2-dev \ + libssl-dev \ + libwebkit2gtk-4.1-dev \ + libxdo-dev \ + patchelf \ + wget + - name: Install cargo-nextest + uses: taiki-e/install-action@0fd46367812ee04360509b4169d9f659d6892bb2 # v2.79.15 + with: + tool: cargo-nextest@0.9.136 + # tauri-build validates the sidecar binaries at compile time; the Desktop + # Tauri recipes create the same empty stubs. + - name: Create sidecar stubs + run: just _ensure-sidecar-stubs + # Same script as flake-rust, against the Tauri workspace's manifest and + # its own .config/nextest.toml `flake` profile. A selected lib test also + # repeats under mesh-llm, as desktop-tauri-test-mesh runs it; the + # restored desktop-core cache does not hold that dependency graph, so a + # lib selection builds it cold, and an integration-test one never does. + - name: Repeat changed desktop Tauri tests + env: + CMAKE_POLICY_VERSION_MINIMUM: "3.5" + SELECTED: ${{ needs.flake-select.outputs.tauri }} + run: node scripts/flake-check.mjs tauri --iterations=10 $SELECTED + - name: Desktop Tauri iteration reports + if: ${{ !cancelled() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: flake-tauri + path: test-results/flake + if-no-files-found: ignore + retention-days: 7 + + flake-node: + name: Flake check (node:test) + needs: [flake-select] + if: needs.flake-select.outputs.node != '' + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: cashapp/activate-hermit@cea9af7913204a965fd488637a8d1811bba2e616 # v1 + - name: Get pnpm store directory + id: pnpm-cache + run: echo "STORE_PATH=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT" + - name: Restore pnpm store cache + uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5 + with: + path: ${{ steps.pnpm-cache.outputs.STORE_PATH }} + key: pnpm-${{ runner.os }}-${{ hashFiles('**/pnpm-lock.yaml') }} + restore-keys: pnpm-${{ runner.os }}- + - name: Install desktop dependencies + run: just desktop-install-ci + - name: Repeat changed node:test files + env: + SELECTED: ${{ needs.flake-select.outputs.node }} + run: node scripts/flake-check.mjs node --iterations=10 $SELECTED + - name: node:test iteration logs + if: ${{ !cancelled() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: flake-node + path: test-results/flake + if-no-files-found: ignore + retention-days: 7 + + flake-playwright: + name: Flake check (Playwright smoke, ${{ matrix.shard }}/${{ needs.flake-select.outputs.playwright_shard_total }}) + needs: [flake-select] + if: needs.flake-select.outputs.playwright != '' + runs-on: ubuntu-latest + # Same bound as a smoke shard: Chromium dependency setup is bounded by + # ci-apt-retry.sh (~25m worst case), and flake-check.mjs sizes the shards + # so the repetitions fit the rest. + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + shard: ${{ fromJSON(needs.flake-select.outputs.playwright_shards) }} + permissions: + contents: read + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: cashapp/activate-hermit@cea9af7913204a965fd488637a8d1811bba2e616 # v1 + - name: Get pnpm store directory + id: pnpm-cache + run: echo "STORE_PATH=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT" + - name: Restore pnpm store cache + uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5 + with: + path: ${{ steps.pnpm-cache.outputs.STORE_PATH }} + key: pnpm-${{ runner.os }}-${{ hashFiles('**/pnpm-lock.yaml') }} + restore-keys: pnpm-${{ runner.os }}- + - name: Install desktop dependencies + run: just desktop-install-ci + - name: Get Playwright version + id: pw-version + run: echo "version=$(cd desktop && node -e "console.log(require('@playwright/test/package.json').version)")" >> "$GITHUB_OUTPUT" + - name: Restore Playwright browser cache + id: playwright-cache + uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5 + with: + path: ${{ env.PLAYWRIGHT_BROWSERS_PATH }} + key: playwright-${{ runner.os }}-${{ steps.pw-version.outputs.version }} + - name: Install Playwright Chromium + if: steps.playwright-cache.outputs.cache-hit != 'true' + run: cd desktop && pnpm exec playwright install chromium + - name: Install Playwright system dependencies + run: cd desktop && ../scripts/ci-apt-retry.sh pnpm exec playwright install-deps chromium + - name: Desktop E2E build + run: pnpm -C desktop build:e2e + # --retries=0 overrides the config's CI retries, so every failed + # repetition reports as unexpected and fails the step. + - name: Repeat changed smoke journeys + env: + PLAYWRIGHT_JSON_OUTPUT_FILE: playwright-report.json + SELECTED: ${{ needs.flake-select.outputs.playwright }} + SHARD: ${{ matrix.shard }} + SHARD_TOTAL: ${{ needs.flake-select.outputs.playwright_shard_total }} + run: cd desktop && pnpm exec playwright test --project=smoke --repeat-each=10 --retries=0 --shard="$SHARD/$SHARD_TOTAL" --reporter=list,json $SELECTED + - name: Summarize repetitions + if: ${{ !cancelled() }} + env: + SHARD: ${{ matrix.shard }} + SHARD_TOTAL: ${{ needs.flake-select.outputs.playwright_shard_total }} + run: node scripts/flake-check.mjs playwright-summary desktop/playwright-report.json "Flake check (Playwright smoke, $SHARD/$SHARD_TOTAL)" + - name: Playwright repetition artifacts + if: ${{ !cancelled() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: flake-playwright-${{ matrix.shard }} + path: | + desktop/playwright-report.json + desktop/test-results + if-no-files-found: ignore + retention-days: 7 + + flake-dart: + name: Flake check (Flutter) + needs: [flake-select] + if: needs.flake-select.outputs.dart != '' + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: cashapp/activate-hermit@cea9af7913204a965fd488637a8d1811bba2e616 # v1 + # Restores the Mobile lane's Hermit and pub caches read-only; that lane + # saves them. + - name: Compute Hermit cache key + id: hermit-bin-hash + run: | + hash="$(find ./bin ! -type d | sort | xargs openssl sha256 | openssl sha256 -r | cut -d' ' -f1)" + echo "hash=$hash" >> "$GITHUB_OUTPUT" + - name: Restore Hermit package cache + uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5 + with: + path: ~/.cache/hermit/pkg + key: ${{ runner.os }}-hermit-cache-${{ steps.hermit-bin-hash.outputs.hash }} + restore-keys: ${{ runner.os }}-hermit-cache- + - name: Prime Flutter SDK + run: flutter --version + - name: Restore pub cache + uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5 + with: + path: ~/.pub-cache + key: pub-${{ runner.os }}-${{ hashFiles('mobile/pubspec.lock') }} + restore-keys: pub-${{ runner.os }}- + - name: Install dependencies + run: cd mobile && flutter pub get + - name: Repeat changed Flutter tests + env: + SELECTED: ${{ needs.flake-select.outputs.dart }} + run: node scripts/flake-check.mjs dart --iterations=10 $SELECTED + - name: Flutter iteration reports + if: ${{ !cancelled() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: flake-dart + path: test-results/flake + if-no-files-found: ignore + retention-days: 7 diff --git a/.github/workflows/_ci-relay.yml b/.github/workflows/_ci-relay.yml index b6cab48aaf1..bf57fb424ba 100644 --- a/.github/workflows/_ci-relay.yml +++ b/.github/workflows/_ci-relay.yml @@ -88,6 +88,21 @@ jobs: uses: Mozilla-Actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11 # zizmor: ignore[cache-poisoning] Bounded trial: only PR 5224 writes to its isolated merge-ref scope; trusted pushes retain production writes. with: version: v0.16.0 + # The sccache server checks its GHA cache storage at startup and refuses to + # start when that fails (egress over the account limit, a failed read), + # which used to fail cargo's first `sccache rustc -vV` and the build. + # SCCACHE_IGNORE_SERVER_IO_ERROR doesn't cover startup in v0.16.0, so start + # it here and build with plain rustc if it won't start. + - name: Start sccache + id: sccache + if: steps.relay-artifacts-cache.outputs.cache-hit != 'true' + run: | + if sccache --start-server; then + echo "wrapper=sccache" >> "$GITHUB_OUTPUT" + else + echo "::warning::sccache did not start; building without it" + echo "wrapper=" >> "$GITHUB_OUTPUT" + fi - name: Install cargo-nextest if: steps.relay-artifacts-cache.outputs.cache-hit != 'true' && matrix.artifact != 'relay-binaries' uses: taiki-e/install-action@0fd46367812ee04360509b4169d9f659d6892bb2 # v2.79.15 @@ -96,12 +111,12 @@ jobs: - name: Build relay binaries if: steps.relay-artifacts-cache.outputs.cache-hit != 'true' && matrix.artifact == 'relay-binaries' env: - RUSTC_WRAPPER: sccache + RUSTC_WRAPPER: ${{ steps.sccache.outputs.wrapper }} run: cargo build --profile ci -p buzz-relay -p git-credential-nostr - name: Build backend integration test archive if: steps.relay-artifacts-cache.outputs.cache-hit != 'true' && matrix.artifact == 'backend-integration-tests' env: - RUSTC_WRAPPER: sccache + RUSTC_WRAPPER: ${{ steps.sccache.outputs.wrapper }} run: | cargo nextest archive \ --cargo-profile ci \ @@ -110,11 +125,13 @@ jobs: -p buzz-test-client \ --lib \ --test e2e_event_reminder \ + --test e2e_author_only_deletion \ + --test e2e_existence_oracle \ --archive-file target/ci/backend-integration-tests.tar.zst - name: Build PostgreSQL test archive if: steps.relay-artifacts-cache.outputs.cache-hit != 'true' && matrix.artifact == 'postgres-tests' env: - RUSTC_WRAPPER: sccache + RUSTC_WRAPPER: ${{ steps.sccache.outputs.wrapper }} run: | postgres_package_args=() while IFS= read -r package; do @@ -189,6 +206,9 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - uses: cashapp/activate-hermit@cea9af7913204a965fd488637a8d1811bba2e616 # v1 + # pgschema has no --version; --help exits 0 once Hermit has fetched it. + - name: Fetch Hermit tools + run: scripts/ci-hermit-retry.sh ./bin/pgschema --help - name: Tune disposable PostgreSQL env: PGPASSWORD: ${{ env.BUZZ_TEST_POSTGRES_PASSWORD }} @@ -241,6 +261,9 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - uses: cashapp/activate-hermit@cea9af7913204a965fd488637a8d1811bba2e616 # v1 + # pgschema has no --version; --help exits 0 once Hermit has fetched it. + - name: Fetch Hermit tools + run: scripts/ci-hermit-retry.sh ./bin/pgschema --help - name: Start integration services run: | for attempt in 1 2 3; do @@ -431,6 +454,9 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - uses: cashapp/activate-hermit@cea9af7913204a965fd488637a8d1811bba2e616 # v1 + # pgschema has no --version; --help exits 0 once Hermit has fetched it. + - name: Fetch Hermit tools + run: scripts/ci-hermit-retry.sh ./bin/pgschema --help - name: Install cargo-nextest uses: taiki-e/install-action@0fd46367812ee04360509b4169d9f659d6892bb2 # v2.79.15 with: @@ -520,6 +546,18 @@ jobs: --archive-file target/ci/backend-integration-tests.tar.zst \ -E 'package(buzz-relay) and test(/api::git::settings::tests::external_infra::/)' \ --run-ignored ignored-only + - name: Admin attachment access-log regression + env: + # This test builds its state from Config::for_test(), which reads the + # relay's own DATABASE_URL and BUZZ_S3_* variables. + DATABASE_URL: postgres://buzz:${{ env.BUZZ_TEST_POSTGRES_PASSWORD }}@localhost:5432/buzz + BUZZ_S3_ENDPOINT: http://localhost:9000 + BUZZ_S3_BUCKET: buzz-media + run: | + cargo nextest run \ + --archive-file target/ci/backend-integration-tests.tar.zst \ + -E 'package(buzz-relay) and test(/api::admin::postgres_tests::external_infra_minio_tests::/)' \ + --run-ignored ignored-only - name: Relay quota accounting and rejection regressions env: BUZZ_TEST_REDIS_URL: redis://localhost:6379 @@ -580,10 +618,12 @@ jobs: run: | cargo nextest run \ --archive-file target/ci/backend-integration-tests.tar.zst \ - -E 'binary(e2e_event_reminder)' \ + -E 'binary(e2e_event_reminder) | binary(e2e_author_only_deletion) | binary(e2e_existence_oracle)' \ --run-ignored ignored-only env: RELAY_URL: ws://localhost:3000 + # schema/schema.sql (applied above) indexes kind:5 content. + BUZZ_TEST_SEARCH_INDEXES_DELETIONS: "1" - name: Upload relay log if: failure() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 @@ -623,6 +663,13 @@ jobs: cargo test -p buzz-test-client --test e2e_relay invite -- --ignored --nocapture cargo test -p buzz-test-client --test e2e_relay nip43_membership_snapshots_are_rejected -- --ignored --nocapture cargo test -p buzz-test-client --test e2e_relay nip29_departure_wire -- --ignored --nocapture + cargo test -p buzz-test-client --test e2e_relay nip29_batch_put_user_wire -- --ignored --nocapture + cargo test -p buzz-test-client --test e2e_relay nip29_announce_channel_wire -- --ignored --nocapture + cargo test -p buzz-test-client --test e2e_relay nip29_unlisted_channel_wire -- --ignored --nocapture + cargo test -p buzz-test-client --test e2e_relay nip29_solo_share_and_convert_wire -- --ignored --nocapture + cargo test -p buzz-test-client --test e2e_relay nip29_member_raises_guest_agent -- --ignored --nocapture + cargo test -p buzz-test-client --test e2e_relay nip29_guest_is_read_only_in_normal_channel -- --ignored --nocapture + cargo test -p buzz-test-client --test e2e_relay nip29_relay_rejects_role_change_by_unprivileged_actor -- --ignored --nocapture env: RELAY_URL: ws://localhost:3000 GIT_CREDENTIAL_NOSTR_BIN: ${{ github.workspace }}/target/ci/git-credential-nostr diff --git a/.github/workflows/_ci-rust.yml b/.github/workflows/_ci-rust.yml index e25639d3ac7..e536e131620 100644 --- a/.github/workflows/_ci-rust.yml +++ b/.github/workflows/_ci-rust.yml @@ -58,7 +58,10 @@ jobs: - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 with: - key: sherpa-cache-v1 + # Named so the advisory flake lane (_ci-flake.yml) can restore it; + # with shared-key set, rust-cache ignores `key`, so the cache-bust + # suffix lives here. Bump it to discard every saved cache. + shared-key: unit-tests-sherpa-cache-v1 save-if: ${{ github.event_name != 'pull_request' }} - name: Install cargo-nextest uses: taiki-e/install-action@0fd46367812ee04360509b4169d9f659d6892bb2 # v2.79.15 @@ -108,9 +111,12 @@ jobs: windows-rust: name: Windows Rust (x86_64-pc-windows-msvc) runs-on: windows-latest - # Windows runners are slow and this compiles the workspace + Tauri crate - # cold across four steps; budget generously. - timeout-minutes: 45 + # With no rust-cache (after a toolchain bump or eviction), the steps take + # 39-41m on main (2026-10-22..26): clippy 17-18m, Tauri check ~6m, Tauri + # tests 10-11m, the rest ~5m. The cache save that follows takes ~4.5m and + # runs inside this cap. If the cap kills it, nothing is saved and the next + # push is cold again. 60m leaves ~15m over that; warm runs take 15-23m. + timeout-minutes: 60 if: inputs.lane == 'required' && (github.event_name == 'push' || inputs.rust || inputs.desktop_rust) permissions: contents: read @@ -120,7 +126,7 @@ jobs: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 # MSVC needs windows.h (aws-lc-sys et al.), so this runs on a real Windows # runner — hermit, used by the Linux jobs, does not provide MSVC. The - # toolchain (1.95.0 + clippy via profile = default) comes from the + # toolchain (1.98.1 + clippy via profile = default) comes from the # repo-root rust-toolchain.toml, which the runner's preinstalled rustup # honors on demand; the host triple already is x86_64-pc-windows-msvc. - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 @@ -141,10 +147,10 @@ jobs: for bin in buzz-acp buzz-agent buzz-dev-mcp git-credential-nostr buzz; do touch "desktop/src-tauri/binaries/${bin}-${TARGET}.exe" done + # Clippy type-checks everything `cargo check` would, so no separate + # workspace check step. - name: Clippy (workspace) run: cargo clippy --workspace --all-targets --target $env:TARGET -- -D warnings - - name: Check (workspace) - run: cargo check --workspace --all-targets --target $env:TARGET - name: Test (buzz-dev-mcp) # The Windows-only bash resolver lives in buzz-dev-mcp; its unit tests # only gate if this crate is tested ON Windows. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 935bc6b8e21..2cafab1bb37 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,9 @@ jobs: changes: name: Detect Changed Paths runs-on: ubuntu-latest - timeout-minutes: 2 + # The steps take about 75 seconds, but GitHub can finish the job minutes + # after the last step. A 2-minute limit cancelled finished runs. + timeout-minutes: 10 permissions: contents: read pull-requests: read @@ -32,6 +34,12 @@ jobs: with: fetch-depth: 2 - uses: cashapp/activate-hermit@cea9af7913204a965fd488637a8d1811bba2e616 # v1 + # The first call of each Hermit tool downloads it. A stalled Node download + # hung the selection step until this job's cap cancelled every gate. One + # invocation fetches both tools, so its worst case (under 6 minutes) + # still leaves the rest of the job inside the 10-minute cap. + - name: Fetch Hermit tools + run: scripts/ci-hermit-retry.sh sh -c './bin/node --version && ./bin/just --version' - uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4.0.3 id: filter with: @@ -51,6 +59,8 @@ jobs: - '.config/nextest.toml' - 'scripts/postgres-test-*.sh' - 'scripts/reconcile-schema-after-pgschema.sql' + - 'scripts/repair-events-pkey.sql' + - 'scripts/start-relay-for-tests.sh' - 'bin/pgschema' - 'bin/.pgschema-*.pkg' - 'scripts/check-postgres-test-discovery.py' @@ -68,6 +78,7 @@ jobs: - 'scripts/model-capabilities.json' - 'scripts/normative-corpus.json' - 'scripts/ci-apt-retry.sh' + - 'scripts/ci-hermit-retry.sh' - 'Justfile' desktop: - '.github/workflows/ci.yml' @@ -82,6 +93,9 @@ jobs: - 'scripts/normative-corpus.json' - 'desktop/**' - 'pnpm-lock.yaml' + - 'scripts/flake-check.mjs' + - 'scripts/flake-check.test.mjs' + - '.github/workflows/_ci-flake.yml' desktop-rust: - 'desktop/src-tauri/**' web: @@ -89,7 +103,10 @@ jobs: - 'pnpm-lock.yaml' mobile: - 'mobile/**' + - 'Justfile' + - 'scripts/test-mobile-gateway-recipes.sh' - 'scripts/mobile-release.sh' + - 'scripts/mobile-release-source.sh' - 'scripts/mobile-worktree-overrides.sh' - 'scripts/mobile-worktree-clean.sh' - 'scripts/publish-mobile-release-candidate.sh' @@ -97,6 +114,7 @@ jobs: - 'scripts/test-mobile-release-contract.sh' - 'scripts/test-mobile-release-candidate-publisher.sh' - 'scripts/test-mobile-worktree-overrides.sh' + - 'scripts/ci-hermit-retry.sh' - '.github/workflows/mobile-release-candidate.yml' - '.github/workflows/ci.yml' - '.github/workflows/_ci-*.yml' @@ -137,7 +155,7 @@ jobs: - name: CI required-context isolation contract run: scripts/test-ci-required-context-isolation.sh - name: CI path selection regression tests - run: node --test scripts/ci-selection.test.mjs scripts/ci-required-gates.test.mjs scripts/ci-apt-retry.test.mjs + run: node --test scripts/ci-selection.test.mjs scripts/ci-required-gates.test.mjs scripts/ci-apt-retry.test.mjs scripts/ci-hermit-retry.test.mjs - name: File size policy run: just file-size-check @@ -255,6 +273,14 @@ jobs: mobile: ${{ needs.changes.outputs.mobile == 'true' }} lane: mobile-swift + # Advisory flake check: repeats the tests a pull request changes. Its lanes + # skip when no test changed and none of the required gates below reads them; + # their own statuses and step summaries carry the result. + flake-check: + name: Flake Check + if: github.event_name == 'pull_request' + uses: ./.github/workflows/_ci-flake.yml + security-domain: name: Security Domain needs: [changes] diff --git a/.github/workflows/codex-security-review.yml b/.github/workflows/codex-security-review.yml index eb6d01bf6dd..b7966d2b1b0 100644 --- a/.github/workflows/codex-security-review.yml +++ b/.github/workflows/codex-security-review.yml @@ -220,7 +220,7 @@ jobs: permissions: contents: read env: - CODEX_MODEL: gpt-5.6-sol + CODEX_MODEL: gpt-6.1-sol CODEX_REASONING_EFFORT: high CODEX_REVIEW_API_KEY_PRESENT: ${{ secrets.CODEX_REVIEW_API_KEY != '' }} REVIEW_CONTEXT: review-context @@ -303,7 +303,7 @@ jobs: ACTIONS_ID_TOKEN_REQUEST_URL: '' with: openai-api-key: ${{ secrets.CODEX_REVIEW_API_KEY }} - codex-version: '0.150.1' + codex-version: '0.161.0' model: ${{ env.CODEX_MODEL }} codex-args: '["-c","model_reasoning_effort=${{ env.CODEX_REASONING_EFFORT }}"]' # The trusted authorization job already required repo write access for the @@ -508,7 +508,7 @@ jobs: contents: read pull-requests: write env: - CODEX_MODEL: gpt-5.6-sol + CODEX_MODEL: gpt-6.1-sol REVIEW_JSON: ${{ needs.security-review.outputs.review_json }} REVIEW_PR_NUMBER: ${{ needs.prepare-review.outputs.pr_number }} REVIEW_TRIGGER_ACTOR: ${{ needs.prepare-review.outputs.trigger_actor }} diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index f3e8955dc3c..1c6eebb85ad 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -17,8 +17,8 @@ name: Docker image # the relay image version tracks crates/buzz-relay/Cargo.toml, never desktop. # # Triggers: -# - push to main → :main + :sha-<7> -# + :debug-main + :debug-sha-<7> +# - push to main → :main + :sha-<7> + :main--<7> +# + the matching :debug-* tags # - push tags relay-v*.*.* → :{version} + :{major}.{minor} + :{major} # + matching :debug-* tags # (+ :latest/:debug-latest for stable releases) @@ -35,6 +35,16 @@ name: Docker image # intact) and the bare version flows straight to the semver parser. On a # real push event value= is empty and the match strips relay-v from the ref. # +# ghcr.io/block/buzz-push-gateway carries the same :main, :sha-<7> and +# :main--<7> tags on main pushes, and publishes only after the +# relay's `qualify` job accepts the same source SHA. +# +# :main--<7> names the same main build as :sha-<7>, prefixed +# with its UTC commit time so the tag sorts by build order. Kargo's Lexical +# strategy picks the newest from the tag list alone; :sha-<7> cannot be +# ordered without fetching every image's metadata. It uses the commit date, +# not the run date, so a rerun of the same commit re-pushes the same tag. +# # The :latest tag tracks the latest STABLE relay release: metadata-action's # `flavor.latest=auto` (its default) emits :latest only for non-prerelease # semver, so relay-v0.3.0-rc.1 publishes :0.3.0-rc.1 without moving :latest, @@ -169,6 +179,7 @@ jobs: type=semver,pattern={{version}},match=^relay-v(.*)$,value=${{ inputs.version }} type=semver,pattern={{major}}.{{minor}},match=^relay-v(.*)$,value=${{ inputs.version }} type=semver,pattern={{major}},match=^relay-v(.*)$,value=${{ inputs.version }} + type=raw,value=main-{{commit_date 'YYYYMMDDHHmmss'}}-{{sha}},enable=${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} labels: | org.opencontainers.image.title=Buzz org.opencontainers.image.description=WebSocket relay server for the Buzz communications platform @@ -383,6 +394,7 @@ jobs: type=semver,pattern={{version}},match=^relay-v(.*)$,value=${{ inputs.version }} type=semver,pattern={{major}}.{{minor}},match=^relay-v(.*)$,value=${{ inputs.version }} type=semver,pattern={{major}},match=^relay-v(.*)$,value=${{ inputs.version }} + type=raw,value=main-{{commit_date 'YYYYMMDDHHmmss'}}-{{sha}},enable=${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} - name: Create and push manifest list id: manifest @@ -623,7 +635,9 @@ jobs: name: Publish public push gateway image if: github.event_name != 'pull_request' runs-on: ubuntu-24.04 - needs: push-gateway-build + # Staging promotes every published main gateway image automatically, so the + # gateway publishes only from a source the relay's same-SHA CI qualified. + needs: [push-gateway-build, qualify] timeout-minutes: 15 permissions: contents: read @@ -660,6 +674,7 @@ jobs: type=sha,prefix=sha-,format=short,enable=${{ github.event_name != 'workflow_dispatch' || inputs.version == '' }} type=semver,pattern={{version}},match=^relay-v(.*)$,value=${{ inputs.version }} type=semver,pattern={{major}}.{{minor}},match=^relay-v(.*)$,value=${{ inputs.version }} + type=raw,value=main-{{commit_date 'YYYYMMDDHHmmss'}}-{{sha}},enable=${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} - name: Merge and publish manifest id: manifest working-directory: /tmp/gateway-digests diff --git a/.github/workflows/loadgen-image.yml b/.github/workflows/loadgen-image.yml new file mode 100644 index 00000000000..35d385fdfc4 --- /dev/null +++ b/.github/workflows/loadgen-image.yml @@ -0,0 +1,236 @@ +name: Loadgen image + +# Builds and publishes the relay load generator image as +# ghcr.io//-loadgen (ghcr.io/block/buzz-loadgen upstream; see +# Dockerfile.loadgen and perf/LOADGEN.md). +# +# Strategy mirrors sprig-image.yml: each architecture builds on its native +# runner, pushes to GHCR by digest, then a merge job stitches the per-arch +# digests into one multi-arch manifest and attests provenance. +# +# Triggers: +# - push to main (paths-filtered) → :main + :sha-<7> +# - pull_request (paths-filtered) → build only, no image push (same-repo +# PRs still write the registry build cache, as in sprig-image.yml) +# - workflow_dispatch → manual publish at the current ref +# +# NOTE: the first push creates the GHCR package PRIVATE by default. An org +# admin must flip the loadgen package to public once (Package settings → +# Change visibility). Subsequent pushes keep the visibility. + +on: + push: + branches: [main] + paths: + - "Dockerfile.loadgen" + - ".github/workflows/loadgen-image.yml" + - ".cargo/config.toml" + - "Cargo.toml" + - "Cargo.lock" + - "rust-toolchain.toml" + - "crates/buzz-test-client/**" + - "crates/buzz-ws-client/**" + - "crates/buzz-core/**" + pull_request: + paths: + - "Dockerfile.loadgen" + - ".github/workflows/loadgen-image.yml" + workflow_dispatch: {} + +concurrency: + group: loadgen-image-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: {} + +env: + # Defaults to the running repository's namespace so forks publish to their + # own registry (ghcr.io/block/buzz-loadgen upstream). Override with the + # GHCR_LOADGEN_IMAGE repo variable (same pattern as docker.yml). + IMAGE_NAME: ${{ vars.GHCR_LOADGEN_IMAGE != '' && vars.GHCR_LOADGEN_IMAGE || format('ghcr.io/{0}-loadgen', github.repository) }} + +jobs: + build: + name: Build (${{ matrix.platform }}) + runs-on: ${{ matrix.runner }} + timeout-minutes: 60 + permissions: + contents: read + packages: write + id-token: write + attestations: write + strategy: + fail-fast: false + matrix: + include: + - platform: linux/amd64 + runner: ubuntu-24.04 + arch: amd64 + - platform: linux/arm64 + runner: ubuntu-24.04-arm + arch: arm64 + + steps: + - name: Normalize image name + shell: bash + run: | + printf 'IMAGE_NAME=%s\n' "${IMAGE_NAME,,}" >> "$GITHUB_ENV" + + - name: Checkout + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0 + with: + # Same OOM cap as docker.yml and sprig-image.yml. + buildkitd-config-inline: | + [worker.oci] + max-parallelism = 2 + + - name: Log in to GHCR + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata + id: meta + uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9 # v6.1.0 + with: + images: ${{ env.IMAGE_NAME }} + flavor: | + latest=false + tags: | + type=ref,event=branch + type=sha,prefix=sha-,format=short + labels: | + org.opencontainers.image.title=Buzz Loadgen + org.opencontainers.image.description=Scenario-driven Buzz relay load generator + org.opencontainers.image.licenses=Apache-2.0 + + - name: Build and push by digest + id: build + uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0 + with: + context: . + file: ./Dockerfile.loadgen + platforms: ${{ matrix.platform }} + labels: ${{ steps.meta.outputs.labels }} + outputs: type=image,name=${{ env.IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=${{ github.event_name != 'pull_request' }} + cache-from: | + type=registry,ref=${{ env.IMAGE_NAME }}-buildcache:${{ matrix.arch }} + cache-to: | + ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && format('type=registry,ref={0}-buildcache:{1},mode=max,compression=zstd', env.IMAGE_NAME, matrix.arch) || '' }} + + - name: Export digest + if: github.event_name != 'pull_request' + env: + DIGEST: ${{ steps.build.outputs.digest }} + run: | + mkdir -p /tmp/digests + touch "/tmp/digests/${DIGEST#sha256:}" + + - name: Upload digest + if: github.event_name != 'pull_request' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: loadgen-digest-${{ matrix.arch }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 + + merge: + name: Merge multi-arch manifest + if: github.event_name != 'pull_request' + runs-on: ubuntu-24.04 + needs: build + timeout-minutes: 15 + permissions: + contents: read + packages: write + id-token: write + attestations: write + + steps: + - name: Normalize image name + shell: bash + run: | + printf 'IMAGE_NAME=%s\n' "${IMAGE_NAME,,}" >> "$GITHUB_ENV" + + - name: Download per-arch digests + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + path: /tmp/digests + pattern: loadgen-digest-* + merge-multiple: true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0 + + - name: Log in to GHCR + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata + id: meta + uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9 # v6.1.0 + with: + images: ${{ env.IMAGE_NAME }} + # Must mirror the build job's tag matrix exactly. + flavor: | + latest=false + tags: | + type=ref,event=branch + type=sha,prefix=sha-,format=short + + - name: Create and push manifest list + id: manifest + working-directory: /tmp/digests + env: + IMAGE_NAME: ${{ env.IMAGE_NAME }} + META_TAGS: ${{ steps.meta.outputs.tags }} + run: | + set -euo pipefail + tags=() + while IFS= read -r tag; do + [ -n "$tag" ] && tags+=("-t" "$tag") + done <<< "$META_TAGS" + + digests=() + for digest in *; do + digests+=("${IMAGE_NAME}@sha256:${digest}") + done + + docker buildx imagetools create "${tags[@]}" "${digests[@]}" + + first_tag=$(echo "$META_TAGS" | head -n1) + merged_digest=$(docker buildx imagetools inspect "$first_tag" \ + --format '{{json .Manifest}}' | jq -r '.digest') + echo "digest=${merged_digest}" >> "$GITHUB_OUTPUT" + + - name: Attest provenance for the merged image + # Verify with: gh attestation verify oci://ghcr.io/block/buzz-loadgen: --owner block + uses: actions/attest-build-provenance@0f67c3f4856b2e3261c31976d6725780e5e4c373 # v4.1.1 + with: + subject-name: ${{ env.IMAGE_NAME }} + subject-digest: ${{ steps.manifest.outputs.digest }} + push-to-registry: true + + - name: Summary + env: + IMAGE_NAME: ${{ env.IMAGE_NAME }} + DIGEST: ${{ steps.manifest.outputs.digest }} + run: | + { + echo "### Loadgen image published" + echo '```' + echo "${IMAGE_NAME}@${DIGEST}" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/mobile-release-candidate.yml b/.github/workflows/mobile-release-candidate.yml index e1f1c1c9184..131e8aab0b5 100644 --- a/.github/workflows/mobile-release-candidate.yml +++ b/.github/workflows/mobile-release-candidate.yml @@ -12,8 +12,13 @@ on: description: Expected next release-candidate number required: true type: string + source_branch: + description: Source branch (main or mobile-vMAJOR.MINOR matching version) + required: true + default: main + type: string target_sha: - description: Exact current block/buzz main commit + description: Pinned commit in the selected source branch history required: true type: string @@ -65,5 +70,6 @@ jobs: GH_TOKEN: ${{ steps.release-tagger.outputs.token }} MOBILE_VERSION: ${{ inputs.version }} CANDIDATE_NUMBER: ${{ inputs.candidate_number }} + SOURCE_BRANCH: ${{ inputs.source_branch }} TARGET_SHA: ${{ inputs.target_sha }} - run: scripts/publish-mobile-release-candidate.sh "$MOBILE_VERSION" "$CANDIDATE_NUMBER" "$TARGET_SHA" + run: scripts/publish-mobile-release-candidate.sh "$MOBILE_VERSION" "$CANDIDATE_NUMBER" "$TARGET_SHA" "$SOURCE_BRANCH" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 040817361a5..26fbbf5401a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -676,7 +676,7 @@ jobs: shell: bash run: scripts/verify-release-ref.sh desktop-v "$VERSION" - - uses: dtolnay/rust-toolchain@e081816240890017053eacbb1bdf337761dc5582 # 1.95.0 + - uses: dtolnay/rust-toolchain@18f0487477426dce244608110aa2660c2893deb7 # 1.98.1 with: targets: ${{ env.TARGET }} diff --git a/.github/workflows/windows-canary.yml b/.github/workflows/windows-canary.yml index 7093efd2dc6..03114c1a47a 100644 --- a/.github/workflows/windows-canary.yml +++ b/.github/workflows/windows-canary.yml @@ -39,7 +39,7 @@ jobs: persist-credentials: false # The Windows runner ships with rustup preinstalled; rust-toolchain.toml - # in the repo root pins the channel (1.95.0) automatically. We only need + # in the repo root pins the channel (1.98.1) automatically. We only need # to ensure the cross-compile target is registered; on windows-latest the # host IS x86_64-pc-windows-msvc so this is typically a no-op. - name: Add Rust target diff --git a/.intersect/sadscan.yaml b/.intersect/sadscan.yaml index 77710ae0d98..7f6c5e57a2c 100644 --- a/.intersect/sadscan.yaml +++ b/.intersect/sadscan.yaml @@ -4,11 +4,15 @@ exclude_rules_for_files: - Cargo.lock # Self-signed test fixture generated solely to exercise reqwest identity parsing. kingfisher.privkey.2: + # Published upstream synthetic CA key, compiled only under cfg(test). + # The gateway pins Apple's root and cannot trust this fixture CA. + - "*appattest-test-ca-key.pem" - "*apns-test-identity.pem" - "*apns-test-key-only.pem" - "*apns-test-encrypted-identity.pem" - "*apns-test-mismatched-identity.pem" np.pem.1: + - "*appattest-test-ca-key.pem" - "*apns-test-identity.pem" - "*apns-test-key-only.pem" - "*apns-test-encrypted-identity.pem" diff --git a/.pgschemaignore b/.pgschemaignore new file mode 100644 index 00000000000..059350e198c --- /dev/null +++ b/.pgschemaignore @@ -0,0 +1,25 @@ +# pgschema reads this file from its working directory; every caller runs it +# from the repository root. +# +# The relay's partition manager creates dated `events` / `delivery_log` +# monthlies at runtime (see crates/buzz-db/src/store/partition/). They are not +# declared in schema/partitions/, so without this file re-applying pgschema to a +# database a relay has used drops them, and their rows, with +# `DROP TABLE IF EXISTS ... CASCADE`. Only the monthlies schema/partitions/ +# declares stay managed by pgschema; keep the `!` exemptions below in sync with +# them. +[tables] +patterns = [ + "events_p2*", + "!events_p2026_01", + "!events_p2026_02", + "!events_p2026_03", + "!events_p2026_04", + "!events_p2026_05", + "!events_p2026_06", + "delivery_log_p2*", + "!delivery_log_p2026_03", + "!delivery_log_p2026_04", + "!delivery_log_p2026_05", + "!delivery_log_p2026_06", +] diff --git a/AGENTS.md b/AGENTS.md index 5932bcd9e11..ec7e7f87cf8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -14,7 +14,12 @@ Before planning or reviewing a non-trivial change: 2. Read the `VISION_*.md` documents relevant to the affected product surface. 3. Read the applicable guidance in [TESTING.md](TESTING.md) and any package-local `TESTING.md`. -4. Check that the proposed design advances, or at least does not contradict, +4. For any change to `migrations/`, `schema/`, or schema-dependent `buzz-db` + queries, read [docs/database.md](docs/database.md): no new triggers, + functions, or foreign keys; no data backfills in migrations; update every + schema location; and keep every migration compatible with the relay that + is currently deployed. +5. Check that the proposed design advances, or at least does not contradict, that product intent. Call out any intentional tension explicitly. Implementation describes the product today; the vision documents describe the @@ -125,10 +130,11 @@ crates/ buzz-test-client # Integration test client and E2E test suite sprig # All-in-one harness bundling ACP, agent, and dev MCP -desktop/ # Tauri 2 + React 19 desktop app +desktop/ # Legacy Tauri 2 + React 19 desktop app (being replaced by block/buzz-app) web/ # Browser web client (repo browser, served by the relay) mobile/ # Flutter mobile app -migrations/ # SQL migrations (auto-applied on relay startup) +migrations/ # SQL migrations (run at relay startup when BUZZ_AUTO_MIGRATE is on: binary default off, Helm default on; deployments at scale apply them out of band — see docs/database.md) +schema/ # Desired-state schema (schema/tables/public/ is the declarative schema source) scripts/ # Dev tooling .env.example # Config template — copy to .env before running ``` @@ -313,7 +319,9 @@ Filters and queries must scope to `h` tags when operating within a channel. This applies to events *inside* a channel. Addressable events that describe a channel carry its id in their `d` tag instead: kind:39000 (metadata), kind:39001, kind:39002 (membership). `get_channels` resolves a user's channels -from the `d` tag of their kind:39002 events, not from `h`. +from the `d` tag of their kind:39002 events, not from `h`. The relay leaves +system channels out of these reads unless the filter asks for them (see +`docs/nips/NIP-CL.md`). **Agent-facing operations go in `buzz-cli`**: New agent-facing features belong in `buzz-cli` — add a subcommand there first, then wire the REST/WebSocket call in `client.rs`. `buzz-dev-mcp` (shell + file tools for `buzz-agent`) is separate. @@ -363,8 +371,13 @@ canonical signed Nostr event fields (`id`, `pubkey`, `kind`, `content`, `{event_id, accepted, message}`; creates add the entity ID. Exit codes: 0=ok, 1=input error, 2=network/relay, 3=auth, 4=other, 5=write conflict (NIP-33 LWW). -`--format compact` is a **global** flag — it goes before the subcommand: +`--format` is a **global** flag — it goes before the subcommand: `buzz --format compact channels list`, NOT `buzz channels list --format compact`. +Without the flag it falls back to `BUZZ_OUTPUT_FORMAT` (unknown values mean +`json`); buzz-acp sets that to `agent` for agent +processes, so their event reads drop `sig`, trim the NIP-OA `auth` tag to +`["auth", ]`, and fold the NIP-10 `root`/`reply` tags into `reply_to` +(plus `thread_root` on nested replies); other tags remain. See `crates/buzz-cli/TESTING.md` for the full live-testing runbook. @@ -600,12 +613,16 @@ description. See [PR #803](https://github.com/block/buzz/pull/803). 4. **Worktrees: `cd` in the same command** — shell CWD doesn't persist between tool calls. Use `cd /path && cargo build` as one command. 5. **Desktop crate excluded from root workspace** — `cargo test` at repo root does NOT run desktop tests. Use `cargo test --manifest-path desktop/src-tauri/Cargo.toml` explicitly. 6. **React render perf: `React.memo` is all-or-nothing** — it only skips a re-render when *every* prop is reference-stable; one unstable prop (inline arrow/JSX, or a hook returning a fresh `{}`/`[]`/`Map` each render) defeats it. Two repeat offenders: (a) React Query results (`useMutation`/`useQuery`) are a **new object each render** — depend on the stable method (`mutation.mutateAsync`), not the object; (b) derived `Map`/array state that recomputes on a version bump — wrap in a content-equality ref cache (`shared/hooks/useStableReference.ts`). When chasing interaction lag, **measure with DevTools closed and no perf probes** (an open Web Inspector + per-keystroke `console.log` inflate the numbers), and isolate by removing one suspect at a time rather than guessing. -7. **`pgschema` omits seed DML and some storage parameters** — `schema/schema.sql` is an ordered `\i` manifest: `schema/tables/public/.sql` holds only that table's `CREATE TABLE` and `CREATE INDEX` statements (one table per file, the shape SchemaBot reads), and types, functions, partitions, triggers, and seed rows live in the sibling `schema/` directories. Fresh desired-state bootstraps use `./bin/pgschema apply`, which does not execute `INSERT` statements or preserve every table storage parameter from `schema/schema.sql`. Put each unsupported invariant in `scripts/reconcile-schema-after-pgschema.sql` as an idempotent convergence statement plus a live catalog or data assertion. Every `pgschema apply` caller must run that script. A string assertion against `schema.sql` alone does not prove the pgschema-created database has the intended state. +7. **`pgschema` omits seed DML and some storage parameters** — `schema/schema.sql` is an ordered `\i` manifest: `schema/tables/public/
.sql` holds only that table's `CREATE TABLE` and `CREATE INDEX` statements (one table per file, the shape out-of-band schema tools read), and types, functions, partitions, triggers, and seed rows live in the sibling `schema/` directories. Fresh desired-state bootstraps use `./bin/pgschema apply`, which does not execute `INSERT` statements or preserve every table storage parameter from `schema/schema.sql`. Put each unsupported invariant in `scripts/reconcile-schema-after-pgschema.sql` as an idempotent convergence statement plus a live catalog or data assertion. Every `pgschema apply` caller must run that script, and must run pgschema from the repository root so it reads `.pgschemaignore`; from any other directory a re-apply plans to drop the partitions the relay created at runtime. A string assertion against `schema.sql` alone does not prove the pgschema-created database has the intended state. --- ## Desktop App +> **Legacy.** `desktop/` will soon be superseded by +> [block/buzz-app](https://github.com/block/buzz-app). Never use it as an +> implementation or design reference for the mobile app, which is current. + The desktop app is Tauri 2 + React 19 + Vite + Tailwind CSS. Features are organized under `desktop/src/features/`. Biome handles linting and formatting. @@ -686,13 +703,20 @@ Key files: The mobile app lives in `mobile/` — a Flutter app using Riverpod + Hooks. +The mobile app is current. Do not use the legacy `desktop/` app as an +implementation or design reference for it: `desktop/` will soon be superseded +by [block/buzz-app](https://github.com/block/buzz-app). This does not relax +cross-client wire compatibility: mobile must keep reading and writing the same +event payloads as other clients (for example the `channel-sort` NIP-78 payload, +which buzz-app also uses), even where a payload is documented only in code. + ### Architecture - **State management:** Riverpod + `flutter_hooks` (`HookConsumerWidget`) -- **Theme:** Catppuccin Latte (light) / Macchiato (dark) — matches desktop +- **Theme:** Catppuccin Latte (light) / Macchiato (dark) - **Features:** Isolated under `lib/features/`, shared code in `lib/shared/` - **Nostr models:** `lib/shared/relay/nostr_models.dart` — event kinds must - stay in sync with `desktop/src/shared/constants/kinds.ts` + stay in sync with `crates/buzz-core/src/kind.rs` ### Rules @@ -763,12 +787,37 @@ usage. ### Testing Conventions -- Prefer **widget tests** over unit tests for UI components — test the - whole widget tree, not individual methods. -- Use `ProviderScope(overrides: [...])` to inject fake notifiers. -- Fake notifiers should extend the real notifier class and override `build()`. -- Use the `WidgetHelpers.testable()` wrapper for simple widget tests or - build a custom `ProviderScope` + `MaterialApp` when you need specific overrides. +Favor a few functional tests over many small ones. Every test is code we +maintain, so each one must protect a distinct user outcome or invariant. + +- **Start with journeys.** For each user-visible flow, write one widget test + that drives it through real widgets, notifiers and coordinators. Assert what + the user sees and what is durably saved, not internal stages or call counts. + Where Flutter can observe it, include semantics and non-primary input in the + journey: screen-reader labels and actions, keyboard, alternate gestures, text + scale and reduced motion. +- **Put failure paths inside the journey.** Retry, cancel, resume and a second + request arriving midway are steps of a journey, not separate tests. +- **Fake only external boundaries:** network, relay, OS, browser and storage. + Keep the flow's own notifiers and coordinators real. Override boundary + providers with `ProviderScope(overrides: [...])`, using + `WidgetHelpers.testable()` or a custom `ProviderScope` + `MaterialApp`. +- **Use focused tests only for what a journey can't enumerate or observe.** + That means security and data-loss invariants, protocol or crypto vectors, + pure functions with many input combinations, and accessibility or platform + behavior owned by native code that a widget journey can't observe, such as + VoiceOver stops and traits, Dynamic Type layout and reduced motion in native + views. Make them table-driven where cases repeat. +- **Extend before adding.** When a review finds a defect, add a step to an + existing journey or a row to an existing table. Don't add a new test per + finding. When a Review-Proven Rule says to test each path, cover the paths + as journey steps or table rows. +- **Delete what a journey covers.** Remove a lower-level test once a journey + demonstrably catches the same failures; passing through the same path is not + enough. Keep distinct safety, interop, accessibility and native-platform + invariants. +- **Prove it can fail.** Break the production code a test guards once and + confirm the test fails. A test that only exercises a fake protects nothing. --- diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 82d7cac7bcb..b7e5df8d711 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -145,7 +145,7 @@ buzz-admin (operator CLI: relay membership + key generation) buzz-test-client (integration test harness + manual CLI) ``` -**Key architectural principle:** The relay is the single source of truth. `buzz-relay` orchestrates all subsystems by calling them directly — it imports `buzz-db`, `buzz-auth`, `buzz-pubsub`, `buzz-search`, `buzz-audit`, and `buzz-workflow`. However, those subsystems are isolated from each other: `buzz-workflow` never calls `buzz-pubsub`, `buzz-search` never calls `buzz-db`, etc. Cross-subsystem coordination happens only through the relay. In multi-community mode, the relay also owns propagation of `TenantContext`; service crates should receive community-scoped inputs rather than independently deriving tenancy from client-controlled event tags. +**Key architectural principle:** The relay is the single source of truth. `buzz-relay` orchestrates all subsystems by calling them directly — it imports `buzz-db`, `buzz-auth`, `buzz-pubsub`, `buzz-search`, `buzz-audit`, and `buzz-workflow`. However, those subsystems are isolated from each other: `buzz-workflow` never calls `buzz-pubsub`, `buzz-search` never calls `buzz-db`, etc. The one sanctioned exception is `buzz-audit`, which calls `buzz_db::deletion::guard_community_write` to admit each audit append into its community before taking the chain lock. Cross-subsystem coordination happens only through the relay. In multi-community mode, the relay also owns propagation of `TenantContext`; service crates should receive community-scoped inputs rather than independently deriving tenancy from client-controlled event tags. --- @@ -189,6 +189,7 @@ The `kind` integer is the only dispatch switch. The relay routes, stores, and fa | 45003 | KIND_FORUM_COMMENT | Forum thread reply | | 46001–46012 | KIND_WORKFLOW_* | Workflow execution events | | 20001 | KIND_PRESENCE_UPDATE | Ephemeral presence heartbeat | +| 20003 | KIND_AGENT_PRESENCE | Ephemeral presence heartbeat an owner sends for its agents | `buzz-core` defines each event kind as a `pub const u32` and exports the full registry as `ALL_KINDS: &[u32]` (127 kinds at the time of writing); `crates/buzz-core/src/kind.rs` is the source of truth for the current list. Kinds are `u32` (NIP-01 specifies unsigned integer; `u32` covers the full range). Buzz uses both standard Nostr kinds (e.g., kind 7 for reactions) and custom ranges (40000+). @@ -305,6 +306,8 @@ Ephemeral events bypass DB storage, audit, and search. Two sub-paths: ``` Presence events skip membership checks and use local-only fan-out. Multi-node presence fan-out would require Redis pub/sub (documented as future work). +**Agent presence (kind 20003):** an owner's one heartbeat for all its agents, so an app that runs agents without a socket per agent can still show them online. Content is the status (`online`, `away` or `offline`) and each `p` tag names an agent, at most 256. WebSocket only. The relay keeps the agents the signer owns (`users.agent_owner_pubkey`, through the owner cache), skips the rest, and sets or clears their presence exactly as their own kind 20001 would, so the presence snapshot reports them unchanged. It is not fanned out, since live presence is matched by author; viewers see it on their next snapshot read. + **Other ephemeral events (e.g., typing indicators):** ``` 1. VERIFY — spawn_blocking(verify_event) @@ -314,6 +317,8 @@ Presence events skip membership checks and use local-only fan-out. Multi-node pr 5. LOCAL FAN-OUT — sub_registry.fan_out → conn_manager.send_to ``` +Typing indicators (kind 20002) are also accepted through HTTP `POST /events`, for clients that sign events but hold no WebSocket, such as app-hosted agents. The HTTP route requires an `h` tag, checks bans/timeouts and the community write fence, then runs steps 1–5 above. + Ephemeral events are never stored in Postgres and never appear in REQ historical queries. ### Handler Semaphore @@ -373,7 +378,7 @@ This prevents a race where a non-member receives live fan-out events from a priv ### Historical Query (EOSE) -After registering, the REQ handler queries Postgres for stored events matching the filters (up to 500 per filter, hard cap). These are sent as `["EVENT", sub_id, event]` frames before `["EOSE", sub_id]`. New events arriving after EOSE are delivered via the fan-out path. +After registering, the REQ handler queries Postgres for stored events matching the filters (up to 500 per filter, hard cap). These are sent as `["EVENT", sub_id, event]` frames before `["EOSE", sub_id]`. New events arriving after EOSE are delivered via the fan-out path. For an ordinary (non-search) REQ, EOSE marks a successful bounded history query, so a failed history read never ends in EOSE: it retires the subscription with `["CLOSED", sub_id, "error: query timed out"]` for a statement timeout or `"error: database error"` for anything else. **Client consumption invariant.** A client rebuilding channel state must open its live subscription before (or overlapping) the finite history @@ -469,7 +474,7 @@ All database access. Uses `sqlx::query()` (runtime, not compile-time macros) — | `user.rs` | User profile storage | | `error.rs` | Database error types | -**Channel types:** `Stream`, `Forum`, `Dm`, `Workflow` +**Channel types:** `Stream`, `Forum`, `Dm`, `Workflow`, `System` **Member roles:** `Owner`, `Admin`, `Member`, `Guest`, `Bot` **Workflow statuses:** `Active`, `Disabled`, `Archived` **Run statuses:** `Pending`, `Running`, `WaitingApproval`, `Completed`, `Failed`, `Cancelled` @@ -496,8 +501,11 @@ shape is an explicit transition, not a flag day: - **Community fence (transition foundation):** runtime admission APIs now offer one shared community deletion lock per transaction, while deletion lifecycle transitions take the matching exclusive lock. A supported serving transaction - never spans communities; unmigrated paths still rely on trigger/function - backstops. + never spans communities. Serving leases use the same application admission. + Startup and deletion catalog validation tolerate the universal fence objects + being absent; inventories still record live fences for older executors. + Desired-state bootstraps install only explicitly declared legacy fences. + New tenant-owned tables rely on application admission, without a fence trigger. - **Replica floor (transition foundation):** runtime now provides a shared replica-floor lock helper for compliant channel-event writers, and the writer probe handshake takes the exclusive counterpart before sampling `S`, scanning @@ -508,7 +516,8 @@ shape is an explicit transition, not a flag day: - **Dual enforcement (current):** application-owned lock+precheck paths run in front of the existing trigger/function enforcement; commit-time trigger checks remain authoritative during this phase. Removing those backstops is a separate - gated migration, not part of the lock foundation. + gated migration, after every pod tolerates their absence and no rollback below + that version is expected. - **Role separation:** relay/runtime code owns admission and lock protocols; operator maintenance/backfill workflows must either use those protocols or run under explicit reviewed procedures that keep routing fences closed. @@ -518,6 +527,11 @@ shape is an explicit transition, not a flag day: behavior, pgschema bootstraps run reconciliation, and lock/pool metrics remain the operational evidence path for rollout safety. +The database end state (a declarative schema source, applied out of band at +scale; no triggers, functions, or foreign keys; backfills outside migrations) +and the expand → deploy → backfill → contract rules every schema change follows +are in [docs/database.md](docs/database.md). + **Does NOT:** cache queries, implement connection pooling logic (delegated to sqlx), or make network calls outside Postgres. --- @@ -568,12 +582,16 @@ re-authorizes every candidate hit before returning it. **Key behaviors:** - `SearchService::new(pool)` wraps a `PgPool`; `search(&SearchQuery)` runs a parameterized FTS query against the `events.search_tsv` GIN index and returns - `SearchResult` (candidate `SearchHit`s). + `SearchResult` (candidate `SearchHit`s). Prefix-mode relevance (outside + profile search) ranks only the newest 1000 matches, which Postgres may read + from the primary key backward instead of the GIN index. - `ChannelScope` makes the channel constraint explicit (`Any` / `ChannelLessOnly` / `Channels` / `ChannelsOrChannelLess`), closing the ambiguity the old `Option> + bool` matrix could not express. -- Every query carries `community_id`; the FTS predicate is BitmapAnd-ed with - the community-leading btree filters so a query never crosses tenants. +- Every query carries `community_id`, and every access path leads with it (the + FTS predicate BitmapAnd-ed with community-leading btree filters, or the + `(community_id, created_at, id)` primary key read backward), so a query never + crosses tenants. - Permission filtering is **caller's responsibility** — `buzz-search` returns candidate hits; the relay re-authorizes each one (channel membership, `#p`, owner gates) before delivering it. @@ -591,7 +609,7 @@ Tamper-evident append-only log with SHA-256 hash chaining. **Hash covers:** seq (big-endian bytes), timestamp (RFC3339), event_id, event_kind (big-endian), actor_pubkey, action string, channel_id (16 bytes or 16 zero bytes if None), canonical metadata JSON (BTreeMap for deterministic key ordering), prev_hash. -**Single-writer guarantee:** `pg_advisory_lock` before each transaction. Lock released in all branches including panic (`catch_unwind`). +**Single-writer guarantee:** each append runs in one transaction that first passes community write admission (the shared community deletion lock plus the active/undeleted check, via `buzz-db`'s `DeletionStore::guard_transaction`), then takes a per-community `pg_advisory_xact_lock`. Both locks release on commit or rollback, including a dropped future. **10 audit actions:** `EventCreated`, `EventDeleted`, `ChannelCreated`, `ChannelUpdated`, `ChannelDeleted`, `MemberAdded`, `MemberRemoved`, `AuthSuccess`, `AuthFailure`, `RateLimitExceeded`. @@ -868,6 +886,18 @@ Every security-sensitive operation uses an explicit, verified pattern. No implic Applied in: `buzz-auth` (JWKS boundary), `buzz-workflow` (CallWebhook action), desktop `link_preview` (SSRF check). +JWKS uses `public_https` by default and applies this public-address check to +every resolved answer before pinning the first permitted address. Explicit +`mesh` mode accepts HTTP only for a DNS hostname (not an IP literal) without +userinfo, query, fragment, or backslash. It checks every DNS answer against the +positive address allow-list in the +[NIP-FI JWKS snapshot contract](docs/nips/NIP-FI.md#jwks-snapshot), rejects +empty or mixed answers, and pins the first checked address. Mesh selection is +an operator assertion, not a transport property the Rust relay verifies. +Operators must enforce mutual TLS peer authentication and integrity +protection, authorize the JWKS endpoint, and restrict egress. Unauthenticated +HTTP permits key substitution and issuer impersonation. + ### Audit Integrity - Hash chain: each entry's SHA-256 covers all fields including `prev_hash` — tampering any entry breaks all subsequent hashes @@ -933,7 +963,10 @@ kinds via `CASE WHEN kind IN (1059, 30300, 30622) THEN NULL`, so those rows are storage-level unsearchable (a `NULL` tsvector never matches `@@`). A GIN index (`idx_events_search_tsv`) backs the `@@` probe; in multi-community mode the community-leading btree filters BitmapAnd with the GIN probe so every query is -fenced to its `community_id`. +fenced to its `community_id`. Prefix-mode relevance ranks only the newest 1000 +matches; for a common prefix Postgres reads the `(community_id, created_at, id)` +primary key backward and stops there, still fenced by its leading +`community_id`. --- diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dbe4ba5dd2b..8495d521b74 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -83,7 +83,8 @@ We review as capacity allows — focused PRs that follow this guide move fastest | Rust | 1.88+ | Install via [rustup](https://rustup.rs/) | | Node.js | 24+ | Required for desktop app commands and `just ci` | | pnpm | 10+ | Required for desktop app commands and `just ci` | -| Flutter | 3.41+ | Required for mobile app — install via [flutter.dev](https://docs.flutter.dev/get-started/install) | +| Flutter | 3.47.7+ | Required for mobile app — install via [flutter.dev](https://docs.flutter.dev/get-started/install) | +| Xcode | 27.1 recommended; 26 supported | Required for iOS builds; see [iOS toolchain setup](mobile/README.md#ios-toolchain) | | Docker | 24+ | For Postgres, Redis, MinIO | | `just` | latest | Task runner — `cargo install just` | | `lefthook` | 2.1.3 (Hermit-pinned) | Auto-installed by `just hooks` — no manual install needed | @@ -303,6 +304,34 @@ just ci This is the same check that runs in CI. PRs that fail `just ci` will not be merged. If `just ci` fails on formatting, `just fix-all` fixes it in one shot (`rustfmt` + Tauri fmt + desktop, web, and mobile formatters). +Pull requests also get an advisory **flake check**. `scripts/flake-check.mjs +select` maps the diff against the base branch to the Rust tests (in the root +and desktop Tauri workspaces), desktop `node --test` files, Playwright smoke +journeys and Flutter test files it changed: a hunk inside a test body selects +that test; a change anywhere else in a test file (imports, helpers, hooks, +the code under test beside Rust tests) or a new file selects the whole file. +An integration test binary's tests include those of the file modules its +root declares (`mod x;`, `#[path = "…"] mod x;`), and a change to a helper +module under a crate's `tests/` selects every integration test binary that +includes it. The `Flake check (…)` lanes then +run that selection ten times with retries off; a repetition that fails is +usually a race in the test or the code under it, worth fixing before it +lands, though a test that needs infrastructure or the regular lane's +cross-file concurrency can fail every repetition instead. Some changed tests +are deliberately not repeated: `#[ignore]`d and Postgres-backed Rust tests, +Playwright specs outside the `smoke` project, `*.perf.ts` measurements, and a +selection too large for its lane (over 64 KiB of entries, or more Playwright +journeys than ten repetitions can finish within the lane's timeout). +A Rust test repeats in every feature graph the regular lanes test its +package in that lists it (LaunchDarkly, the App Attest development feature, +the desktop crate's lib under `mesh-llm`), since a `#[cfg(feature = …)]` arm +compiles only in its own graph; the step summary marks a row from a feature +graph with that feature. A selected test that no such graph lists on the +Linux runner, such as a macOS-only one, fails its lane by name rather than +passing unrepeated. +The lanes skip when no test changed and are not required to merge; their +step summaries show the selection and the test × iteration table. + --- ## Code Style diff --git a/Cargo.lock b/Cargo.lock index 078365c1aa7..b7e7e325c85 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -158,24 +158,6 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" -[[package]] -name = "appattest" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d0a66767aaf7d483c556386fb68ca2fba9347684d8bb17a4bd8b755851870f7" -dependencies = [ - "arrayvec", - "aws-lc-rs", - "base64 0.22.1", - "byteorder", - "ciborium", - "minicbor", - "openssl", - "p256", - "rustls-pki-types", - "rustls-webpki", -] - [[package]] name = "apple-native-keyring-store" version = "1.0.2" @@ -540,19 +522,19 @@ checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "aws-config" -version = "1.12.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8d7b388a9fc3a6db15a5ec778c38b354eff1364882c94d08e0252f7a47dcaa4" +checksum = "939e256cfe0ed68e96d32f194339d88210d5c10685cff0787400158f852c6819" dependencies = [ "aws-credential-types", "aws-runtime", "aws-sdk-sts", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.65.0", "aws-smithy-runtime", "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 1.1.0", "aws-smithy-types", "aws-types", "bytes", @@ -566,9 +548,9 @@ dependencies = [ [[package]] name = "aws-credential-types" -version = "1.3.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e93964ffdaf57857f544be3666a5f57570bb699e934700f11b49708f61bb556e" +checksum = "576e8da35751dc85409ec1cd15999f692c277a1fbe9336780fe6f15e49dd637d" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -627,9 +609,9 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.10.0" +version = "1.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b8a9911551b4ea6ca13805ef52ed96f7d2bbb43cc3b4a14cb0776a71f33cfaa" +checksum = "9a0cc2144962cc5e358467b16e3d82981b1d424a7c3d34602be0036103814a1a" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -655,20 +637,20 @@ dependencies = [ [[package]] name = "aws-sdk-dynamodb" -version = "1.131.0" +version = "1.134.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18d4e5ed975d72bce813432317e0cf63d4b37b5f62e9bc0a4922b2fe2b2c4347" +checksum = "0970e2aa17bb199c48f9f679a3fcea64a5153cebcc74884e7abd08dc61f4828b" dependencies = [ "arc-swap", "aws-credential-types", "aws-runtime", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.65.0", "aws-smithy-observability", "aws-smithy-runtime", "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 1.1.0", "aws-smithy-types", "aws-types", "bytes", @@ -693,13 +675,13 @@ dependencies = [ "aws-smithy-checksums", "aws-smithy-eventstream", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.63.1", "aws-smithy-observability", "aws-smithy-runtime", "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 0.2.1", "aws-smithy-types", - "aws-smithy-xml", + "aws-smithy-xml 0.62.1", "aws-types", "bytes", "fastrand", @@ -718,20 +700,20 @@ dependencies = [ [[package]] name = "aws-sdk-secretsmanager" -version = "1.120.0" +version = "1.123.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "832dc9d5dbd19c0ac6b24e21ef9f3e553695d0f3928de7134567407b3b7a447d" +checksum = "430ad7b100af8d62d9fa1f22dfb41e31bf4b462a1df0755c70946f3e4b1ee727" dependencies = [ "arc-swap", "aws-credential-types", "aws-runtime", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.65.0", "aws-smithy-observability", "aws-smithy-runtime", "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 1.1.0", "aws-smithy-types", "aws-types", "bytes", @@ -743,23 +725,23 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.119.0" +version = "1.122.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03e490aa904849b38e770922faac779dd245f5cf8be81f19065acf18d276e4ad" +checksum = "a10491638d9e560db90c484dbc77d73b8ad202780cdc9b9e92180603acb07ada" dependencies = [ "arc-swap", "aws-credential-types", "aws-runtime", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.65.0", "aws-smithy-observability", "aws-smithy-query", "aws-smithy-runtime", "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 1.1.0", "aws-smithy-types", - "aws-smithy-xml", + "aws-smithy-xml 0.64.0", "aws-types", "fastrand", "http 1.4.2", @@ -769,9 +751,9 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.6.0" +version = "1.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2312577f088c9fbf4206dfdb884cf1de9407b43e1a923cbed5237775116fc24b" +checksum = "9d3d48355cb86b15ccbb38b8bc6b87500c824ee4ed8cbc10a961b435242b5e16" dependencies = [ "aws-credential-types", "aws-smithy-eventstream", @@ -791,9 +773,9 @@ dependencies = [ [[package]] name = "aws-smithy-async" -version = "1.3.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f02e407fb3b54891734224b9ffac8a71fdd35f542500fa1af95754a6b2beb316" +checksum = "de681a9df7ff79823b759ee17061076ec049cd61a8fa28ece4293d89c9f6964e" dependencies = [ "futures-util", "pin-project-lite", @@ -823,9 +805,9 @@ dependencies = [ [[package]] name = "aws-smithy-eventstream" -version = "0.61.4" +version = "0.61.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80c2051c2f1016fb8e6548dd07b8bc2ac9c3fe583721444b92f515e856d31609" +checksum = "14c8a643d0e114cae39e45415e3225793a6396698b7779a0c9e3ede731d8fc71" dependencies = [ "aws-smithy-types", "bytes", @@ -834,9 +816,9 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.64.1" +version = "0.64.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "639b4d8f8555f24a9be649811c3eb0b4d4616f4d61daf0c32e28873bc1ea9af1" +checksum = "ba8960956e0ee0ca3f3161e62913a61bc65a7ef638ca3ef1d4c7159f8443e398" dependencies = [ "aws-smithy-eventstream", "aws-smithy-runtime-api", @@ -850,15 +832,14 @@ dependencies = [ "http-body-util", "percent-encoding", "pin-project-lite", - "pin-utils", "tracing", ] [[package]] name = "aws-smithy-http-client" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51c89cc3f1f281d659a67a519a1b5c6d445b5ce09fa7e5aee40c2c2707e9509d" +checksum = "aeca69f65362f7375891008ea525fd440ad6db1836458ac188c87b126804114e" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -885,7 +866,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3385d469edbe8b60cc72002784652b5efca39178192aa9cc4b44c9875c6bdc18" dependencies = [ "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 0.2.1", + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-json" +version = "0.65.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f6835fb38bc6fc8c91e584b0366eb4ff595e5c485134dbf4f0bdeaa18856666" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema 1.1.0", "aws-smithy-types", ] @@ -900,29 +892,29 @@ dependencies = [ [[package]] name = "aws-smithy-query" -version = "0.62.1" +version = "0.64.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1d1d71f6562be974caa85442ecd90194c40fdb5df045f182a6c2e872ce95056" +checksum = "165f6265342a313ad6f7a1ac17a0cdba30c522039d3f41d8d6871ce8a2b34ecd" dependencies = [ "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 1.1.0", "aws-smithy-types", - "aws-smithy-xml", + "aws-smithy-xml 0.64.0", "urlencoding", ] [[package]] name = "aws-smithy-runtime" -version = "1.16.0" +version = "1.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6e302ac1d88b99652489df31abdec6ac42a2ab2ac3982ad0ac49f64dfaf28ba" +checksum = "79cef0c41e8ca40f74f68ed8ee4d1ddbde12bb847b95ef9b3c025681ddfae58a" dependencies = [ "aws-smithy-async", "aws-smithy-http", "aws-smithy-http-client", "aws-smithy-observability", "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 1.1.0", "aws-smithy-types", "bytes", "fastrand", @@ -930,16 +922,15 @@ dependencies = [ "http-body 1.0.1", "http-body-util", "pin-project-lite", - "pin-utils", "tokio", "tracing", ] [[package]] name = "aws-smithy-runtime-api" -version = "1.19.0" +version = "1.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0730c16f91124c6a2abb4932c77e299288b3dd9f967ea2e9ec48cc6731e87a4" +checksum = "06788004dea79018413f381891da6dcca23196acd79c4d51b767fbdacc76eae4" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api-macros", @@ -955,9 +946,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api-macros" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "221eaa237ddf1ca79b60d1372aad77e47f9c0ea5b3ce5099da8c61d027dc77b3" +checksum = "ce237f890a2283c9b328955c3b0918d2d5bf66a1cb6ceccd36a0047246fbc6e6" dependencies = [ "proc-macro2", "quote", @@ -975,11 +966,22 @@ dependencies = [ "http 1.4.2", ] +[[package]] +name = "aws-smithy-schema" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bec8dae1015a6d959a5faf86fbdf77486cfe19e4ccb6ee009d8a46efc45028a4" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-types", + "http 1.4.2", +] + [[package]] name = "aws-smithy-types" -version = "1.8.1" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69bb407740a197147da48238ecc94498493c9e85445732360cec180296ca45f1" +checksum = "55752025c39c0e094962fd73f1c57014b191f678612e517f219468e575af9822" dependencies = [ "base64-simd", "bytes", @@ -993,7 +995,6 @@ dependencies = [ "itoa", "num-integer", "pin-project-lite", - "pin-utils", "ryu", "serde", "time", @@ -1008,21 +1009,33 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b932c8d6dc127fc980eecd78f8694ae9b9551b69a93a7def2a199c1c0033daf" dependencies = [ "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 0.2.1", + "aws-smithy-types", + "xmlparser", +] + +[[package]] +name = "aws-smithy-xml" +version = "0.64.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6581618a2fd006159016d1c36881d94d61b4a017f4a8d631310875e18252413" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema 1.1.0", "aws-smithy-types", "xmlparser", ] [[package]] name = "aws-types" -version = "1.6.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "209f3a6d82a6e9e5f94abbed94c7a26e1c052341002bf57a5fb5481f625896fc" +checksum = "6ba0d970d3947bb064d2ae1d823efa186c1dbde88f88afa8f6549946e5d8deee" dependencies = [ "aws-credential-types", "aws-smithy-async", "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 1.1.0", "aws-smithy-types", "rustc_version", "tracing", @@ -1537,14 +1550,26 @@ dependencies = [ "webbrowser", ] +[[package]] +name = "buzz-audio-client" +version = "0.1.0" +dependencies = [ + "futures-util", + "neteq", + "opus", + "serde_json", + "tokio", + "tokio-tungstenite 0.29.0", +] + [[package]] name = "buzz-audit" version = "0.1.0" dependencies = [ "buzz-core", "buzz-datastore-tracing", + "buzz-db", "chrono", - "futures-util", "hex", "metrics", "serde", @@ -1577,6 +1602,8 @@ dependencies = [ "thiserror 2.0.18", "tokio", "tracing", + "unicode-normalization", + "unicode-properties", "url", "uuid", ] @@ -1718,6 +1745,7 @@ dependencies = [ "sqlx", "thiserror 2.0.18", "tokio", + "tokio-util", "tracing", "uuid", ] @@ -1734,6 +1762,7 @@ dependencies = [ "clap", "deadpool-redis", "hex", + "nostr 0.44.7", "redis", "serde", "serde_json", @@ -1812,6 +1841,7 @@ dependencies = [ "tracing", "ulid", "uuid", + "webp", ] [[package]] @@ -1905,21 +1935,25 @@ name = "buzz-push-gateway" version = "0.1.0" dependencies = [ "aes-gcm", - "appattest", "async-trait", + "aws-lc-rs", "axum", "base64 0.22.1", - "byteorder", "chrono", + "ciborium", "getrandom 0.4.3", "hex", "metrics", "metrics-exporter-prometheus", "minicbor", "nostr 0.44.7", + "openssl", + "p256", "proptest", "rand 0.10.1", "reqwest 0.13.4", + "rustls-pki-types", + "rustls-webpki", "serde", "serde_json", "sha2 0.11.0", @@ -1932,6 +1966,7 @@ dependencies = [ "tracing-subscriber", "url", "uuid", + "x509-parser", ] [[package]] @@ -1939,6 +1974,7 @@ name = "buzz-relay" version = "0.2.1" dependencies = [ "anyhow", + "arc-swap", "async-compression", "async-trait", "axum", @@ -1969,6 +2005,7 @@ dependencies = [ "http-body-util", "infer", "jsonwebtoken", + "lru 0.16.4", "metrics", "metrics-exporter-prometheus", "metrics-util", @@ -2002,6 +2039,7 @@ dependencies = [ "tracing", "tracing-opentelemetry", "tracing-subscriber", + "tungstenite 0.29.0", "url", "uuid", ] @@ -2016,6 +2054,8 @@ dependencies = [ "hex", "hmac 0.13.0", "iroh", + "metrics", + "metrics-util", "nostr 0.44.7", "postcard", "proptest", @@ -2145,8 +2185,10 @@ dependencies = [ "buzz-sdk", "buzz-ws-client", "chrono", + "clap", "futures-util", "hex", + "image", "nostr 0.44.7", "rand 0.10.1", "reqwest 0.13.4", @@ -2159,6 +2201,7 @@ dependencies = [ "thiserror 2.0.18", "tokio", "tokio-tungstenite 0.29.0", + "toml", "tracing", "tracing-subscriber", "url", @@ -2202,9 +2245,11 @@ dependencies = [ "serde", "serde_json", "serde_yaml", + "sqlx", "thiserror 2.0.18", "tokio", "tracing", + "tracing-subscriber", "uuid", ] @@ -2638,12 +2683,6 @@ dependencies = [ "serde_core", ] -[[package]] -name = "const-oid" -version = "0.9.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" - [[package]] name = "const-oid" version = "0.10.2" @@ -2913,12 +2952,16 @@ checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" [[package]] name = "crypto-bigint" -version = "0.5.5" +version = "0.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +checksum = "d811ea23fdfbcce2e2d5132344d471abdb8f901ac496a0893a3df92e70525e52" dependencies = [ - "generic-array", - "rand_core 0.6.4", + "cpubits", + "ctutils", + "getrandom 0.4.3", + "hybrid-array", + "num-traits", + "rand_core 0.10.1", "subtle", "zeroize", ] @@ -3010,6 +3053,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38d28fb6aa7dd680b3ffd1a3e64e115c177bdba63c0ae7f9cd48129ff05c4fa0" dependencies = [ "cmov", + "subtle", ] [[package]] @@ -3275,25 +3319,14 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5729f5117e208430e437df2f4843f5e5952997175992d1414f94c57d61e270b4" -[[package]] -name = "der" -version = "0.7.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" -dependencies = [ - "const-oid 0.9.6", - "pem-rfc7468 0.7.0", - "zeroize", -] - [[package]] name = "der" version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a878c850e9e421b20262e9b41f9c860e4785fa07541c266b62ff9d1ef998a80a" dependencies = [ - "const-oid 0.10.2", - "pem-rfc7468 1.0.0", + "const-oid", + "pem-rfc7468", "zeroize", ] @@ -3397,7 +3430,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", - "const-oid 0.9.6", "crypto-common 0.1.7", "subtle", ] @@ -3409,7 +3441,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ "block-buffer 0.12.0", - "const-oid 0.10.2", + "const-oid", "crypto-common 0.2.2", "ctutils", ] @@ -3507,16 +3539,17 @@ checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" [[package]] name = "ecdsa" -version = "0.16.9" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" +checksum = "c0681a4fc24c767085329728d8dfba959af91228aa4610cca4f8ce317ba46ae0" dependencies = [ - "der 0.7.10", - "digest 0.10.7", + "der", + "digest 0.11.3", "elliptic-curve", "rfc6979", - "signature 2.2.0", - "spki 0.7.3", + "signature 3.0.0", + "spki", + "zeroize", ] [[package]] @@ -3525,7 +3558,7 @@ version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29fcf32e6c73d1079f83ab4d782de2d81620346a5f38c6237a86a22f8368980a" dependencies = [ - "pkcs8 0.11.0", + "pkcs8", "serdect", "signature 3.0.0", ] @@ -3569,19 +3602,20 @@ dependencies = [ [[package]] name = "elliptic-curve" -version = "0.13.8" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" +checksum = "900f37e7e18d74a2347389af6cd69e3a30d5cc9e697f40004865679aa75e9c7c" dependencies = [ - "base16ct 0.2.0", + "base16ct 1.0.0", "crypto-bigint", - "digest 0.10.7", + "crypto-common 0.2.2", + "digest 0.11.3", "ff", - "generic-array", "group", - "pem-rfc7468 0.7.0", - "pkcs8 0.10.2", - "rand_core 0.6.4", + "hybrid-array", + "pem-rfc7468", + "pkcs8", + "rand_core 0.10.1", "sec1", "subtle", "zeroize", @@ -3840,11 +3874,11 @@ dependencies = [ [[package]] name = "ff" -version = "0.13.1" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" +checksum = "a1f686ab92a9fb0eaf188f6c6c87b89490baa6fdb0db4544ba4dc47f7942489f" dependencies = [ - "rand_core 0.6.4", + "rand_core 0.10.1", "subtle", ] @@ -4257,6 +4291,12 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "glob" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" + [[package]] name = "globset" version = "0.4.18" @@ -4284,12 +4324,12 @@ dependencies = [ [[package]] name = "group" -version = "0.13.0" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" +checksum = "7fd1a1c7a5206c5b7a3f5a0d7ccd3ff85d0c8f5133d62a02680255b0004af5f4" dependencies = [ "ff", - "rand_core 0.6.4", + "rand_core 0.10.1", "subtle", ] @@ -4704,7 +4744,9 @@ version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c" dependencies = [ + "subtle", "typenum", + "zeroize", ] [[package]] @@ -5818,6 +5860,16 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "libwebp-sys" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54cd30df7c7165ce74a456e4ca9732c603e8dc5e60784558c1c6dc047f876733" +dependencies = [ + "cc", + "glob", +] + [[package]] name = "line-clipping" version = "0.3.7" @@ -6694,6 +6746,21 @@ name = "minicbor" version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0452a60c1863c1f50b5f77cd295e8d2786849f35883f0b9e18e7e6e1b5691b0" +dependencies = [ + "half", + "minicbor-derive", +] + +[[package]] +name = "minicbor-derive" +version = "0.15.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd2209fff77f705b00c737016a48e73733d7fbccb8b007194db148f03561fb70" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] [[package]] name = "minidom" @@ -6987,6 +7054,19 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "neteq" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57fbe9601485d95750187c92344fa056e75055027b00e375c53d9f4b8b883112" +dependencies = [ + "log", + "ringbuf", + "serde", + "thiserror 1.0.69", + "web-time", +] + [[package]] name = "netlink-packet-core" version = "0.8.1" @@ -7645,6 +7725,15 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" +[[package]] +name = "openssl-src" +version = "300.6.1+3.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46eb8fb9fb3b61ce1c0f8a026c4c1a0714d3a9e138e7fbde78753ce2babc3846" +dependencies = [ + "cc", +] + [[package]] name = "openssl-sys" version = "0.9.116" @@ -7653,6 +7742,7 @@ checksum = "f28a22dc7140cda5f096e5e7724a6962ca81a7f8bfd2979f9b18c11af56318c4" dependencies = [ "cc", "libc", + "openssl-src", "pkg-config", "vcpkg", ] @@ -7743,6 +7833,24 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "opus" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33718946cc77d4032911d4efe03a66dbcbfbd2bb16c3da06aaeadcc637c32216" +dependencies = [ + "opusic-sys", +] + +[[package]] +name = "opusic-sys" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9d1ecdf206421bc74343ab3bb2f30ad2abbfee41fa341f7181fecbaf957769a" +dependencies = [ + "cmake", +] + [[package]] name = "ordered-float" version = "2.10.1" @@ -7825,14 +7933,15 @@ checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" [[package]] name = "p256" -version = "0.13.2" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" +checksum = "d2c9239b2dbc807adbbe147e8cf72ea7450c3a0aabe62cb8e75ff4ec22e1f72a" dependencies = [ "ecdsa", "elliptic-curve", + "primefield", "primeorder", - "sha2 0.10.9", + "sha2 0.11.0", ] [[package]] @@ -7957,15 +8066,6 @@ dependencies = [ "serde_core", ] -[[package]] -name = "pem-rfc7468" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" -dependencies = [ - "base64ct", -] - [[package]] name = "pem-rfc7468" version = "1.0.0" @@ -8143,12 +8243,6 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" -[[package]] -name = "pin-utils" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13bee6c73da26345c729282832b60b0363cf3dd9f4bfd81d8551b7a1c889a113" - [[package]] name = "piper" version = "0.2.5" @@ -8160,24 +8254,14 @@ dependencies = [ "futures-io", ] -[[package]] -name = "pkcs8" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" -dependencies = [ - "der 0.7.10", - "spki 0.7.3", -] - [[package]] name = "pkcs8" version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "451913da69c775a56034ea8d9003d27ee8948e12443eae7c038ba100a4f21cb7" dependencies = [ - "der 0.8.2", - "spki 0.8.0", + "der", + "spki", ] [[package]] @@ -8375,13 +8459,31 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "primefield" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c555a6e4eb7d4e158fcb028c835c3b8642206ddc279b5c6b202ef9a8bdb592f4" +dependencies = [ + "crypto-bigint", + "crypto-common 0.2.2", + "ff", + "rand_core 0.10.1", + "subtle", + "zeroize", +] + [[package]] name = "primeorder" -version = "0.13.6" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" +checksum = "5c9f42978c78a00e3d68f69fc03e57a234debae69da4020a4fb588fcdcd07b06" dependencies = [ "elliptic-curve", + "once_cell", + "primefield", + "serdect", + "wnaf", ] [[package]] @@ -9310,12 +9412,12 @@ checksum = "1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7" [[package]] name = "rfc6979" -version = "0.4.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" +checksum = "b4a459cddafb3fe76b31fd8f1108007566c40301feb64dc7b54656eb7388172b" dependencies = [ - "hmac 0.12.1", - "subtle", + "crypto-bigint", + "hmac 0.13.0", ] [[package]] @@ -9332,6 +9434,15 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "ringbuf" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79abed428d1fd2a128201cec72c5f6938e2da607c6f3745f769fabea399d950a" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "rmcp" version = "1.8.0" @@ -9745,14 +9856,14 @@ dependencies = [ [[package]] name = "sec1" -version = "0.7.3" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" +checksum = "d56d437c2f19203ce5f7122e507831de96f3d2d4d3be5af44a0b0a09d8a80e4d" dependencies = [ - "base16ct 0.2.0", - "der 0.7.10", - "generic-array", - "pkcs8 0.10.2", + "base16ct 1.0.0", + "ctutils", + "der", + "hybrid-array", "subtle", "zeroize", ] @@ -10201,7 +10312,6 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" dependencies = [ - "digest 0.10.7", "rand_core 0.6.4", ] @@ -10211,6 +10321,7 @@ version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d567dcbaf0049cb8ac2608a76cd95ff9e4412e1899d389ee400918ca7537f5" dependencies = [ + "digest 0.11.3", "rand_core 0.10.1", ] @@ -10492,16 +10603,6 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" -[[package]] -name = "spki" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" -dependencies = [ - "base64ct", - "der 0.7.10", -] - [[package]] name = "spki" version = "0.8.0" @@ -10509,7 +10610,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d9efca8738c78ee9484207732f728b1ef517bbb1833d6fc0879ca898a522f6f" dependencies = [ "base64ct", - "der 0.8.2", + "der", ] [[package]] @@ -11463,6 +11564,7 @@ dependencies = [ "futures-util", "log", "rustls", + "rustls-native-certs", "rustls-pki-types", "tokio", "tokio-rustls", @@ -12317,6 +12419,15 @@ dependencies = [ "web-sys", ] +[[package]] +name = "webp" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c071456adef4aca59bf6a583c46b90ff5eb0b4f758fc347cea81290288f37ce1" +dependencies = [ + "libwebp-sys", +] + [[package]] name = "webpki-root-certs" version = "1.0.10" @@ -12956,6 +13067,18 @@ dependencies = [ "windows-core 0.62.2", ] +[[package]] +name = "wnaf" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "795ca18b3fdb5e62bf982199278341ddcf7ebf7d32e25e212ad05d496e95f6fa" +dependencies = [ + "ff", + "group", + "hybrid-array", + "primefield", +] + [[package]] name = "writeable" version = "0.6.3" diff --git a/Cargo.toml b/Cargo.toml index de4d2d9890f..90fbc6d96ff 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,6 +31,7 @@ members = [ "crates/buzz-mesh-smoke", "crates/buzz-dev-mcp", "crates/buzz-voice", + "crates/buzz-audio-client", "crates/buzz-backend-kubernetes", "crates/buzz-backend-relay", "crates/buzz-remote-sessions", @@ -127,6 +128,8 @@ zeroize = "1.8" # Concurrent data structures dashmap = "6" moka = { version = "0.12", features = ["sync"] } +arc-swap = "1" +lru = "0.16" # Async stream utilities futures-util = "0.3" diff --git a/Dockerfile b/Dockerfile index 2c1e8adbb5a..02446e144f8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ # # Public Buzz relay image — published as ghcr.io/block/buzz:. # -# Builds the `buzz-relay` binary (Rust 1.95) and the `buzz-web` static bundle +# Builds the `buzz-relay` binary (Rust 1.98) and the `buzz-web` static bundle # (pnpm + vite), then assembles them into a small debian-slim runtime with # `git` available (the relay shells out to git for repo hydrate / receive-pack # / upload-pack — see crates/buzz-relay/src/api/git). @@ -11,7 +11,7 @@ # native arm64 runners (see .github/workflows/docker.yml). The Dockerfile # itself is platform-agnostic; do not add --platform pins. -ARG RUST_VERSION=1.95 +ARG RUST_VERSION=1.98 ARG NODE_VERSION=24 ARG DEBIAN_VERSION=bookworm @@ -54,6 +54,7 @@ FROM chef AS builder RUN apt-get update \ && apt-get install -y --no-install-recommends \ build-essential \ + cmake \ pkg-config \ libssl-dev \ ca-certificates \ diff --git a/Dockerfile.loadgen b/Dockerfile.loadgen new file mode 100644 index 00000000000..181b28fa9e8 --- /dev/null +++ b/Dockerfile.loadgen @@ -0,0 +1,37 @@ +# syntax=docker/dockerfile:1.7 +# +# Relay load generator image — published as ghcr.io/block/buzz-loadgen. +# Runs `buzz-loadgen` (crates/buzz-test-client, see perf/LOADGEN.md), for +# example as a Kubernetes Indexed Job with one `run --shard i/n` per pod. +# +# Multi-arch is produced by building this file on native amd64 and arm64 +# runners (see .github/workflows/loadgen-image.yml). Keep both bases pinned to +# manifest-list digests so either architecture resolves to immutable bytes. +FROM rust:1.98-alpine3.22@sha256:a1796ca6fa216d6727b5f61c69e4c665b120b1a4dcb969639e2f25f1ed309456 AS builder + +RUN apk add --no-cache \ + build-base \ + cmake \ + musl-dev \ + perl \ + pkgconf +WORKDIR /build +COPY . . +# --help proves the stripped binary starts on each native architecture. +RUN cargo build --locked --release -p buzz-test-client --bin buzz-loadgen \ + && strip target/release/buzz-loadgen \ + && target/release/buzz-loadgen --help >/dev/null + +FROM alpine:3.22@sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc1f695dce + +LABEL org.opencontainers.image.source="https://github.com/block/buzz" + +# Numeric UID/GID so Kubernetes runAsNonRoot can verify the user. +RUN apk add --no-cache ca-certificates \ + && addgroup -g 10001 loadgen \ + && adduser -D -H -u 10001 -G loadgen loadgen + +COPY --from=builder --chmod=0755 /build/target/release/buzz-loadgen /usr/local/bin/buzz-loadgen + +USER 10001:10001 +ENTRYPOINT ["/usr/local/bin/buzz-loadgen"] diff --git a/Dockerfile.push-gateway b/Dockerfile.push-gateway index 5b0aca8bd3e..f91e21b58dc 100644 --- a/Dockerfile.push-gateway +++ b/Dockerfile.push-gateway @@ -1,5 +1,5 @@ # syntax=docker/dockerfile:1.7 -ARG RUST_VERSION=1.95 +ARG RUST_VERSION=1.98 ARG DEBIAN_VERSION=bookworm FROM rust:${RUST_VERSION}-${DEBIAN_VERSION} AS chef @@ -13,7 +13,7 @@ RUN cargo chef prepare --recipe-path recipe.json FROM chef AS builder ARG BUZZ_PUSH_CARGO_FEATURES="" RUN apt-get update \ - && apt-get install -y --no-install-recommends build-essential pkg-config libssl-dev ca-certificates \ + && apt-get install -y --no-install-recommends build-essential cmake pkg-config libssl-dev ca-certificates \ && rm -rf /var/lib/apt/lists/* COPY --from=planner /build/recipe.json recipe.json RUN cargo chef cook --release --recipe-path recipe.json --features "$BUZZ_PUSH_CARGO_FEATURES" diff --git a/Dockerfile.sprig b/Dockerfile.sprig index ee8ac1a64f1..a2867f53d03 100644 --- a/Dockerfile.sprig +++ b/Dockerfile.sprig @@ -2,7 +2,7 @@ # Multi-arch is produced by building this file on native amd64 and arm64 runners. # Keep both bases pinned to manifest-list digests so either architecture resolves # to immutable source bytes. -FROM rust:1.95-alpine3.22@sha256:064dfc925d68d1a63f4fd2871bd7dc6e6ea56692989a487185855d62885d90aa AS builder +FROM rust:1.98-alpine3.22@sha256:a1796ca6fa216d6727b5f61c69e4c665b120b1a4dcb969639e2f25f1ed309456 AS builder RUN apk add --no-cache \ build-base \ diff --git a/Justfile b/Justfile index 014a27dee61..1492969ec06 100644 --- a/Justfile +++ b/Justfile @@ -397,8 +397,14 @@ test-unit: #!/usr/bin/env bash set -euo pipefail ./scripts/test-ensure-local-relay-key.sh + # Shared Huddle transport, codec, and loopback authentication regressions. + cargo test -p buzz-audio-client if command -v cargo-nextest &>/dev/null; then cargo nextest run -p buzz-core -p buzz-auth --lib + # Mesh JWKS proxy-bypass regression: infra-free loopback I/O with + # child-process environment isolation. Its integration-test binary + # is not selected by --lib above. Keep the Cargo fallback in step. + cargo nextest run -p buzz-auth --test mesh_jwks_proxy cargo nextest run -p buzz-audit --lib # Cross-pod disconnect payload tests (NIP-FI and community archive # fence); all infra-free. @@ -411,6 +417,10 @@ test-unit: # already runs it. cargo test -p buzz-auth --doc cargo nextest run -p buzz-voice --lib + # Inter-relay mesh: membership, gossip, registry, and loopback-UDP + # runtime tests. Infra-free (Redis-backed paths are unit-faked or + # never dialed). Keep the Cargo fallback in step. + cargo nextest run -p buzz-relay-mesh cargo nextest run -p buzz-cli # buzz-sdk builder/validation unit tests: pure event-builder and input # validation (e.g. the canvas writer-discipline/skew guard and the @@ -419,6 +429,9 @@ test-unit: # invocation hits. Enumerated explicitly because nothing in CI runs # `cargo test --workspace` — membership buys clippy/check, not tests. cargo nextest run -p buzz-sdk --lib + # buzz-workflow parser, cron, condition and denial-logging unit tests + # (infra-free). Its ignored Postgres cases run in the PostgreSQL lane. + cargo nextest run -p buzz-workflow --lib # buzz-acp owns the relay-to-agent trust boundary. Run its tests here so # forged relay events cannot regain a path into agent routing unnoticed. cargo nextest run -p buzz-acp @@ -432,6 +445,10 @@ test-unit: # #[ignore]d, so --lib runs only the infra-free set. Without this gate a # stray file in migrations/ or a broken lint ships green. cargo nextest run -p buzz-db --lib + # buzz-search unit tests: pure query-builder and cancel-outcome checks + # (no infra). Its Postgres suite runs in the PostgreSQL lane. Keep + # scripts/run-tests.sh in step. + cargo nextest run -p buzz-search --lib # buzz-db source-policy tests: infra-free scans of the crate sources # that enforce the event-write admission chokepoint, metric/provenance # contracts, and their fixtures. They live in an integration-test @@ -446,10 +463,11 @@ test-unit: # suites are otherwise absent from the infra-free unit lane. cargo nextest run -p buzz-media --lib \ -E 'test(=bucket_index::tests::bucket_snapshot_json_round_trip_preserves_community_keys)' - # buzz-admin: storage-snapshot worker plus the community archive - # command parser, validation, and propagation-evidence tests. + # buzz-admin: storage-snapshot worker, the community archive command + # parser, validation, and propagation-evidence tests, and the partition + # audit outcome contract, NIP-FI policy/backfill, and user-deletion intake. cargo nextest run -p buzz-admin \ - -E 'test(storage_snapshot) + test(/^communities::tests::/) + test(/^tests::communities_/)' + -E 'test(storage_snapshot) + test(/^communities::tests::/) + test(/^tests::communities_/) + test(=tests::partition_audit_outcomes_keep_their_json_names_and_exit_codes) + test(/^nip_fi::tests::/) + test(/^user_deletions::tests::/)' # Multi-tenant conformance gate (buzz-conformance): the independent # replay checker + golden fixtures. No infra — pure in-process trace # replay — so it belongs in the unit job. Run all targets (lib + the @@ -496,7 +514,8 @@ test-unit: # #[ignore]d Postgres suites — so these non-ignored tests ran in no lane # and a red one could ship green (exactly how a broken admin test slipped # past every gate once). Scoped to api::admin, not the whole buzz-relay - # --lib, because api::media has non-ignored tests that require Postgres. + # --lib, because much of handlers::req has non-ignored tests that require + # Postgres. # DB-backed api::admin tests are #[ignore]d and run in the PostgreSQL # lane; the non-ignored ones reject before touching the database. Any # new non-ignored test here must stay DB-free: without a database, a @@ -517,20 +536,27 @@ test-unit: # unit job either. The REQ subscription-lifecycle tests are picked by # exact name for the same reason: the rest of handlers::req needs a DB. # The third clause adds the NIP-FI HTTP ingress and its router/config - # neighbours: nip_fi_http, nip_fi_config, router, api::parse_query_tests, - # and the Git transport off_mode_precedence_tests. All are infra-free + # neighbours: nip_fi_http, nip_fi_config, nip_fi_resolver (snapshot + # resolver and its WS/HTTP/audio router tests), router, api::parse_query_tests, + # the profile search request bounds (api::profiles), the per-user + # search cap (api::bridge::search_slots), and the Git + # transport off_mode_precedence_tests. All are infra-free # and finish in well under a second with no DATABASE_URL, so none wait # out the sqlx acquire timeout. `--bin buzz-relay` adds main.rs's # `tests::` module (JWKS refresh cadence) and `composition_tests::` # (JWKS source + refresh-loop composition), which `--lib` cannot reach # because they live in the binary target; the nested # `tests::postgres_tests::` stays in the PostgreSQL lane. - # The fourth clause adds the startup step timers (startup_steps) and the - # env- and log-capture tests in telemetry and config. All three modules - # are infra-free; config parses env into a struct and never connects. + # The fourth clause adds the startup step timers (startup_steps), the + # mesh boot wiring (mesh_boot; Redis and Postgres point at refused + # ports), and the env- and log-capture tests in telemetry and config. + # All four modules are infra-free; config parses env into a struct and + # never connects. # `^config::` is anchored so it does not also select nip_fi_config. + # subscription::tests covers the in-memory registry and its fan-out + # indexes; it is infra-free. cargo nextest run -p buzz-relay --lib --bin buzz-relay \ - -E 'test(/^api::admin::/) + test(/^handlers::channel_authz::/) + test(/^handlers::moderation_authz::/) + test(/^handlers::side_effects::tests::/) + test(/^storage_sweep::tests::/) + test(/^nip_fi_core::tests::/) + test(/^nip_fi_http::tests::/) + test(/^nip_fi_config::tests::/) + test(/^readiness::tests::/) + test(/^router::tests::/) + test(/^api::parse_query_tests::/) + test(/^api::git::transport::off_mode_precedence_tests::/) + test(/^audio::join::tests::/) + test(/^audio::handler::tests::/) + test(/^nip_fi_gate::tests::/) + test(/^nip_fi_session::tests::/) + test(/^nip_fi_shadow::tests::/) + test(/^nip_fi_shadow_session::tests::/) + test(/^startup_steps::tests::/) + test(/^telemetry::tests::/) + test(/^config::tests::/) + test(=state::tests::neither_a_confirmed_inactive_community_nor_a_failed_lookup_admits_the_socket) + test(=state::tests::periodic_revalidation_disconnects_inside_the_fenced_callback) + test(=state::tests::bare_community_disconnect_fails_closed_when_the_fence_is_unavailable) + test(=connection::tests::send_loop_sends_policy_close_when_community_is_archived) + test(=state::tests::community_disconnect_then_nip_fi_keeps_community_deleted_reason) + test(=state::tests::disconnect_community_wins_reason_losing_nip_fi_does_not_enqueue_frame) + test(=handlers::req::tests::timed_out_historical_read_deregisters_before_closed) + test(=handlers::req::tests::superseded_timeout_leaves_replacement_intact) + test(=handlers::req::tests::search_claim_retires_live_and_yields_to_replacement) + test(=handlers::req::tests::concurrent_claims_and_stale_teardowns_keep_the_last_owner) + test(=handlers::req::tests::timeout_closed_is_emitted_before_a_replacement_can_claim) + test(=handlers::req::tests::revoke_then_replacement_keeps_replacement_whole) + test(=handlers::req::tests::claims_after_connection_cleanup_are_refused) + test(=handlers::req::tests::dropped_terminal_frame_cancels_connection) + test(=handlers::req::tests::revoke_dropped_terminal_frame_cancels_connection) + (kind(bin) & (test(/^tests::/) + test(/^composition_tests::/) + test(/^env_filter_tests::/)) - test(/^tests::postgres_tests::/))' + -E 'test(/^subscription::tests::/) + test(/^api::admin::/) + test(/^access::tests::/) + test(/^api::git::transport::access_log_tests::/) + test(=api::bridge::postgres_tests::nip98_replay_check_logs_only_its_refusal) + test(=api::bridge::postgres_tests::warn_only_artifact_read_lines_keep_the_filter_summary) + test(=connection::tests::close_totals_count_frames_the_writer_flushed) + test(=connection::tests::dispatcher_counts_each_req_and_count_frame_once) + test(=handlers::req::tests::unqueued_eose_logs_aborted_not_served) + test(=api::bridge::postgres_tests::reject_code_keeps_only_a_known_prefix) + test(=api::bridge::postgres_tests::nip98_replay_check_fails_closed_when_guard_errors) + test(=api::bridge::postgres_tests::verify_bridge_auth_rejects_nip98_event_signed_for_wrong_communitys_host) + test(=api::bridge::postgres_tests::verify_bridge_auth_can_require_payload_tag_for_json_body_endpoints) + test(=api::bridge::postgres_tests::verify_bridge_auth_accepts_nip98_event_signed_for_matching_host) + test(=api::bridge::postgres_tests::moderation_read_query_bearing_nip98_event_verifies_with_matching_query) + test(=api::bridge::postgres_tests::moderation_read_query_bearing_nip98_event_rejected_against_bare_path) + test(=api::bridge::postgres_tests::moderation_read_audit_query_bearing_nip98_event_verifies) + test(=api::bridge::postgres_tests::moderation_read_restricted_no_query_still_verifies) + test(=connection::tests::aborting_lifecycle_owner_is_drop_safe) + test(=connection::tests::shutdown_terminalizes_auth_stalled_on_database_before_late_success) + test(=connection::terminal_close_delivery_tests::upgrade_would_deny_line_without_auth_logs_absent_at_close) + test(=handlers::event::tests::observer_owner_cache_is_scoped_to_community) + test(=state::tests::shared_counter_between_direct_and_fanout) + test(=connection::tests::startup_cancellation_writes_one_close_line) + test(=connection::tests::connection_limit_refusal_writes_one_close_line) + test(=connection::tests::dropped_authenticated_handler_writes_one_close_line) + test(=connection::tests::close_totals_count_a_batch_flushed_by_the_drain_after_cancellation) + test(=connection::tests::close_totals_skip_a_frame_the_socket_rejects) + test(=connection::tests::close_totals_skip_frames_stranded_by_a_failed_flush) + test(=connection::tests::read_before_login_logs_one_denial_from_the_handler) + test(=connection::tests::read_refused_by_a_full_handler_semaphore_logs_one_denial) + test(=connection::tests::read_refused_by_unavailable_admission_logs_one_denial) + test(=handlers::auth::tests::repeat_auth_after_a_terminal_logs_one_denial) + test(=nip_fi_upgrade::tests::deny_protected_logs_one_denied_auth_line) + test(=api::bridge::moderation_refusal_tests::moderation_lookup_failure_is_unavailable) + test(/^push_runtime::/) + test(/^operator_listener::tests::/) + test(/^handlers::channel_authz::/) + test(/^handlers::moderation_authz::/) + test(/^handlers::side_effects::tests::/) + test(/^handlers::coalesced_refresh::tests::/) + test(/^handlers::system_channels::tests::/) + test(/^nip11::tests::/) + test(/^storage_sweep::tests::/) + test(/^nip_fi_core::tests::/) + test(/^nip_fi_http::tests::/) + test(/^nip_fi_config::tests::/) + test(/^nip_fi_resolver::tests::/) + test(/^readiness::tests::/) + test(/^router::tests::/) + test(/^api::parse_query_tests::/) + test(/^api::profiles::/) + test(/^api::bridge::search_slots::tests::/) + test(/^api::git::transport::off_mode_precedence_tests::/) + test(/^audio::join::tests::/) + test(/^audio::handler::tests::/) + test(/^nip_fi_gate::tests::/) + test(/^nip_fi_session::tests::/) + test(/^nip_fi_shadow::tests::/) + test(/^nip_fi_shadow_session::tests::/) + test(/^startup_steps::tests::/) + test(/^mesh_boot::tests::/) + test(/^telemetry::tests::/) + test(/^config::tests::/) + test(=state::tests::accessible_channel_invalidation_is_scoped_to_community) + test(=state::tests::accessible_channels_join_member_and_open_halves) + test(=state::tests::channel_deleted_invalidation_is_scoped_to_community) + test(=state::tests::neither_a_confirmed_inactive_community_nor_a_failed_lookup_admits_the_socket) + test(=state::tests::periodic_revalidation_disconnects_inside_the_fenced_callback) + test(=state::tests::bare_community_disconnect_fails_closed_when_the_fence_is_unavailable) + test(=connection::tests::send_loop_sends_policy_close_when_community_is_archived) + test(=state::tests::community_disconnect_then_nip_fi_keeps_community_deleted_reason) + test(=state::tests::disconnect_community_wins_reason_losing_nip_fi_does_not_enqueue_frame) + test(=handlers::req::tests::timed_out_historical_read_deregisters_before_closed) + test(=handlers::req::tests::superseded_timeout_leaves_replacement_intact) + test(=handlers::req::tests::search_claim_retires_live_and_yields_to_replacement) + test(=handlers::req::tests::failed_search_closes_without_eose_or_trying_later_filters) + test(=handlers::req::tests::concurrent_claims_and_stale_teardowns_keep_the_last_owner) + test(=handlers::req::tests::timeout_closed_is_emitted_before_a_replacement_can_claim) + test(=handlers::req::tests::revoke_then_replacement_keeps_replacement_whole) + test(=handlers::req::tests::claims_after_connection_cleanup_are_refused) + test(=handlers::req::tests::dropped_terminal_frame_cancels_connection) + test(=handlers::req::tests::revoke_dropped_terminal_frame_cancels_connection) + test(=handlers::req::tests::count_fallback_covers_filters_that_can_match_private_deletions) + test(=handlers::req::tests::group_state_t_and_creator_tags_reach_sql_only_for_group_state_kinds) + test(=api::bridge::postgres_tests::bridge_search_mode_extension_accepts_prefix_snake_or_camel_case) + test(=api::bridge::postgres_tests::bridge_search_sort_extension_selects_newest_only_when_named) + test(=api::invites::postgres_tests::restriction_tests::postgres_tests::destructive_fixture_rejects_shared_database_urls) + (kind(bin) & (test(/^tests::/) + test(/^composition_tests::/) + test(/^env_filter_tests::/)) - test(/^tests::postgres_tests::/))' # Note on audio::join::tests scope: the full suite is infra-free (no DB, # no Redis). The infra-free audio/FI regression witnesses — bootstrap # ordering barrier, CommitConfirmed arm, pending-close invisibility, @@ -539,7 +565,9 @@ test-unit: # DB-backed audio join tests use #[ignore] and run in the postgres lane. # NIP-FI (S3/S4) relay witnesses: the wholly-new nip_fi_upgrade and # api::nip_fi modules, - # the auth metrics contract module, plus the exact NIP-FI tests added, + # the auth metrics contract module, the DB-free partition alert series + # tests (their Postgres case is #[ignore]d and runs in the PostgreSQL + # lane), plus the exact NIP-FI tests added, # or whose assertions changed, in mixed modules (audio::room, # connection, handlers::*, state). nip_fi_config and router are # selected whole by the command above. Mixed modules are listed by exact @@ -550,6 +578,7 @@ test-unit: cargo nextest run -p buzz-relay --lib -E ' test(/^nip_fi_upgrade::/) + test(/^metrics::contract_tests::/) + + test(/^metrics::partition_alert_tests::/) + test(=audio::room::tests::roster_revisions_are_ordered_and_snapshot_is_authoritative) + test(=connection::tests::auth_lifecycle_reconciles_every_terminal_and_never_leaks_gauge) + test(=audio::room::tests::b1_pending_peer_removed_before_commit_emits_no_delta) @@ -567,12 +596,26 @@ test-unit: + test(=connection::tests::expiry_notice_queued_on_ctrl_before_cancel) + test(=connection::tests::f3_root_outer_wrapper_delivers_denial_on_bootstrap_cancellation) + test(=connection::tests::f3_root_pre_built_expired_gate_terminates_connection) + + test(=connection::terminal_close_delivery_tests::key_pairing_mismatch_closes_with_1008_on_real_socket) + + test(=connection::terminal_close_delivery_tests::session_expiry_denial_closes_with_1008_on_real_socket) + + test(=connection::terminal_close_delivery_tests::server_holds_socket_for_client_close_reply) + + test(=connection::terminal_close_delivery_tests::server_finishes_promptly_when_client_answers_close) + + test(=connection::terminal_close_delivery_tests::stalled_cleanup_does_not_hold_socket_open) + + test(=connection::terminal_close_delivery_tests::bootstrap_drain_holds_socket_for_client_close_reply) + + test(=connection::terminal_close_delivery_tests::stalled_expiry_barrier_does_not_hold_socket_open) + + test(=connection::terminal_close_delivery_tests::blocked_frame_handler_does_not_hold_socket_open) + + test(=connection::terminal_close_delivery_tests::refused_bootstrap_holds_connection_slot_until_socket_ends) + + test(=connection::terminal_close_delivery_tests::stalled_bootstrap_holds_connection_slot) + + test(=connection::terminal_close_delivery_tests::peer_wait_replies_to_client_initiated_close) + test(=handlers::auth::tests::b2_pre_cancelled_connection_never_becomes_authenticated) + test(=handlers::auth::tests::fi_ban_check_error_emits_terminal_authorization_unavailable) + test(=handlers::auth::tests::fi_root_authorization_denied_rows_emit_identical_frames) + test(=handlers::auth::tests::fi_invalid_nip42_proof_emits_terminal_evidence_rejected) + test(=handlers::auth::tests::handle_auth_pairing_mismatch_runs_full_root_denial_path) + test(=handlers::auth::tests::shadow_root_auth_records_pairing_denial) + + test(=handlers::auth::tests::shadow_root_auth_records_unavailable_owner_read) + + test(=handlers::ingest::postgres_tests::relay_admin_ban_maps_to_blocked_auth_failure) + + test(=handlers::ingest::postgres_tests::relay_admin_internal_maps_to_error_not_client_fault) + test(=handlers::auth::tests::shadow_root_invalid_nip42_retires_without_record) + test(=handlers::auth::tests::nip42_denial_class_separates_internal_failure_from_bad_evidence) + test(=handlers::event::tests::fanout_access::owner_only_kinds_keep_only_the_owner) @@ -588,14 +631,26 @@ test-unit: + test(=state::tests::conn_manager_disconnect_nip_fi_is_issuer_scoped) + test(=state::tests::conn_manager_disconnect_nip_fi_sets_authorization_denied_reason) + test(=state::tests::nip_fi_disconnect_audio_is_issuer_scoped) + + test(=state::tests::nip_fi_owner_disconnect_closes_owned_agent_sockets_on_root_and_audio) + test(=state::tests::nip_fi_disconnect_closes_proven_audio_socket_and_sends_policy_close_reason) + test(=state::tests::nip_fi_disconnect_closes_target_audio_only_and_preserves_collocated_peer) + test(=state::tests::nip_fi_disconnect_does_not_close_different_pubkey_audio_socket) + test(=state::tests::nip_fi_disconnect_does_not_close_unproven_audio_socket) + test(=state::tests::community_disconnect_then_nip_fi_keeps_community_deleted_reason) + + test(=state::tests::enforce_end_is_recorded_only_by_the_winning_terminal) + test(=state::tests::disconnect_community_wins_reason_losing_nip_fi_does_not_enqueue_frame) + test(=state::tests::manager_disconnect_sets_reason_enqueues_frame_then_cancels) + test(/^api::nip_fi::/)' + # Invitation drain/cutoff boundary contracts are infra-free. + cargo nextest run -p buzz-relay --lib -E 'test(/^invite_token::/) + test(/^config::invite_cutoff_tests::/)' + # Relay read caches: the admitted-connection index, the shared-read + # gates (open set, restriction, tenant host), tenant host cache, the + # overlong-host rejection and the ban disconnect's restriction-cache + # drop. Infra-free; the + # database URL refuses at once. Keep scripts/run-tests.sh in step. + cargo nextest run -p buzz-relay --lib -E 'test(=state::tests::admitted_index_follows_every_key_change) + test(/^shared_reads::tests::/) + test(=state::tests::concurrent_open_channel_misses_share_the_in_flight_read) + test(=state::tests::concurrent_restriction_and_host_misses_share_the_in_flight_read) + test(=tenant::tests::app_state_resolver_serves_fresh_cached_hosts_only) + test(=tenant::tests::an_overlong_host_fails_closed_before_the_resolver) + test(=state::tests::disconnect_drops_the_cached_restriction_row)' + # Retain partially consumed mesh receives across housekeeping ticks. + cargo nextest run -p buzz-relay --lib -E 'test(=mesh_boot::tests::demo_echo_retains_pending_receive_across_housekeeping_ticks)' # boot_lifecycle spawns the real relay binary and asserts its startup # lifecycle, including that buzz_startup_phase_* reaches /metrics. Its # non-ignored tests need no Postgres or Redis; the Postgres cases are @@ -606,6 +661,9 @@ test-unit: # relay events and agent prompts. They are infra-free; ignored lifecycle # tests remain excluded and run in their dedicated integration lanes. cargo nextest run -p buzz-acp --lib + # Load generator scenario, identity, shard and stats unit tests; all + # infra-free. Keep scripts/run-tests.sh in step. + cargo nextest run -p buzz-test-client --lib -E 'test(/^loadgen::/)' else ./scripts/run-tests.sh unit fi @@ -884,7 +942,7 @@ staging *ARGS: bootstrap _ensure-sidecar-stubs chmod +x "desktop/src-tauri/binaries/${bin}-${TARGET}" done cd {{desktop_dir}} - export BUZZ_RELAY_URL="wss://buzz.test.blockstaging.build" + export BUZZ_RELAY_URL="wss://buzz.block.builderlabstaging.xyz" source ../scripts/instance-env.sh # Ctrl+C kills the Tauri app before its in-process sweep finishes, leaking # agent workers. Reap this instance's agents on exit as a backstop. @@ -997,6 +1055,7 @@ mobile-test: /bin/bash ./scripts/test-mobile-gateway-recipes.sh unset GIT_DIR GIT_WORK_TREE; cd {{mobile_dir}} && flutter test --dart-define=BUZZ_PUSH_GATEWAY_URL=https://push.example unset GIT_DIR GIT_WORK_TREE; cd {{mobile_dir}} && flutter test test/shared/push/push_unconfigured_build_test.dart + unset GIT_DIR GIT_WORK_TREE; cd {{mobile_dir}}/packages/buzz_secp256k1 && dart pub get && dart test # Regenerate the emoji dataset asset from desktop's emoji-mart install. # Output is committed — rerun after bumping @emoji-mart/data. @@ -1020,10 +1079,14 @@ mobile-build-android: mobile-dev: #!/usr/bin/env bash set -euo pipefail - if ! pgrep -x Simulator &>/dev/null; then - open -a Simulator - sleep 3 + developer_dir="$(xcode-select -p)" + simulator_app="$developer_dir/../Applications/DeviceHub.app" + if [[ ! -d "$simulator_app" ]]; then + simulator_app="$developer_dir/Applications/Simulator.app" fi + # Use the selected Xcode even when another version's app is already running. + open -a "$simulator_app" + sleep 3 ./scripts/mobile-worktree-overrides.sh gateway_url="${BUZZ_PUSH_GATEWAY_URL:-}" overrides_file="{{mobile_dir}}/ios/Flutter/AppOverrides.xcconfig" diff --git a/NOSTR.md b/NOSTR.md index cce70f2f77f..b3d628786f4 100644 --- a/NOSTR.md +++ b/NOSTR.md @@ -50,27 +50,29 @@ PGPASSWORD=buzz_dev psql -h localhost -U buzz -d buzz -c \ | **Reactions (kind:7)** | ✅ | Standard NIP-25; channel derived from target event's `#e` tag (client `#h` ignored) | | **Deletions (kind:5)** | ✅ | Standard NIP-09; self-authored only. `#h` optional, `#e` required | | **User profiles (kind:0)** | ✅ | NIP-01 metadata; synced to users table (display_name, avatar, about, NIP-05). NIP-05 handles must canonicalize to this relay's domain — off-domain or invalid handles are silently cleared. If a NIP-05 handle collides with another user's (UNIQUE constraint), the handle is skipped but other profile fields (display_name, avatar, about) are still synced. | -| **Group creation (kind:9007)** | ✅ | NIP-29; include `name` tag, optional `visibility` and `channel_type` | -| **Add user (kind:9000)** | ✅ | Open: any user, subject to target's `channel_add_policy` (`owner_only`/`nobody` can block). Private: owner/admin only. Self-add bypasses agent policy but not private-channel auth. | +| **Group creation (kind:9007)** | ✅ | NIP-29; include an `h` tag with the new channel id and a `name` tag, optional `visibility`, `channel_type`, label `t` and `unlisted` tags. The relay refuses a 9007 without `h`. It refuses a listed channel (open, not unlisted, stream or forum) whose name another listed channel already has, ignoring case and including archived ones: `invalid: an open channel named "" already exists`. | +| **Add user (kind:9000)** | ✅ | Open: any user, subject to target's `channel_add_policy` (`owner_only`/`nobody` can block). Private: owner/admin only. Self-add bypasses agent policy but not private-channel auth. A guest cannot add anyone. A new member with no role tag gets the channel's `join_role`; a non-member adding themselves may ask for `join_role` or a lower role. One event may name up to 100 people: one `p` tag each, with an optional role as the `p` tag's third element (NIP-29), falling back to the event's `role` tag. Each person is checked and added on their own. When the event names several people, the OK message is `members: ` followed by JSON listing the non-empty `added`, `updated` (role changed), `already_member` and `rejected` (`[{"reason", "pubkeys"}]`) results. The event is refused only when it is malformed or nobody it names may be added; then one person keeps the single-add error, and several get `invalid: members: {…}`. The event and its membership changes are stored together, so an `error:` rejection stores nothing and the same event can be resent; a resent stored event returns `duplicate:` without results. Re-adding someone at their current role posts no second `member_joined` notice. | | **Remove user (kind:9001)** | ✅ | Self-remove allowed (with last-owner guard). Removing others: owner/admin only. | -| **Edit group metadata (kind:9002)** | ✅ | `name`/`about` tags: owner/admin. `topic`/`purpose` tags: any member. | +| **Edit group metadata (kind:9002)** | ✅ | `name`/`about`/label `t`/`visibility`/`ttl`/`archived`/`join_role`/`unlisted` tags: owner/admin. `topic`/`purpose` tags: any member except a guest. `auto_join` tag: relay owner/admin (see [Default channels](#default-channels)). A relay operator may also send a 9002 that changes only `name` on a listed channel, archived ones included, whatever its channel role, so that duplicate names can be repaired; the channel stays archived. The relay refuses a `name`, `visibility` or `unlisted` change that leaves a listed channel with a name another listed channel already has: `invalid: an open channel named "" already exists`. Buzz applies 9002 as a partial update: an absent tag leaves that setting unchanged. | | **Admin delete event (kind:9005)** | ✅ | Event author can always delete own. Otherwise owner/admin required. Target must be in same channel. | | **Group deletion (kind:9008)** | ✅ | Owner only. | | **Leave group (kind:9022)** | ✅ | Any member. Last-owner guard prevents orphaned groups. | -| **Group metadata (kind:39000)** | ✅ | Relay-signed; always `d`, `name`, `closed` tags; `about` only if description non-empty; `private` if applicable; `hidden` for DM channels | -| **Group admins (kind:39001)** | ✅ | Relay-signed; `d` tag + `p` tags with roles (`owner`, `admin`) | -| **Group members (kind:39002)** | ✅ | Relay-signed; `d` tag + `p` tags for all members | +| **Group metadata (kind:39000)** | ✅ | Relay-signed; always `d`, `name`, `t`, `P`, `created_at`, `join_role`; `public` or `private`; `restricted` and `closed` per the access flags below; `unlisted` when the channel is unlisted; `about` only if description non-empty; `hidden` for DM channels and private system channels; `auto_join` while the channel is a default channel | +| **Unlisted channels** | ✅ | kind:9002 `["unlisted", "true"\|"false"]`, owner/admin, or at creation on kind:9007. An unlisted open channel is left out of every read that does not name it, for non-members: channel lists, kind:39000 without `#d`, messages, COUNT, NIP-50 search and artifact queries. Naming it by `#h` (or `#d` on 39000–39003) reads it, and kind:9021 joins it, as for any open channel. See [Unlisted channels](#unlisted-channels). | +| **Group admins (kind:39001)** | ✅ | Relay-signed; `d`, channel type `t` and creator `P` tags + `p` tags with roles (`owner`, `admin`) | +| **Group members (kind:39002)** | ✅ | Relay-signed; `d`, channel type `t` and creator `P` tags + `p` tags for all members | | **Membership notifications** | ✅ | kind:44100 (added) / kind:44101 (removed); relay-signed, community-global scope (`channel_id=None` inside the connected community) | | **Presence (kind:20001)** | ✅ | Ephemeral; arbitrary status string (truncated to 128 chars); writes to Redis (`set_presence`/`clear_presence` on `"offline"`), then fan-out to local subscribers. In multi-community mode presence is scoped to the connected community. | +| **Agent presence (kind:20003)** | ✅ | Ephemeral; an owner's presence for the agents it owns, named by `p` tags (at most 256); `online`/`away` set and `offline` clears each agent's presence as its own kind:20001 would. Not fanned out. | | **Typing indicators (kind:20002)** | ✅ | Ephemeral, not stored; published via Redis pub/sub (multi-node capable unlike presence fan-out) | | **NIP-42 authentication** | ✅ | Proactive challenge; optional pubkey allowlist | | **NIP-11 relay info** | ✅ | `GET /` with `Accept: application/nostr+json` | | **Blossom media** | ✅ | `PUT /media/upload` (BUD-02), `GET /media/{sha256}.{ext}` (BUD-01) | -| **NIP-50 search** | ✅ | One-shot search REQs: `{"search":"query","kinds":[9],"#h":[""]}` → relevance-sorted results → EOSE. Not registered as persistent subscriptions. | +| **NIP-50 search** | ✅ | One-shot search REQs: `{"search":"query","kinds":[9],"#h":[""]}` → relevance-sorted results → EOSE. Not registered as persistent subscriptions. HTTP `POST /query` also accepts `search_sort: "newest"` (advertised as `buzz-search-sort` in NIP-11 `supported_extensions`) for newest-first results, paged with `page`. | | **NIP-10 threads** | ✅ | WS-submitted replies with `["e","","","reply"]` tags create `thread_metadata` atomically. Visible in REST thread queries. Unknown parents rejected. | | **NIP-17 DMs (gift wrap)** | ✅ | kind:1059 accepted with ephemeral signing keys. Stored community-globally (`channel_id=None` inside the connected community). Delivered via `#p`-filtered subscriptions. Not indexed in search. | | **DM discovery** | ✅ | DM creation emits kind:39000 (with `hidden` tag) + kind:44100 membership notifications. NIP-29 clients discover DMs via standard group discovery flow. | -| **Join request (kind:9021)** | ✅ | Open channels only. Adds member, emits system message + group discovery events + kind:44100 membership notification. Private channels rejected at ingest. | +| **Join request (kind:9021)** | ✅ | Open channels only. Adds the joiner with the channel's `join_role` (`member` by default), emits system message + group discovery events + kind:44100 membership notification. Private channels rejected at ingest. The same body joins new community members to [default channels](#default-channels). | | **Edits (kind:40003)** | ⚠️ | Works on the wire but Buzz-only — no standard NIP-29 client renders these | | **Rich content (kind:40002)** | ⚠️ | Works on the wire but Buzz-only — no standard NIP-29 client renders these | @@ -99,16 +101,220 @@ relay to specific external Nostr identities without granting full access. SELECT encode(pubkey, 'hex'), added_at, note FROM pubkey_allowlist; ``` +### Announce channels + +Roles have the same permissions in every channel. A channel's settings +decide only who can join and which role a new person gets: + +- `visibility` (`open` | `private`) decides who can find, join and read. +- `join_role` (`member` | `guest`) is the role a person gets when nobody + picks one: a self-join (kind:9021, or a kind:9000 self-add), or a kind:9000 + add with no `role` tag. The default, `member`, keeps today's behavior. + +An open channel with `join_role = guest` is an **announce channel**: anyone +can find, read and join it, but people who join are guests and cannot write. + +What each role can do: + +| Action | Guest | Member or bot | Admin / owner | +|--------|-------|---------------|---------------| +| Read, leave | Yes | Yes | Yes | +| Post, reply, edit, delete own, react, artifacts, canvas, forum, pins, typing, workflow messages, huddles | No | Yes | Yes | +| Topic, purpose (kind:9002) | No | Yes | Yes | +| Add people (kind:9000) | No | Yes, as member, guest or bot | Yes, any role | +| Push to the channel's git repositories | No | Yes | Yes | +| Name, about, visibility, TTL, `join_role`, `unlisted` | No | No | Yes | + +A non-member may write only where joining would give a role that can write: +an open channel with `join_role = member`. A refused write gets +`restricted: only members can post in this channel`. + +Other rules: + +- A non-member adding themselves with kind:9000 may ask for `join_role` or a + lower role. A higher role is refused. +- Any member who can write (member, bot, owner or admin) can make a guest a + member with kind:9000 (`role` `member`). This is the same for a person and + an agent. Only owners and admins can make any other role change. +- The human owner of an agent can edit (kind:40003) and delete (kind:9005) + that agent's messages, also as a guest or a non-member. This is existing + behavior. It lets a person correct or remove what their agent posted; it + does not let them post. +- Owners and admins change the setting with kind:9002 + `["join_role", "guest"]` or `["join_role", "member"]`. A missing tag means + "no change". Changing it does not change anyone's role: existing members + can still write after a change to `guest`, and existing guests stay + read-only after a change to `member`. A stored value the relay does not + know is treated as `guest`. + +kind:39000 publishes the NIP-29 access flags from these settings, and an +explicit `["join_role", ...]` tag so clients can tell what joining gives: + +| `visibility` | `join_role` | `private` | `public` | `restricted` | `closed` | +|--------------|-------------|-----------|----------|--------------|----------| +| open | `member` | | ✓ | | | +| open | `guest` | | ✓ | ✓ | | +| private | either | ✓ | | ✓ | ✓ | + +### Unlisted channels + +An **unlisted** open channel works like a link-shared document: anyone in +the community who has its id (for example from a shared link) can read it +and join it, but it does not show up for people who are not members. + +- Owners and admins set it with kind:9002 `["unlisted", "true"]` and clear + it with `["unlisted", "false"]`. A missing tag means "no change". It can be + sent in the same event as `["visibility", "open"]`, so a private channel + becomes open and unlisted in one step. +- kind:9007 also takes `["unlisted", "true"|"false"]`, so a channel can + start unlisted; a missing tag means listed. Any other value is refused + before the event is stored, as on kind:9002. +- For a non-member, an unlisted open channel, of any type, is left out of + every read that does not name it: a filter without `#h` (including + kind:39000 discovery without `#d`, and the system-channel opt-in by `#P` + or `#t:["system"]`), COUNT, NIP-50 search, artifact queries and the + channel list. A filter that names it by `#h`, or by `#d` + on a filter whose kinds are all 39000–39003, reads it under the ordinary + open-channel rules. This is the same "named-only" rule that applies to + system channels ([NIP-CL](docs/nips/NIP-CL.md)). +- Joining (kind:9021) and posting work as on any open channel; `join_role` + still decides what a joiner gets. +- Members see it everywhere, as before. +- The flag is stored on its own and matters only while the channel is open. + Private channels are already invisible to non-members. A private channel + can be marked unlisted ahead of time, so opening it later never lists it. +- kind:39000 carries a bare `["unlisted"]` flag whenever the setting is on, + whatever the visibility. It does not add NIP-29 `hidden`: the metadata + stays readable by id. +- Relays that support it advertise `buzz-unlisted-channels` in NIP-11 + `supported_extensions`. An older relay applies `["visibility", "open"]` + and ignores `unlisted`, so a client must check for it before opening a + channel as unlisted. +- Unlisted is not an access control. Anyone with the id can read the channel. + +### Default channels + +A community's owners and admins can mark a channel as a **default channel**: +one the relay joins every new community member into. The setting is +`auto_join` on the channel. + +- Only a *relay* owner or admin (the NIP-43 roster role, not a channel role) + can change it, with kind:9002 `["auto_join", "true"]` or + `["auto_join", "false"]`. The actor need not be a member of the channel. + A missing tag means "no change". +- It is accepted only on a `stream` channel the event leaves open, and `true` + also needs it unarchived. `false` is refused on a private channel too: the + actor need not be a member, so neither value may write into a channel they + cannot read. The relay decides this in the same transaction as the other + kind:9002 changes, so one event may open a channel and flag it. Making a + flagged channel private or archiving it is allowed and leaves the stored + value in place; to take a private channel out of the defaults, clear the + flag before making it private or once it is open again. +- kind:39000 carries a bare `["auto_join"]` tag only while the setting is + effective: set, open and unarchived. What the event says is what the relay + does, so a client never has to combine the flag with the access flags. +- Nothing is a default channel until someone asks for it: new channels and + existing channels start with `auto_join` off. `buzz channels update + --channel --auto-join true` sets it. + +Every change is announced by one kind:40099 `auto_join_changed` system +message in the channel. + +When someone becomes a community member (an invite claim or a kind:9030 add) +the relay joins them to every effective default channel as a self-join: the +body of kind:9021, so the channel's `join_role` applies and the channel gets +its `member_joined` kind:40099, kind:39002 and kind:44100. There is no cap on +how many channels that is: whatever carries the kind:39000 tag is joined, and +the work a new member costs grows with the number of channels an owner or +admin flags. On a community that requires relay membership, the relay does +this once, at that moment, and never again on login or reconnect, so a later +kind:9022 leave stands. A kind:9030 add is +made by someone else, so the new member's `channel_add_policy` applies first, +as it would to that actor's kind:9000: if it refuses them, no default channel +is joined. The claim response waits for the joins, and a claimer who +disconnects before it arrives does not cut them short. A channel that fails +to join, or an add the member's policy refuses, is logged and counted +(`buzz_auto_join_failures_total`), not retried: the kind:39000 tag is the +record a client reconciles against. That includes a channel archived or made +private while the join runs: the membership write decides both again under +the channel's locks, so the member is not added and nothing is announced. +`buzz-admin add-member` writes the roster row only and +does not join default channels. + +A community that does not require relay membership has no roster insert to +join people at, so there the relay joins a person when it admits their +WebSocket AUTH, after the OK, to every effective default channel they have +never been a member of. "Never a member" is the channel's own membership +history: a kind:9022 leave or a kind:9001 removal keeps the row, so the +person is not joined back, and a new device or another relay pod finds the +row and does nothing. The check runs again under the channel's membership +lock, so two admissions of the same person join them once. A channel flagged +later is joined on each person's next admission. Agents are skipped: an AUTH +with a NIP-OA owner, a NIP-FI agent token, or a stored agent owner joins +nothing. A membership write that fails leaves no row, so the next admission +tries that channel again. + ### Group Discovery The relay emits NIP-29 group state events when channels are created, updated, or membership changes. +[NIP-CL](docs/nips/NIP-CL.md) defines the channel identity tags on these events and how clients +use them to find channels. All discovery events include a `d` tag set to the channel UUID (NIP-29 addressable event convention): | Kind | Tags | Content | |------|------|---------| -| **39000** | `d=`, `name`, `closed` (always); `about` (if description non-empty); `private` (if applicable); `hidden` (DM channels only) | Group metadata. **Note:** `closed` is always emitted per NIP-29 convention (Buzz channels require explicit membership), but open channels are still readable/writable by non-members at runtime. The tag reflects the membership model, not access enforcement. | -| **39001** | `d=`, `p` tags with role label (`owner`, `admin`) | Admin list | -| **39002** | `d=`, `p` tags for all members | Member list | +| **39000** | `d=`, `name`, `t=`, `P=`, `created_at=`, `join_role` (always); `public` or `private`; `restricted` (private channels, and `join_role=guest`); `closed` (private channels only); `unlisted` (unlisted channels); `about` (if description non-empty); `hidden` (DM channels and private system channels); `auto_join` (open, unarchived channels with the setting on, see [Default channels](#default-channels)) | Group metadata. The access flags follow the channel settings (see [Announce channels](#announce-channels)). | +| **39001** | `d=`, `t=`, `P=`, `p` tags with role label (`owner`, `admin`) | Admin list | +| **39002** | `d=`, `t=`, `P=`, `p` tags for all members | Member list | + +On all three kinds the channel type `t` tag comes first, then one `t` tag +per channel label, then `P`. Clients read the first `t` value as the +channel type. `P` is +the public key that signed the channel's kind:9007. The relay writes it +from the database, and it does not change when ownership moves. Uppercase +`P` follows NIP-22, NIP-34 and NIP-72, where it names the author of the root +object; lowercase `p` already lists members and DM participants on these +kinds. + +kind:39000 also carries `["created_at", ]`: the time the +channel was created. The event's own `created_at` is the time of the last +edit, so use the tag to find the older of two channels. + +**Channel labels.** A label groups channels across channel types, for +example `["t", "workspace"]`. Set labels with `["t",
.sql` | The table's final `CREATE TABLE` and its `CREATE INDEX` statements, one table per file. This is the declarative schema source. | +| `schema/schema.sql` | An `\i` line for each new file, after the tables it depends on. | +| `schema/types/`, `schema/partitions/`, `schema/seeds/` | The matching non-table objects. The layout test permits nothing else here. Rule 1 forbids any addition to `functions/` and `triggers/`. | +| `scripts/reconcile-schema-after-pgschema.sql` | Idempotent convergence, plus a live assertion, for anything `pgschema` does not preserve: seed rows, some storage parameters, partition attachment, and the shapes listed in AGENTS.md *Common Gotchas* #7. | +| `crates/buzz-push-gateway/migrations/` **and** the relay copies | The six `push_gateway_*` tables have two homes. The gateway applies its own migration history to its own database, through a pre-install/pre-upgrade Helm Job ([push-gateway-deployment.md](push-gateway-deployment.md)). The relay schema also defines them: `migrations/` (`0015`, `0043`, `0045`, `0060`), `schema/tables/public/push_gateway_*.sql`, and `schema/seeds/push_gateway_*.sql`. Until the duplication is removed, a gateway schema change goes in both, as relay `0060` mirrors gateway `0006`. Every rule here applies to both. The target is for the gateway to own its tables alone, in its own migrations and declarative schema; do not add new gateway tables to the relay schema. | + +A fresh desired-state database and a migrated database MUST converge on the +same live catalog. The desired-state layout test in +`crates/buzz-db/src/runtime/migration.rs` checks file structure only. Parity +tests cover selected surfaces, and the full drift gate does not exist yet. For +tables and indexes you touch, add or extend a parity assertion that compares +the live catalog from both bootstrap paths, following +`admin_schema_parity_between_desired_state_and_migrations`. A string match +against `schema.sql` does not count. + +The existing migration lints still apply. Every tenant-scoped table has +`community_id NOT NULL`, every key on a scoped table leads with `community_id`, +and operator-global tables are registered in all three places listed in the +provenance-only checklist under Rule 1. + +### 5. Every schema change is backward compatible with the running app + +Buzz supports two ways of applying migrations, and every change MUST be safe +under both: + +- **Auto-migration.** The relay binary defaults `BUZZ_AUTO_MIGRATE` to off, + but the Helm chart defaults `migrate.autoMigrate` to `true`. With it on, the + first new pod migrates at boot while old pods are still serving. +- **Out of band.** An operator applies the migrations while the current relay + keeps serving traffic, then deploys the new relay. Block's hosted + deployment works this way at scale, with `migrate.autoMigrate=false`. + +Either way, for a period the **old relay runs against the new schema**. If the +new relay is rolled back, the old relay runs against the new schema again. +Every migration MUST be safe for the currently deployed release to read and +write against. + +The relay has no general schema-version compatibility check. It verifies only +specific catalog surfaces, such as the deletion serving fence and the replica +floor guard, so the ordering below is the main protection. When +`BUZZ_AUTO_MIGRATE` is on, an older binary refuses a database that has +migrations it does not know (sqlx `VersionMissing`), so `helm rollback` with +auto-migration on fails at startup, and so does any old pod that restarts +mid-rollout after a new pod has migrated. A rollback therefore depends on the +older binary not running the migrator. Making rollback work under +auto-migration is part of supporting both modes. + +One feature therefore usually ships as several PRs, each deployed before the +next starts: + +1. **Expand (schema).** Add only things the old relay ignores: nullable columns, + columns with a constant default, new tables, new indexes. Add nothing the old + relay could violate. +2. **Deploy (code).** Ship the relay that writes the new shape on every write + path and reads both old and new rows correctly. +3. **Backfill (data).** After the new code is fully rolled out and old writers + are drained, repair historical rows with a backfill (Rule 3). Readers stay + correct for both repaired and unrepaired rows throughout. +4. **Contract (schema, then code).** After the backfill completes and + validates, add the constraints you wanted from the start: `NOT NULL`, + `CHECK`, `UNIQUE`. When nothing deployed still reads them, drop the old + columns, indexes, triggers, or functions. Each tightening is its own + migration, applied after the release that stops depending on the old shape. + +Skip a step only when it has nothing to do. For example, a new table with no +historical data needs no backfill. + +**Under auto-migration the steps can collapse.** The migrator applies every +pending migration at one boot, before the new code serves and before any +backfill or repair script runs. An operator who upgrades across several +releases therefore applies expand and contract together. How auto-migration +should order such an upgrade is unresolved. Until it is, a contract depends on +one or both of these, and each is handled differently: + +- **Data (a backfill or repair has run).** The contract migration MUST fail + closed when it has not. `VALIDATE CONSTRAINT` and `SET NOT NULL` already + fail on unrepaired rows. A contract that drops a column or table, or + otherwise loses data, MUST first check that the backfill or repair has + completed, and `RAISE` from a top-level `DO` block when unrepaired rows + remain. The PR MUST add a Postgres-gated test that applies the expand and + contract migrations in one run against unrepaired rows and asserts the + migration errors without dropping data. +- **Writers (a release that changes how the app writes is fully deployed).** + Dropping a trigger, function, or foreign key whose work moved into the app, + and adding a constraint that old writers could violate, depend on a release, + not on rows. No row check can prove it: the data may already be valid, or + there may be nothing to backfill, while old writers still serve. + Auto-migration does not protect this case. + +For either kind, the release notes MUST name the release to stop at (the one +that ships the writer) and any repair to run there, before upgrading past the +contract release. A failed data check stops the relay from starting. That is +an outage, not data loss, but recovery is not clean: the expand migrations +before the contract have already committed, so an old pod that restarts hits +`VersionMissing`. + +#### What breaks the old relay + +| Change | Why it breaks | Safe form | +|---|---|---| +| Add a `NOT NULL` column without a default | Old `INSERT`s omit it and fail | Add it nullable, deploy writers, backfill, then add the constraint (see below) | +| Rename a column or table | Old queries reference the old name | Add the new name, dual-write, backfill, move readers, then drop the old name | +| Change a column type | Old binds and decodes mismatch, and the change may rewrite the table | Add a new column and migrate through the expand/contract sequence | +| Add `CHECK`, `UNIQUE`, or a narrower index | Old writes violate it, or an old `ON CONFLICT` target no longer matches an index | Deploy writers that satisfy it, backfill, then add it. Add a `CHECK` as `NOT VALID`, then `VALIDATE CONSTRAINT` in a later step. Keep every index an old `ON CONFLICT` names until no deployed code uses it | +| Add an enum value that new code writes | Old code fails to decode the unknown variant | Deploy readers that accept the value, then deploy writers that produce it | +| Drop a column, table, index, trigger, function, or FK | Old code reads it or relies on its side effect | Contract step only, after the release that stops relying on it is fully deployed | +| Add a table with a `community_id` column | The old relay's exact-set deletion catalog check (`EXPECTED_SCOPED_TABLES`) sees an unknown table, so community deletion fails closed until the new release deploys. Serving is unaffected, apart from the auto-migration restart case in Rule 5 | Allowed. For a tenant-owned table, only after the desired-state change in Rule 1. Keep the gap between expand and deploy short, and do not run community deletion during it | +| `SELECT *` or `RETURNING *` decoded by position | Column order differs between desired-state and migrated databases and changes when columns are added | Select named columns | + +#### Adding `NOT NULL` to a populated table + +`ALTER TABLE ... SET NOT NULL` scans the whole table under `ACCESS EXCLUSIVE`. +Instead, ship three migration files, each its own step. sqlx runs each file in +a transaction, which `SET LOCAL` needs; pasted into autocommit `psql`, +`SET LOCAL` only warns and has no effect. + +```sql +-- contract migration A +SET LOCAL lock_timeout = '5s'; +ALTER TABLE t ADD CONSTRAINT t_col_not_null CHECK (col IS NOT NULL) NOT VALID; + +-- contract migration B (separate step; scans but does not block writes) +SET LOCAL lock_timeout = '5s'; +ALTER TABLE t VALIDATE CONSTRAINT t_col_not_null; + +-- contract migration C (PostgreSQL 12+ uses the validated CHECK and skips the scan) +SET LOCAL lock_timeout = '5s'; +ALTER TABLE t ALTER COLUMN col SET NOT NULL; +ALTER TABLE t DROP CONSTRAINT t_col_not_null; +``` + +The desired-state file declares only the final `col ... NOT NULL`, and that +change lands in the same PR as migration C. Until then, the desired state keeps +the column nullable. Between A and C, migrated databases carry +`t_col_not_null` and fresh ones do not, so exclude that constraint from any +parity assertion on the table for that window. + +#### Retiring a trigger, function, or foreign key + +1. Ship application code that does the trigger's work, or the foreign key's + check and cascade, while the database object is still in place. The two + paths MUST be idempotent with each other, so the duplicate work is harmless. +2. After that release is fully deployed, drop the database object in a + separate migration. Remove it from `schema/` in the same PR. The release + notes name the step-1 release as a required stop (Rule 5). +3. Removing a guard requires a falsifiable test showing that the application + path enforces what the guard enforced. + +### 6. Bound every lock and statement + +- Every migration MUST start with `SET LOCAL lock_timeout`, and SHOULD set + `SET LOCAL statement_timeout` when a statement can scan. A migration that + cannot get its lock fails and is retried. It MUST NOT queue behind + traffic on a hot table. +- In `migrations/`, an index on an existing populated table MUST be built + with `CREATE INDEX CONCURRENTLY`, because auto-migrating deployments have no + online schema change tool to build it for them. CONCURRENTLY cannot run + inside the migration transaction, so follow the prebuild-then-verify + procedure in [thread-window-deployment.md](thread-window-deployment.md). + Partitioned parents need per-partition concurrent builds attached to an + `ON ONLY` parent index. +- The declarative schema source declares indexes with plain `CREATE INDEX`. + The tool that applies it out of band decides how to build them online, so + `schema/` files MUST NOT add `CONCURRENTLY`. The desired-state layout test + rejects it. +- A change MUST NOT rewrite or fully scan a large table under a + write-blocking lock. That includes column type changes, volatile defaults, + `SET NOT NULL` without a validated check, and changes to the generated + `events.search_tsv` column. Use the expand/contract forms above. +- Write changes that out-of-band apply can make online: `CREATE TABLE`, + `CREATE INDEX`, and bounded `ALTER TABLE` on tables below a size ceiling, + with short lock and statement budgets. Do not rely on a multi-statement + change applying atomically. A change outside that envelope needs a + separately reviewed operational procedure, and the PR MUST say so. + +### 7. Old and new relays coexist during rollout + +Rolling deploys run two relay versions against one database. Durable records +written by the new release, such as backfill rows, outbox items, and new enum +values, MUST be handled safely by the old one. This means ignoring them, +leaving them for the new release, or failing closed. The old release MUST NOT +misinterpret them. + +## Review checklist + +A PR that touches the schema or a schema-dependent query states in its +description: + +- [ ] Which phase this is (expand, deploy, backfill, or contract), and which + PRs or releases come before and after it. +- [ ] That no trigger, function, procedure, rule, or foreign key is added. +- [ ] That no migration mutates existing rows, and the backfill ID (or, + until the framework ships, the `scripts/maintenance/` script and how it + takes the community deletion lock) for any repair. +- [ ] For a contract migration, how it fails closed when its backfill has not + run, and which writer release the release notes name as a required stop. +- [ ] That every location in Rule 4 is updated, with a catalog-level parity + test for each changed table and index. +- [ ] That the currently deployed relay works against the new schema, and how + that was checked. +- [ ] Lock and statement timeouts, concurrent index builds in `migrations/`, + and any operator prebuild step. + +## Related documents + +- [ARCHITECTURE.md](../ARCHITECTURE.md): *Supported Writer Contract* and the + `buzz-db` crate. +- [data-backfills.md](data-backfills.md): the backfill specification. +- [multi-tenant-conformance.md](multi-tenant-conformance.md): tenant-scoping + obligations. +- [thread-window-deployment.md](thread-window-deployment.md): the brownfield + concurrent-index procedure. +- [crates/buzz-db/TESTING.md](../crates/buzz-db/TESTING.md): desired-state + and migration-mode tests. +- `schema/schema.sql` header: the desired-state layout. diff --git a/docs/formal/nip-pl/delivery.py b/docs/formal/nip-pl/delivery.py index 2c8c5ac3502..39a1ca9d348 100644 --- a/docs/formal/nip-pl/delivery.py +++ b/docs/formal/nip-pl/delivery.py @@ -7,7 +7,7 @@ """ from itertools import permutations, product -FIXED_BODY = b'{"aps":{"alert":{"body":"Reconnect to your relay now"},"mutable-content":1}}' +FIXED_BODY = b'{"aps":{"alert":{"body":"Reconnect to your relay now"},"sound":"default","mutable-content":1}}' class Gateway: def __init__(self): diff --git a/docs/formal/nip-pl/fixed_payload.py b/docs/formal/nip-pl/fixed_payload.py index e69af9c4346..5b424cd21f6 100644 --- a/docs/formal/nip-pl/fixed_payload.py +++ b/docs/formal/nip-pl/fixed_payload.py @@ -5,7 +5,7 @@ """ from itertools import product -C = b'{"aps":{"alert":{"body":"Reconnect to your relay now"},"mutable-content":1}}' +C = b'{"aps":{"alert":{"body":"Reconnect to your relay now"},"sound":"default","mutable-content":1}}' DOMAINS = [ (b"request-a", b"request-b"), # exact signed body (b"auth-a", b"auth-b"), # NIP-98 event/header diff --git a/docs/native-notification-enrichment.md b/docs/native-notification-enrichment.md new file mode 100644 index 00000000000..f4e6c71d567 --- /dev/null +++ b/docs/native-notification-enrichment.md @@ -0,0 +1,75 @@ +# Sender information in iOS notifications + +Buzz must show a sender's name and avatar even when the main app has never +loaded that sender's profile. The notification service extension fetches and +caches the information it needs. The main app supplies signed-in accounts, +relay configuration and credentials in the shared Keychain. It does not export +profiles, channel details or avatar images for notifications. + +## Display behavior + +The extension uses one freshness threshold of one hour for profiles, channel +details, and avatar refresh checks. The period starts at the last successful +check. Reading a record does not extend it. When a record is older, the +extension tries to refresh it and can use the previous record if the refresh +fails. This setting does not extend authorization. Image requests also follow +HTTP caching rules, which can avoid downloading unchanged image data again. + +After it resolves the message, the extension allows up to 1.5 seconds for the +sender and channel details. It then delivers the best available notification. +If no sender name or avatar is available, it omits those values where iOS +supports this. It must not use a public key as the displayed name. + +When a sender changes their avatar, the previous image remains usable until +the replacement arrives. When a verified profile removes the avatar, the old +image is no longer shown. A late response must not restore a removed avatar. + +The extension supports the same avatar sources as the app: Buzz uploads, +external HTTP(S) images and supported inline images. It uses existing server +thumbnails for eligible Buzz uploads. A separate resized-image cache is not +required. Animated avatars use the sender-selected still image. Inline SVGs +are rendered natively. Image decoding has size limits; SVG decoding also limits +nesting and repeated references. Embedded fonts, external SVG resources and +recursive references are rejected. An image that cannot be decoded follows the +same stale-image or missing-image behavior as a failed download. +An empty profile query is not a verified removal and does not clear stale data. + +## Fetching and storage + +The extension uses native networking and its own persistent storage. Image +responses use URLSession and URLCache; verified profiles and channel details +have a separate bounded store. Each account and relay has separate cache data. +Credentials are not stored in these caches. Profile and channel requests run +independently, so a large or slow channel roster cannot block a sender profile +or avatar that is available before the delivery deadline. +When only channel metadata or membership refreshes successfully, the extension +keeps that record and removes its old counterpart in the same write. It does +not derive a group identity or recipient count from a mixed pair. If neither +request succeeds, the previous pair remains available. + +Delivery happens only once. Ordinary requests can continue after delivery if +iOS keeps the process alive. A late profile response can start an avatar fetch, +and completed results can be saved for the next notification. These requests +use normal network timeouts. There are no background sessions, and successful +notification delivery does not depend on work continuing after delivery. + +A cache miss is normal. Interrupted writes must leave complete old or new +records, not partial records. Invalid cache entries are reported and treated +as misses. The extension must work after iOS removes cache files or starts a +new process. + +Removing an account immediately prevents access to its cached information. +The extension deletes the account's cache files when it next runs. Results +from requests started before removal cannot become usable in a new sign-in. + +## Validation + +Tests cover a sender never loaded by the app, cache hits and misses, stale +records, avatar replacement and removal, account removal, late responses and +exactly-once delivery. Storage tests cover concurrent access and interruption. +Device validation checks actual notification appearance, authenticated image +cache reuse, process restarts and notification bursts. In particular, accepting +nil name and image values in an API does not by itself prove how iOS renders +those values. + +Persistent image caching for the main app is separate from this change. diff --git a/docs/nips/NIP-AE.md b/docs/nips/NIP-AE.md index c24078d8a45..906c30e02e1 100644 --- a/docs/nips/NIP-AE.md +++ b/docs/nips/NIP-AE.md @@ -149,6 +149,23 @@ A body MAY reference other slugs using wiki-link syntax: `[[]]`, where `", - "created_at": , - "tags": [ - ["p", ""] // agent subject, or: - // ["t", ""] // team subject - ], - "content": "" -} -``` +A client stores config on the relay only when the relay's [NIP-11](11.md) document shows that it supports both of these: -- The owner authors every version. Agents do not author instructions versions. -- An event MUST carry exactly one **subject tag**: either one `p` tag whose value is the agent's 64-character lowercase hex pubkey, or one `t` tag whose value is a team id (see "Team identity"). A subject tag MUST have exactly two elements. Tags are counted by their first element, so an event with both a `p` and a `t` tag, two `p` tags, two `t` tags, or a valueless `["p"]` or `["t"]` is invalid. -- `t` values are case-sensitive. Clients and relays MUST NOT lowercase them, unlike [NIP-24](24.md) hashtags. -- The event MUST NOT carry an `h` tag; it belongs to no channel. +- NIP-CL system channels and labels; +- NIP-AR artifacts. -Implementations MAY include a [NIP-31](31.md) `["alt", "agent instructions version"]` tag. Other tags are not defined by this NIP and, apart from the prohibited `h` tag, have no effect on validity. +How a relay advertises each one is defined by the specification of that feature. Otherwise the client MUST keep config local, and MUST NOT create config channels. -### Content +### Config channels -`content` MUST be [NIP-44](44.md) v2 ciphertext encrypted with the conversation key of `(seckey_o, pubkey_o)`: the owner encrypts to themselves. +| Channel | One per | Created by | Family label | Subject label | +| --- | --- | --- | --- | --- | +| Agent config channel | agent | the agent's owner | `agent-config` | the agent's 64-character lowercase hex pubkey | +| Team config channel | team | the team's owner | `team-config` | `team-` followed by the first 59 lowercase hex characters of the SHA-256 hash of the team id's UTF-8 bytes | -The plaintext is the three ASCII bytes `v1:` followed by the instructions text in UTF-8. The prefix lets an empty text be encrypted; NIP-44 cannot encrypt an empty plaintext. +The owner creates a config channel with a kind:9007 that carries `["channel_type", "system"]`, both labels as `t` tags, and no `visibility` or `ttl` tag. A config channel MUST be private and MUST NOT have a TTL. The owner MUST NOT change a config channel's labels. -- The text MAY be empty. An empty version means "no instructions" for its subject. -- The text MUST be valid UTF-8, MUST NOT contain U+0000, and MUST NOT exceed 131,072 bytes. -- A plaintext of 65,536 bytes or more uses NIP-44 v2's extended length prefix: two zero bytes followed by the plaintext length as a 32-bit big-endian integer. An implementation that supports only the two-byte prefix cannot decrypt such a version and MUST treat it as invalid, not absent. -- With these bounds, `content` is at most 218,548 bytes. +Both subject labels are 64 characters. A team id can contain characters a label cannot, so the team label is derived from a hash; the team id itself is stored on the `buzz.team` artifact (see "Config artifacts"). -Writers MUST enforce the text rules before encrypting. Readers MUST check, in this order: +An agent config channel's members are the owner, the agent, and any others the owner adds. A team config channel holds the owner and the team's members, and can hold others who are not members (see "Membership"). -1. `content` is at most 218,548 bytes, before decoding or decrypting it; -2. it decrypts under NIP-44 v2; -3. the plaintext starts with `v1:`; -4. the text after the prefix is valid UTF-8, contains no U+0000, and is at most 131,072 bytes. +### Discovery -A version that fails any check is **invalid**. Ciphertext length cannot enforce the text bound by itself: a maximum-length text and a text one byte longer produce `content` of the same length. +To find a config channel, a client queries by creator and subject label: -### Current version and history +``` +Agent: {kinds: [39000], "#P": [pubkey_o], "#t": []} +Team: {kinds: [39000], "#P": [pubkey_o], "#t": []} +``` -A subject's **current version** is its event with the greatest `created_at`; ties are broken by lowest event `id`, as in [NIP-01](01.md). +To find the teams an agent is on, the agent's client queries the channels the agent belongs to, opting in to system channels: ``` -Agent: {kinds: [44300], authors: [pubkey_o], "#p": []} -Team: {kinds: [44300], authors: [pubkey_o], "#t": []} +{kinds: [39002], "#p": [], "#t": ["system"]} ``` -With `limit: 1`, these filters return the current version. Clients query one subject per filter. +then reads those channels' kind:39000 by `#d`. -History is the same filter, ordered by `created_at` descending and then by `id` ascending, and paged with a composite cursor: the `created_at` and `id` of the last event on the previous page, sent as `until` and `before_id`. The next page holds events with `created_at < until`, or with `created_at = until` and `id > before_id`. `before_id` is an extension to the NIP-01 filter, not a standard field. A timestamp-only cursor is not sufficient: concurrent saves on different devices can give several versions of one subject the same `created_at`, and paging by `until` alone either repeats or skips them. A relay MAY return fewer events than `limit`, so a client has read a subject's complete history only when a page returns no events. +In every case the client keeps only channels whose type is `system`, whose labels include the family label, and whose `P` is the expected owner. It MUST NOT put the family label in the same `#t` filter, because values in one filter combine with OR. When more than one channel remains for one agent or team, the client uses the oldest, by NIP-CL's "Choosing one channel" rule. A client creates a config channel only when this query returns none. A client MUST NOT delete or archive a config channel because it was not selected. -If a subject's current version is invalid, clients MUST show the error and MUST NOT substitute an older version, a definition's prompt, or a legacy copy. +### Config artifacts -### Writing +| Type | Channel | `title` | Extra tags | Content | +| --- | --- | --- | --- | --- | +| `buzz.agent.settings` | agent config | `Agent settings` | | `{"version": 1, "agentType": "", "settings": { ... }}` | +| `buzz.agent.instructions` | agent config | `Agent instructions` | | `{"version": 1, "text": ""}` | +| `buzz.team` | team config | the team name | `["team", ""]` | `{"version": 1, "name": "", "instructions": ""}` | -To save text `x` for a subject: +A config channel holds at most one artifact of each of its types. Its `d` is the UUIDv5 of the type name in the namespace of the channel's UUID, so two clients creating the same artifact collide on `d` and one create fails. The client whose create fails fetches the current revision and edits it; it MUST NOT choose a different `d`. A client accepts a config artifact only when its `d`, `type`, and `h` match the config channel it was read from. A `buzz.team` artifact has no member list; members come from the channel (see "Membership"). -1. Validate `x` against the text rules. Reject if invalid. -2. Fetch the subject's current version and let `T` be its `created_at` (or 0 if none). Set `created_at := max(now, T + 1)`. -3. Encrypt `v1:` followed by `x` (see "Content"). Tag the event with the subject. -4. Sign with `seckey_o` and publish. +**Team id.** A `buzz.team` artifact carries exactly one `team` tag, whose value is the team id. A client MUST check that the team id's subject label equals the channel's subject label, and MUST treat the artifact as invalid otherwise. -There is no compare-and-swap. If two devices save concurrently, the relay stores both, the newer becomes current, and the other remains in history. A client SHOULD fetch the current version again after publishing and tell the user when its save was stored but is not current. +**Team name.** `name` is also the artifact's `title`, so it MUST be nonblank and at most 512 UTF-8 bytes, and MUST equal the `title` tag. -Before a subject's first version, a client SHOULD publish the text the subject currently resolves to (see "Launch composition" and "Team instructions"), so the text from before the first edit can be restored. +**Instructions.** `text` and `instructions` MAY be empty, which means "no instructions". They MUST be valid UTF-8 and MUST NOT contain U+0000. -- **Restore** publishes an older version's text as a new version. -- **Reset to template**, for an agent, publishes its linked definition's current `system_prompt` as a new version, or empty text when it has none. +**Settings.** `settings` is a JSON object of the agent's launch configuration. Its fields are defined by the client and harness that launch the agent, not by this NIP. It MUST NOT contain credentials, secrets, machine-local paths, or device state. A client MUST NOT launch an agent with a setting it does not support silently replaced by a default; it shows the error instead. -Clients MUST NOT delete versions to roll back. Deleting the current version makes the previous version current; it is not a restore. +**Agent type.** `agentType` is a string that names the agent type the settings are for. Its values are defined by the client, and clients compare them as exact strings. A writer MUST include it. A reader treats settings without `agentType` as being for the agent's current type. A client MUST NOT launch an agent with settings for a different agent type; it shows the error instead. Changing an agent's type is an edit that writes the new `agentType`. A client that reads a current revision naming a type other than the agent's MAY change the agent to that type when it supports it. -### Launch composition +**Size.** A config artifact's `content` MUST NOT exceed 131,072 bytes after JSON encoding, and `buzz.agent.settings` content MUST NOT exceed 65,535 bytes. The signed event MUST also fit within the relay's advertised event-size limits. Writers check these limits on the encoded content and the signed event, reject an edit that exceeds them, and MUST NOT truncate. -An agent's own instructions layer is the first of these that exists: +An artifact whose content does not parse, has an unknown `version`, or breaks these rules is **invalid**. + +### Artifact state -1. its current agent-subject version. An empty version yields no instructions and stops resolution; an invalid version blocks the start; -2. the `system_prompt` of its linked `kind:30175` definition; -3. for a definition-less instance, the `system_prompt` in its legacy `kind:30177` event. +When a client reads a config artifact, the artifact is in one of these states: -The team layer is resolved independently (see "Launch with teams"). The client delivers both layers to the agent's harness as separate layers, the agent's own first. +- **present:** its current revision was read and is valid; +- **invalid:** its current revision was read and is invalid; +- **absent:** the relay confirmed that the artifact's address has never been used; +- **deleted:** a lookup by `d` returned a deletion; +- **unavailable:** anything else, including a failed read, an unreadable channel, an artifact moved to another channel, a redacted current revision, or a lookup that returns nothing without that confirmation. -A version takes effect at the agent's next start; a running agent is unchanged. A client MAY restart an agent to apply a new version. A client that lets local configuration override the resolved instructions SHOULD show the owner that an override is in effect. +Only an absent artifact lets a client fall back to another source (see "Launch composition"). An invalid, deleted, or unavailable artifact blocks the start: the client MUST show the error and MUST NOT substitute an older revision, another source, or local config, except as "Offline start" below allows for an unavailable artifact. A deleted artifact is recovered with NIP-AR `restore`. -This NIP does not define how `kind:44300` relates to the private managed-agent aggregate reserved by [NIP-PMA](NIP-PMA.md). That integration is left to NIP-PMA. +**Offline start.** When the relay cannot be reached, because the read failed in transport rather than returning an answer, a client MAY start the agent from the last revision it confirmed as present and kept locally. It MUST NOT do so if the most recent answer it read from the relay showed that the artifact is invalid or deleted, that its current revision was redacted, that the config channel is archived, or that it has lost access to the channel, and it MUST NOT use any other source. -### Launch records +### Editing -A client SHOULD record, for each agent start, which instructions it used: +Every channel member who is not a guest can edit the channel's artifacts. This includes members who are not on the team, such as humans and agents with another owner. On a relay whose guest roles are read-only, guests can only read them, and the owner can make an agent read-only for a config channel by giving it the guest role there. -- the agent-subject version id, or that the layer came from a definition, came from a legacy event, or was empty; -- the team id and version id of every team that contributed; -- a SHA-256 hash of each layer's text as delivered, after any local override, and the source of any override; -- a run identifier and the start time. +Every edit is a new NIP-AR revision and names the current revision in `prev`. On conflict, the client reconciles with the current revision as NIP-AR requires. An edit MUST keep any JSON fields and tags the client does not recognise. The one exception: an edit of `buzz.agent.settings` that changes `agentType` MAY drop the fields inside `settings`, because they belong to the old type. -A client that records launches SHOULD write the record once the agent has started and retry until the relay stores it. The record MUST be encrypted to the owner, for example as a [NIP-AE](NIP-AE.md) engram. Version ids and hashes MUST NOT appear in plaintext tags. A launch record shows what the client handed to the harness, not that the harness used it. +### History -### Privacy +Every revision of a config artifact has a stable ID, its event ID. A config artifact's revisions are its history. History is kept until the owner deliberately purges it or a moderator removes a revision. Config channels MUST be exempt from age-based community retention, and a client MUST NOT configure a config artifact to keep only its latest revision. A client shows a removed revision as a gap in history, not as the absence of history. -Instructions versions are readable only by their author. The agent's key has no access to them; the client decrypts the agent's instructions and hands them to the agent at start. The `p` tag names the subject, not a reader, and grants the agent nothing. +**Revert** publishes an older revision's content as an `op=update` revision carrying a complete, valid snapshot. Reverting a team's instructions keeps the team's current name. Clients MUST NOT redact revisions to roll back. + +### Launch composition + +An agent's own instructions layer is the first of these that exists: -The relay still sees each event's subject tag, timestamp, and padded size, so it can observe how often each agent's or team's instructions change and roughly how long they are. +1. the `text` of its current `buzz.agent.instructions` artifact. Empty text yields no instructions and stops resolution. An invalid, deleted, or unavailable artifact blocks the start (see "Artifact state"); +2. while that artifact is absent, instructions held in local configuration; +3. the `system_prompt` of its linked `kind:30175` definition; +4. for a definition-less instance, the `system_prompt` in its legacy `kind:30177` event. -Clients MUST publish instructions versions, and deletion requests for them, only to the relay community that holds the subject. +A client moving local instructions to the relay MUST create the `buzz.agent.instructions` artifact and confirm that the relay accepted it before it stops using the local copy. -### Deletion: kind:44300 +Team sections are resolved independently (see "Launch with teams"). The client delivers the agent's own layer and each team section to the agent's harness separately, the agent's own first. -Owners MAY publish [NIP-09](09.md) deletion requests for instructions versions, one `e` tag per version. A deletion request for a `kind:44300` event MUST carry `["k", "44300"]`; the relay classifies it by that tag, so it stays private after its target is removed (see "Access control: kind:44300 author-only"). A relay MAY accept only one target per request. Clients SHOULD publish one request per version, retry until every version is deleted, and MUST NOT report a subject's history deleted after deleting only part of it. +A change to the agent's instructions or to any of its teams MUST take effect no later than the agent's next session start. A client MAY apply it sooner, including to an active session between turns, and MAY start a new session to apply it. An active session need not be interrupted. A client that lets local configuration override the resolved instructions SHOULD show the owner that an override is in effect. -- A client that deletes an agent SHOULD delete all of that agent's versions. It MUST NOT delete any team's versions. -- A client that deletes a team follows "Deleting a team". +### Deleting an agent -Deletion withholds versions from readers. It does not erase copies held by the relay operator, and it does not reach copies in exports, `kind:30178` shares, or launch records. +A client deletes an agent by archiving its agent config channel. A client MUST NOT start a new session for an agent whose config channel is archived. An active session need not be interrupted. The archived channel stays discoverable, so the agent can be recovered by unarchiving it. Purging the channel is a separate action. Deleting an agent never purges a team's config or history. ### Older clients -Clients that do not implement `kind:44300` ignore it. They resolve an agent's instructions from `kind:30175` and `kind:30177` as before, so they launch agents with the definition's prompt rather than the agent's current version, and they do not see team versions. +Clients that do not implement config channels do not see them, because NIP-CL leaves system channels out of reads that do not name them. They resolve an agent's instructions from `kind:30175` and `kind:30177` as before, so they launch agents with the definition's prompt rather than the agent's current instructions, and they do not see teams stored on the relay. ## Relay behavior @@ -454,26 +455,6 @@ Kind `30178` is stored globally and its content is unvalidated, exactly as for ` - The relay MUST enforce **exactly one** `d` tag whose value is non-empty, at most 64 characters, and free of Unicode control characters and whitespace. Tags are counted by their first element, so a valueless `["d"]` counts toward the total and fails the value check on its own — otherwise `["d"]` alongside `["d",""]` would pass, and a consumer that reads `["d"]` as an empty-valued first `d` tag would address the event at `""` while this relay addresses it at ``. Without the non-empty check, generic NIP-33 storage maps a missing or empty `d` to the empty coordinate, collapsing every team into the single `(pubkey_o, 30178, "")` slot — last-write-wins data loss. The character bound keeps the value usable as a NIP-33 coordinate and as a log field. - The relay MUST NOT apply the persona slug grammar to a `30178` `d` tag; team ids legitimately contain characters (notably `:`) that the slug grammar forbids. -### Ingest validation: kind:44300 - -- The relay MUST reject, with `invalid:`, a `kind:44300` event that does not carry exactly one subject tag as defined in "Event envelope", whose `p` value is not 64-character lowercase hex, or whose `t` value does not match the team id grammar. -- The relay MUST reject a `kind:44300` event with an empty `content` or a `content` longer than 218,548 bytes. It cannot validate the plaintext; that is a client responsibility. -- The relay MUST reject a `kind:44300` event that carries an `h` tag. -- The relay MUST reject a `kind:5` deletion request that targets a stored `kind:44300` event and does not carry `["k", "44300"]`. -- The relay stores `kind:44300` events globally, outside any channel. - -### Access control: kind:44300 author-only - -The relay MUST withhold `kind:44300` events, and their existence, from every reader except their authenticated author, on every read surface: historical REQ delivery, NIP-01 `ids` lookup, live fan-out, COUNT, and the HTTP bridge's query and count endpoints. - -The relay MUST apply the same rule, on the same read surfaces and in search results, to every `kind:5` deletion request that carries `["k", "44300"]`, whether or not its target is still stored. A deletion request reveals a version's id and the time it was deleted. - -The relay MUST exclude `kind:44300` from full-text search results for every reader, including the author. - -The relay MUST apply a `#p` or `#t` filter on `kind:44300` before `ORDER BY … LIMIT`, so that `limit: 1` returns a subject's current version even when the owner has newer versions for other subjects. - -The relay MUST support the composite history cursor (see "Current version and history") for `kind:44300`. A filter carrying `before_id` without `until`, or a `before_id` that is not 64 hexadecimal characters, MUST be rejected rather than ignored. - ### Access control: author-only-unless-shared Kind `30175` uses **shared-tag-gated read semantics** to protect system prompts and `respond_to_allowlist` from being visible to all community members as a side-effect of device sync. @@ -507,13 +488,13 @@ These rules are enforced at the following relay read surfaces (content and event ## Security considerations -- **No encryption for definitions.** Persona and team-catalog events store system prompts, model names, runtime identifiers, and all configuration unencrypted. `kind:44300` instructions versions are the exception: they are encrypted to the owner. Shared persona events are readable community-wide. Operators MUST NOT store secrets in persona event content. +- **No encryption for definitions.** Persona and team-catalog events store system prompts, model names, runtime identifiers, and all configuration unencrypted. Shared persona events are readable community-wide. Operators MUST NOT store secrets in persona event content. - **System prompt protection.** System prompts and `respond_to_allowlist` pubkeys are sensitive. The relay's author-only-unless-shared gate ensures they are not visible to other community members unless the owner explicitly opts in by publishing a `["shared", "true"]` head. Shared persona events are readable community-wide; operators who need additional confidentiality should use relay-level access controls or choose not to share. - **Write authority.** Only the holder of `seckey_o` can publish or replace persona events. NIP-33 replacement is scoped by pubkey — no spoofing risk from other relay members. - **Slug collision across pubkeys.** Two different owners can publish personas with the same slug. Clients MUST always scope queries by author pubkey, not just slug. - **Metadata exposure.** The `(pubkey, kind:30175, slug)` triple reveals persona existence. Event timestamps reveal edit history. - **No owner write authority over agents.** Persona events define *what* an agent should be; they do not grant runtime control over a running agent. The agent consumes the persona at spawn time. Updates to the persona event do not automatically propagate to running agents. -- **Instructions history is owner-private, not secret from the owner's clients.** `kind:44300` versions are encrypted to the owner and withheld from other readers. Any client holding `seckey_o` can read every version; deleting versions does not revoke copies already exported, shared, or recorded at launch. +- **Config is readable by every config channel member.** Agent settings, agent instructions, and team instructions are not encrypted. Every member and guest of a config channel can read its artifacts and their history, so adding someone to a config channel shares that config with them. Every member who is not a guest can also edit it, including humans and agents that are not on the team. The relay operator can read them too. Removing someone, archiving the channel, or redacting a revision does not revoke copies already read, exported, or shared. - **Sharing a team shares every member's instructions.** A `kind:30178` head carrying `["shared","true"]` exposes the team's own fields *and* the embedded projection of every member — including members whose own `kind:30175` heads are unshared and therefore still private. Clients MUST make this explicit at the point of sharing; the relay cannot infer it. ## Reference test vectors diff --git a/docs/nips/NIP-AR.md b/docs/nips/NIP-AR.md index 3f0a8874996..da0c36a17db 100644 --- a/docs/nips/NIP-AR.md +++ b/docs/nips/NIP-AR.md @@ -93,7 +93,7 @@ Relays MUST reject kind-5 deletion requests targeting artifact revisions with an ## Current state and queries -An artifact's current state is its latest accepted revision, as determined by the `prev` rules above. Current-state queries MUST return only that revision and omit deleted artifacts. For example, a task whose current revision belongs to Project B no longer appears in Project A's list, even if an earlier revision belonged there. Revision history is queried separately. This NIP does not define a public acceptance order or history pagination; relays record the order in which they accept revisions, and a later sync specification defines how clients observe it. +An artifact's current state is its latest accepted revision, as determined by the `prev` rules above. Current-state queries MUST return only that revision and omit deleted artifacts. For example, a task whose current revision belongs to Project B no longer appears in Project A's list, even if an earlier revision belonged there. Revision history is queried separately. This NIP does not define a public acceptance order or history pagination. Relays record the order in which they accept revisions; clients observe the resulting current state through sync (see Sync), not through timestamps or `prev` chains. Relays MUST support querying current artifacts across channels the reader can access using exact tag-name/value matches, including multi-character tag names. Conditions on different tag names are combined with AND; requested values for one name are combined with OR. A match compares the tag name and its first value in the same tag. Access checks and tag matching apply before pagination and any counts. Older revisions do not contribute matches. @@ -101,8 +101,274 @@ Relays MUST advertise and enforce limits on tag-name/value sizes, total tag byte Artifact revisions are state changes, not conversation messages. Receiving one MUST NOT increase chat or reply counts, add a chat unread, mark a conversation read, or repeat mention or push notifications for tags carried over from the previous revision. +## Sync + +Sync lets a client hold an exact local copy of a set of artifacts and keep it current. The relay owns the order of changes: the relay already decides which revision is current (the `prev` check above), and sync only reports those decisions by always sending the current head. Clients do not order changes by `created_at`, which the signer chooses, or by walking `prev` chains, which needs every revision. + +Sync is additive. A client that does not use it keeps working unchanged, and a relay advertises each part separately (see Relay information document). A reader receives through sync only artifacts that it can already read with a current-state query, under the same channel read permission. + +Incremental reads (below) are a separate HTTP protocol with their own capability. They let a client update a saved store after a disconnect without reading the whole set again, but they do not prove the complete set that `EOSE` proves here. + +```mermaid +sequenceDiagram + participant C as Client + participant R as Relay + C->>R: REQ sub {"artifact":"current", "#h":[...], "#type":[...]} + Note over R: Register live interest first,
then read all matching heads
in one snapshot + R-->>C: EVENT sub head (0..n) + R-->>C: EOSE sub (the set is complete at one cut) + Note over C: Replace local set only here + loop live + R-->>C: EVENT sub head or kind 45011 removal + end + R-->>C: CLOSED sub "resync: ..." (access changed, or a change could not be delivered) +``` + +### Sync subscription + +**Request.** Sync is a NIP-01 `REQ` whose single filter has `"artifact": "current"`. It is the current-state filter of HTTP `/query`; the `REQ` transport adds the complete set, `EOSE` and live heads. The client does not page: the relay streams the whole set until `EOSE`. Other views (`history`, `head`) have no `REQ` form and stay HTTP-only: + +```json +["REQ", "s1", {"artifact": "current", "#h": [""], "#type": ["buzz.task"]}] +``` + +The filter MUST include `#h` with 1 or more channels. It accepts the same `#name` predicates and limits as a NIP-AR current-state query, with the same AND/OR rules, and also `kinds: [45010]`. It accepts no `limit` or `offset`. A relay rejects a filter without `#h`, or with an unknown key, with `CLOSED`. + +**Scope.** The sync's scope is the set of artifacts that a NIP-AR current-state query with the same filter returns to this reader at that time. The rules below use this one definition for access, matching and system channels. + +**Response.** The relay answers with ordinary NIP-01 messages. Only the meaning of `EOSE` is stronger: for a sync, it means "this is the complete set at one point in time", not only "end of stored events". + +| Message | Meaning | +|---|---| +| `["EVENT", sub, ]` | This revision is the artifact's current head. If it is a deletion, or it no longer matches the filter, the client removes the artifact from its set. | +| `["EVENT", sub, ]` | Live only. A move removed the artifact from a channel the reader can read. The client removes it. | +| `["EOSE", sub]` | Sent once, after the initial set. The set has been delivered in full, as one consistent cut (rule 2). | +| `["CLOSED", sub, "resync: "]` | The relay can no longer keep this subscription exact. The client starts a new sync. | +| `["CLOSED", sub, "restricted: "]` | The reader cannot read a channel that `#h` names, or the channel does not exist. The message does not say which. | +| `["CLOSED", sub, "error: "]` | The sync failed. The client keeps its last complete set and treats it as possibly stale. | + +Rules: + +1. **Live before stored.** The relay MUST register live interest, and wait until it is active, before it opens the snapshot of rule 2. Live interest is active only when the pub/sub server has confirmed the subscription (for Redis, the reply to `SUBSCRIBE`) and the node's pub/sub connection is still up. While that connection is down or reconnecting, a new sync waits, or gets `CLOSED "error: ..."`. Then every change that commits after the snapshot causes a live entry, or a resync when the relay detects that the entry was lost (rule 5). This is the order that `REQ` already uses. The relay MUST NOT send a live entry before `EOSE`. During the read it holds the `d` of each changed artifact. After `EOSE` it reads and sends their heads as in rule 3. A held change can already be in the snapshot; then the relay sends the same head again, a newer one, or nothing (rule 4). The held list is bounded: if it grows past `max_events` distinct `d`, the relay sends `CLOSED "resync: ..."` and no `EOSE`. +2. **Initial set: one consistent cut, read in pages.** The relay reads the set inside one read-only transaction that has one database snapshot (Postgres `REPEATABLE READ`). It opens the transaction after it receives the `REQ` and after rule 1, on the writer database or on a replica proven at least as fresh. So every change that the relay accepted with `OK` before it received the `REQ` is in the set. Inside the transaction it reads pages ordered by `d`. Each page starts after the last `d` of the page before it (keyset pages, not offset pages). All pages see the same snapshot. So the set at `EOSE` is the exact set at one point in time: a combination of heads that existed together, never a mix of earlier and later states. This is the complete-set read. The relay sends one `EVENT` for each artifact in scope as it reads it, then `EOSE`. The client sees one stream; it never joins pages. A snapshot does not need the whole set in one result, so there is no byte limit on the whole set. The relay bounds how long it holds a snapshot open, with a budget that it defines. It MAY copy pages into a bounded buffer so that it can close the snapshot before a slow reader has received them. If the read fails, the snapshot is lost, or the budget runs out (for example, the reader is too slow), the relay sends `CLOSED "error: ..."` and no `EOSE` (rule 8). It MUST NOT continue the read on another snapshot. If the reader cannot read a channel that `#h` names, the relay sends `CLOSED "restricted: ..."` and no `EOSE`, whether or not the channel exists. +3. **Live changes.** After `EOSE`, when an artifact in scope changes, the relay reads its current head. If the reader can read the head's channel, it sends the head, even when the head is a deletion or no longer matches. If the artifact moved to a channel the reader cannot read, it sends the kind 45011 removal stored in the source. It never sends anything about an artifact the reader cannot read. A change is an accepted revision (any `op`), or a move's removal from the source. An artifact is in scope for a change when it was in scope before the change or is in scope after it. "Before" is the client's copy as this sync sent it: the initial set plus later heads that matched. "After" is a current head, not a deletion, that matches every predicate of the filter with the NIP-AR match rule (the tag name and its first value in the same tag). The relay sends nothing for an artifact that is in scope neither before nor after, so a head that never matched the filter never reaches the client. +4. **Heads only, never older.** The relay sends the current head, never an older revision. It MAY skip intermediate revisions and MAY repeat a head the reader already has. For any one `d`, it MUST NOT send a head that it read before the head it last sent for that `d`. To keep this, for each sync the relay reads the head of one `d` one read at a time, starts each read only after the result of the read before it (a head, a kind 45011 removal or nothing) is in the outbound queue, and reads from the writer database or a replica proven at least as fresh. Clients that need every revision use the NIP-AR history view. +5. **No silent loss.** Live fan-out can lose entries: a publish can fail, a node's pub/sub connection can drop for seconds, and a node's local fan-out can lag. Pub/sub does not replay any of these. A sync MUST NOT lose a change silently. So: + - A node MUST send `CLOSED "resync: ..."` to every sync it serves that a loss on that node can affect: its pub/sub connection dropped or reconnected; its local fan-out lagged or dropped entries, including a message that it could not decode; it could not check the reader's access to a change for that sync (for example, a failed membership lookup); or the connection's own send queue was full and dropped an entry for that sync. One drop is enough: the slow-client grace limit does not apply to a sync. It MAY resync every sync it serves. + - If the node cannot queue the `CLOSED "resync: ..."` itself (for example, because the same send queue is full), it MUST close the connection. A client starts a new sync on any disconnect (rule 7), so a resync is never lost. + - A node MUST retry a failed publish until it succeeds or the node stops. Delivery order does not matter. For a sync, a live entry is only a signal that one `d` changed. The reader's node never sends the published event itself to a sync; it reads the current head (rule 3). So a late retry of an older change sends the current head or nothing, never an older one (rule 4). + + So a sync that is open and has had no resync has received every change, except when a node crashes after it commits a change and before it publishes it. A client that must bound that window can sync again on its own schedule. The relay does not read the set again on a timer. +6. **Retirement.** A moderation redaction that retires the current revision of an artifact in scope makes the relay send `CLOSED "resync: ..."`. The relay withholds the redacted head, so a new sync is the only way the client learns that the artifact left the set. A relay MUST NOT retire a head in any other way that open syncs cannot observe, such as an operator delete outside NIP-29; it refuses such deletes or ends the affected syncs with `resync`. +7. **Commit point.** A client MUST replace its local set only when it receives `EOSE`. If the connection drops before `EOSE`, it discards what it received. After `EOSE`, it applies each live entry as it arrives. On any disconnect it starts a new sync; there is no resume. Before a new sync after a resync, an error or a dropped connection, the client waits a random delay that grows after each failure in a row (backoff with jitter). +8. **No partial success.** If the relay cannot deliver the full set within its limits or budget, it MUST send `CLOSED` and MUST NOT send `EOSE`. A closed connection without `EOSE` is never proof of an empty set. The relay bounds its buffers for a sync in bytes as well as in count, and it ends the snapshot and its reads as soon as the client disconnects or sends `CLOSE`. +9. **Resync.** The relay MUST send `CLOSED "resync: ..."` when the reader's access to a channel that `#h` names changes: membership, channel visibility or channel deletion. This closes the whole sync, also when `#h` names other channels. An ordinary `REQ` can keep its other channels; a sync cannot, because its set is no longer exact. Archive does not change reads (see Home channel lifecycle), so it does not cause a resync. + +**Short-lived clients.** A CLI that needs the set one time sends the same `REQ`, waits for `EOSE`, then sends `CLOSE`. It gets the cut of rule 2: a complete set that includes every change the relay accepted before it received the `REQ`. It does not see changes after the cut. A writer catches those with `prev` (NIP-AR compare-and-set). There is no separate one-shot request. + +**Set digest.** `set_digest` is the lowercase hex SHA-256 of the UTF-8 bytes `nip-ar-set-v1`, a zero byte, and then `:\n` for each artifact in the set, sorted by `d`. It is a content identifier. It is not a counter, not a `prev` precondition and not a live-delivery cursor. An unchanged set has the same digest. A client can compute it over the set that it holds, or over any part of that set, and get the same value as any other client with the same heads. + +The relay does not send the digest. A client computes it over the set that it holds at `EOSE` or later. + +**Offset pages are views.** The existing `/query` offset pages stay for display and browsing. This section states that they are not evidence of a complete set. + +### Head lookup, including deletions + +Retries and "add or restore" flows need the current revision of one artifact even when it is deleted. Today a current-state query omits deleted artifacts, and history returns every revision. + +Relays that support sync also support the query view `"artifact": "head"`. It requires `#d` and returns the current revision of each named artifact the reader can read, **including** a deletion. It returns nothing for an unknown `d`, or for one the reader cannot read; it does not say which. + +### Deterministic identity and the ensure flow + +Clients MAY derive `d`. This section defines one standard derivation, so that independent clients compute the same `d` for the same logical record: + +``` +d = UUIDv8( first 16 bytes of SHA-256("nip-ar-artifact\0" + h + "\0" + type + "\0" + key) ) +``` + +`h` is the home channel at creation, `type` is the artifact type, and `key` is a client-chosen name, unique within that type and channel (for example a config entry name). Version and variant bits are set as in RFC 9562. + +A derived `d` is a name, not a proof of authorship. Anyone who can write in `h` can create it first. Readers decide which authors to trust, as with every artifact. + +**Ensure (create or restore, safe to retry).** To make "this record exists with this content" true, a client: + +1. Sends `create`. If accepted or a duplicate of an accepted event, it is done. +2. On `conflict`, reads the head (see Head lookup). + - If the head is a deletion, sends `restore` with `prev` set to it. + - If the head is live with the same content and tags, it is done. + - Otherwise sends `update` with `prev` set to the head, or reports the conflict, as the client's type requires. +3. Repeats step 2 on a further conflict, up to a client-defined limit. + +The relay changes nothing for this flow except the head view. A deleted `d` stays reserved (see Identity and edits). + +### Home channel lifecycle + +| Home channel event | Writes | Current-state reads and sync | +|---|---|---| +| Archived | Rejected with the kind-9 archive reason. | Unchanged. Readers keep reading. To keep editing, unarchive or `move` the artifacts out first. | +| Unarchived | Allowed again. | Unchanged. | +| Deleted | Rejected. | Its artifacts leave every current-state result. Each live sync that covered the channel gets `resync`. A later channel with a different ID does not inherit them. | + +### Client guidance (non-normative) + +- **Type is the record family.** `type` cannot change. Put variants that can change (for example watch vs timer) in `content` under one type. If a family must change type, create new artifacts and delete the old ones. +- **Version the content.** Put `"version": n` in `content`. Ship readers that accept `n+1` before writers that produce it. Editors already keep unknown fields or decline the edit (see Client behavior). +- **Titles are visible to every reader of the channel.** Put no secrets in `title`. +- **Conflicts are normal.** On `conflict`, reload with the head view, reapply the user's change to the new head, and show the user when that is not possible. Do not resubmit with only `prev` replaced. + +### Relay information document + +The NIP-11 relay information document has an `artifacts` object. A relay that supports sync adds: + +```json +"sync": {"version": 1, "max_events": 4096}, +"modes": ["current", "history", "head"] +``` + +`max_events` is the largest set that a sync serves. It bounds each initial read and the held list (rule 1). There is no byte limit on the whole set: the relay reads and sends it in pages from one snapshot, and the relay's existing event size limit bounds each event. The relay does not advertise its snapshot time budget (rule 2); a reader that is too slow gets `CLOSED`. + +and sets `websocket_query_extensions` to `true`. + +Relays advertise `sync` and `head` only when every backend that can serve the connection runs them. + ## Client behavior Clients validate content before interpreting it as a known type. Unknown types or malformed content render a safe fallback using the title, type, and home link. Editors MUST preserve unfamiliar fields and annotation tags or decline the edit. References may be unavailable or cyclic; clients must handle them safely without assuming relay validation. Artifact text is untrusted. A type name, relationship tag, or payload cannot authorize a privileged action. Moving, deleting, or revoking access cannot erase copies already downloaded. + +## Incremental reads + +Incremental reads maintain an eventually consistent local store. They do not prove a complete set at one database snapshot. Absence from a page never proves deletion or loss of access. + +This HTTP protocol uses `artifacts.incremental`. The WebSocket Sync subscription uses a separate `artifacts.sync` capability and defines its own complete-set guarantee. Clients select each protocol through its own capability. + +The relay advertises `artifacts.incremental` when incremental reads are enabled. +Each signed `POST /query` request contains one filter. The query uses the +same community, relay admission, channel access, and exact tag rules as an +ordinary artifact read. The existing `current` and `history` array responses +keep their format. +Unlisted open channels require membership or an explicit `#h` scope. +Known-ID reads do not bypass this rule. + +```json +[{"artifact":"incremental","query":{"artifact":"current","#type":["buzz.task"]}}] +``` + +The initial read pages through current identities. A later read supplies the +last completed `checkpoint`. A continuation supplies only `cursor`, with +the same `query`. The server signs these opaque strings. They bind the +community, reader, exact query, access-policy boundary, and database start +time. Tokens contain a fixed-size digest of the query. The cursor also binds +the last page position. Relay instances use the +same configured signing identity, so another instance can serve a continuation. +No server-side manifest or Redis allocation is required. + +```json +{ + "version": 1, + "startedAt": "2026-10-21T12:00:00.123456Z", + "overlapSeconds": 120, + "changes": [ + {"type":"upsert","artifactId":"22222222-2222-4222-8222-222222222222","event":{}}, + {"type":"remove","artifactId":"33333333-3333-4333-8333-333333333333"}, + {"type":"removeChannel","channelId":"44444444-4444-4444-8444-444444444444"} + ], + "nextCursor": null, + "checkpoint": "opaque" +} +``` + +`event` is the complete signed current revision. `remove` has one meaning: +the requested artifact is unavailable for this query. It does not distinguish +a missing identity, deletion, redaction, query exit, or denied access. +`removeChannel` names a removed membership retained for this exact reader. +It applies only to local rows whose current saved home is that channel. +An earlier move upsert to another home therefore survives a later source +channel removal. + +Each pass fixes `startedAt` from the database clock. A later pass reads from +the previous completed start time minus 120 seconds through its own start +time, inclusive. New head writes, move-source records, and membership changes +use database `clock_timestamp()`. The protocol assumes the delay from this +stamp to commit visibility stays within the accepted 120-second overlap. +That overlap is an operating assumption. It is not a measured upper bound. + +Pages order by `(change timestamp, source, stable identity)`. The source +separates membership removals, head changes, grants, and move-source records. +The identity breaks equal timestamp ties. It does not encode creation time. +The time window and page position are stable. The current contents can change +between requests. Replays must be idempotent. A later pass repairs changes +that move beyond the current window. + +Only the final page returns a checkpoint. The client saves it after every +page and local commit succeeds. A failed read keeps the prior checkpoint. +Neither an initial read nor delta completion establishes absence. The client +must not remove unseen rows. A fresh access grant replays older current +heads from that channel. A move retains its source channel in the existing +revision ledger, so a reader who cannot read the destination receives a +current revalidation result without destination metadata. + +### Current reads and write confirmation + +```json +[{"artifact":"incremental","action":"current","query":{"artifact":"current","#type":["buzz.task"]},"artifactIds":["22222222-2222-4222-8222-222222222222"]}] +``` + +The response has `version`, database `startedAt`, and `changes`. It contains +exactly one upsert or remove for each requested ID, in request order. +The request permits 1–100 unique nonempty strings, each at most 256 UTF-8 +bytes. A string that is not a canonical artifact UUID returns remove. +Duplicates and invalid bounds fail with `incremental_invalid`. The server does not +truncate the request. + +Clients serialize a full delta pass and its commits before a current-ID read. +Live revision pushes request a current-ID read; they do not replace confirmed +rows directly. A current-ID read that starts after publication acceptance +reads that commit or a later state on the writer. It can return a later head +or remove. An empty delta does not confirm an accepted edit. + +### Access recovery and bounds + +Relevant channel access changes atomically advance one community policy +stamp, including transitions on empty channels. Ordinary artifact writes do +not update this stamp. A different stamp +returns HTTP 409 `incremental_reset` with `reason: "policy_changed"`, without naming +private channels. A checkpoint older than 30 days returns the same code with +`reason: "checkpoint_expired"`. + +Recovery runs a new initial read to discover current readable records and +bounded current-ID reads for the IDs already saved locally. These reads +revalidate known IDs without a channel roster or an absence scan. Save new +progress only after discovery, revalidation, and local commits succeed. +Preserve preferences when a record becomes unavailable. + +The server retains task heads, membership removal rows, and move-source +ledger records. Removing this recovery history requires an explicit policy +reset before a client can resume. Whole-community deletion closes admission +and purges these rows through the existing deletion path. + +NIP-11 advertises: + +```json +{"version":1,"overlapSeconds":120,"pageRows":100,"currentRows":100,"responseBytes":8388608,"permanentChannelsOnly":true,"checkpointLifetimeSeconds":2592000} +``` + +Pages shrink to the response byte bound. A single oversized result returns +HTTP 413 `incremental_limit`. A current-ID batch can return that error; the client +can split the batch. No page or batch silently truncates its result. The +server permits four concurrent operations per process, a two-second SQL +statement budget, and an eight-second operation budget. There is no +identity-wide membership count limit on this path. + +Incremental reads include only matching readable artifacts in permanent channels. +TTL channels do not contribute discovery candidates. A known-ID read returns +`remove` for an artifact in a TTL channel. Moves into a TTL channel return +removals for the permanent source. An explicit `#h` scope limits candidates +to those channels, including retained move sources and membership removals. +Initial reads omit deleted heads. Later reads retain deletion removals. +HTTP 400 `incremental_invalid` +rejects invalid requests or tokens. HTTP 503 `incremental_unavailable` reports a +service or work-budget failure. These failures do not advance progress. + +Incremental completion does not establish a live subscription to every channel. diff --git a/docs/nips/NIP-CL.md b/docs/nips/NIP-CL.md new file mode 100644 index 00000000000..27ed3a8a400 --- /dev/null +++ b/docs/nips/NIP-CL.md @@ -0,0 +1,306 @@ +NIP-CL +====== + +Channel Identity and Discovery +------------------------------ + +`draft` `optional` `relay` + +This NIP adds identity facts to the relay-signed NIP-29 channel-state events +(kinds 39000–39002). With these facts, a client can find channels with +ordinary NIP-01 filters. It needs no new filter key and no relay-specific API. + +## Motivation + +Plugins and agents often create a channel for their own use and must find it +again later, for example "the config channel that this agent created". With +plain NIP-29, a client cannot: + +- tell who created a channel; +- read the channel type from any event except kind:39000; or +- tell which of two channels is older. The `created_at` of a kind:39000 event + is the time of the last edit, not the time of creation. + +A client also cannot mark its own channels so that it can find them again. +Channel labels (below) do this. + +Also, every channel that a person belongs to shows in their client as a chat. +So a group that exists only to control who can read some data appears in +sidebars, search results and unread counts. System channels (below) fix this. + +## Channel identity tags + +On every channel-state event that the relay signs, the relay MUST put these +tags in this order, before any other `t` or `P` tag: + +1. `["t", ]`, for example `stream`, `forum`, `dm`, `workflow`, `system`. +2. `["t",