diff --git a/.eslintignore b/.eslintignore index 168d4a14..90c978b3 100644 --- a/.eslintignore +++ b/.eslintignore @@ -1,2 +1 @@ -example/RNBackgroundExample/node_modules -example/RNBackgroundExample/e2e +example/ diff --git a/.eslintrc.js b/.eslintrc.js index 40c6dcd0..256c16b8 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -1,4 +1,8 @@ module.exports = { root: true, - extends: '@react-native-community', + extends: '@react-native', + plugins: ['unused-imports'], + rules: { + 'unused-imports/no-unused-imports': 'error', + }, }; diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index a72f54c9..0d0ca920 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,37 +1,37 @@ -## Objective \* + -> _Briefly describe the purpose of this PR._ +# Summary -## Details and Context \* + -> _Insert ticket link here._ +## Test Plan -> _Insert Loom video link here if available._ + -## Pre-Submission Checklist \* +### What's required for testing (prerequisites)? -- [ ] I have reviewed all the changes one final time before opening this PR. +### What are the steps to reproduce (after prerequisites)? -## Tests +## Compatibility -- [ ] I have added automated tests for all new code introduced in this PR. -- [ ] I have not added automated tests, because... (explain below) +| OS | Implemented | +| ------- | :---------: | +| iOS | ✅❌ | +| Android | ✅❌ | -### Risks and uncertainty (optional) +## Checklist -- [ ] This PR doesn't introduce any new risks or uncertainties. -- [ ] This PR introduces new risks or uncertainties, because... (explain below) + -## Additional Information - -- Feedback Ladder Reminder - - - Mountain: Blocker & immediate action needed. - - Boulder: Blocker & requires action - - Pebble: Non-blocking, but requires future action (in either this or a future PR) - - Sand: Non-blocking, but is generally best practice - - Dust: Non-blocking comment - -- [**Pull Request Standards**](https://sitemate.atlassian.net/wiki/spaces/ENG/pages/143458410/Branching+Pull+Requests+Code+Reviews#%E2%9C%A8--Pull-Requests) +- [ ] I have tested this on a device and a simulator +- [ ] I added the documentation in `README.md` +- [ ] I updated the typed files (TS and Flow) +- [ ] I've added Detox End-to-End Test(s) +- [ ] I've created a snack to demonstrate the changes: LINK HERE diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml deleted file mode 100644 index b5518ac0..00000000 --- a/.github/workflows/android.yml +++ /dev/null @@ -1,75 +0,0 @@ -name: Test Android Example App - -on: [push, pull_request] - -jobs: - react-native-android: - runs-on: macos-latest - if: "!contains(github.event.head_commit.message, '[skip ci]')" - - steps: - - name: Checkout project - uses: actions/checkout@v3 - - - name: Specify node version - uses: actions/setup-node@v3 - with: - node-version: 16 - - - name: Use specific Java version for sdkmanager to work - uses: actions/setup-java@v2 - with: - distribution: 'temurin' - java-version: '8' - architecture: 'x64' - - - name: Setup Android emulator - run: | - echo y | sudo $ANDROID_HOME/tools/bin/sdkmanager "system-images;android-33;google_apis;x86_64" > /dev/null - $ANDROID_HOME/tools/bin/avdmanager -s create avd -n emu -k "system-images;android-33;google_apis;x86_64" -b "x86_64" -c 1G -d 7 -f - - - name: Install node_modules - working-directory: example/RNBackgroundExample/ - run: - yarn install --frozen-lockfile - - - name: Use specific Java version for the build - uses: actions/setup-java@v2 - with: - distribution: 'temurin' - java-version: '11' - architecture: 'x64' - - - name: Deploy - working-directory: example/RNBackgroundExample/ - run: - yarn e2e/deploy/android - - # - name: Start Emulator - # working-directory: example/RNBackgroundExample/ - # timeout-minutes: 5 - # run: | - # export PATH=$PATH:$ANDROID_HOME/platform-tools - # $ANDROID_HOME/emulator/emulator @emu -noaudio -no-boot-anim -netdelay none -accel on -no-snapshot & - - # - name: Wait for emulator to be ready - # working-directory: example/RNBackgroundExample/ - # timeout-minutes: 15 - # run: | - # sh ./wait-for-emulator.sh - - # - name: Android test - # working-directory: example/RNBackgroundExample/ - # timeout-minutes: 8 - # run: | - # mkdir -p ./artifacts - # node e2e/start-server.js & - # adb reverse tcp:8080 tcp:8080 - # yarn e2e/test/android --record-videos failing - - # - uses: actions/upload-artifact@master - # name: Provide videos of failed E2E tests - # if: failure() - # with: - # name: android-failing-e2e-videos - # path: example/RNBackgroundExample/artifacts diff --git a/.github/workflows/deploy-release.yml b/.github/workflows/deploy-release.yml deleted file mode 100644 index 3d42dc10..00000000 --- a/.github/workflows/deploy-release.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: Deploy & Release - -on: - push: - branches: - - master - -jobs: - deploy-release: - runs-on: ubuntu-22.04 - if: "!contains(github.event.head_commit.message, '[skip ci]')" - - steps: - - name: Checkout project - uses: actions/checkout@v3 - - - name: Specify node version - uses: actions/setup-node@3 - with: - node-version: 16 - - - name: Release to NPM - if: github.ref == 'refs/heads/master' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - NPM_TOKEN: ${{ secrets.NPM_TOKEN }} - run: | - yarn install --frozen-lockfile - yarn release diff --git a/.github/workflows/iOS.yml b/.github/workflows/iOS.yml deleted file mode 100644 index 99066c32..00000000 --- a/.github/workflows/iOS.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: Test iOS Example App - -on: [push, pull_request] - -jobs: - react-native-ios: - runs-on: macos-latest - if: "!contains(github.event.head_commit.message, '[skip ci]')" - - steps: - - name: Checkout project - uses: actions/checkout@v3 - - - name: Specify node version - uses: actions/setup-node@v3 - with: - node-version: 16 - - - name: Setup Detox - run: | - brew tap wix/brew - brew install applesimutils - - - name: Install node_modules - working-directory: example/RNBackgroundExample/ - run: | - yarn install --frozen-lockfile - npx pod-install ios - - - name: Deploy - working-directory: example/RNBackgroundExample/ - run: - yarn deploy/release/ios - - - name: iOS test - working-directory: example/RNBackgroundExample/ - timeout-minutes: 8 - run: | - npx detox clean-framework-cache && npx detox build-framework-cache - node e2e/start-server.js & - yarn e2e/test/ios diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml index 4c145618..46803663 100644 --- a/.github/workflows/node.yml +++ b/.github/workflows/node.yml @@ -1,24 +1,81 @@ -name: Node Environment +name: CI on: [push, pull_request] jobs: - node-lint-tests: - runs-on: ubuntu-22.04 + checks: + runs-on: ubuntu-latest if: "!contains(github.event.head_commit.message, '[skip ci]')" steps: - - name: checkout - uses: actions/checkout@v3 + - name: checkout + uses: actions/checkout@v4 - - name: setup node - uses: actions/setup-node@v3 - with: - node-version: 16 + - name: setup node + uses: actions/setup-node@v4 + with: + node-version: 20 - - name: install node_modules - run: yarn install --frozen-lockfile + - name: install node_modules + run: | + yarn install --frozen-lockfile + yarn --cwd example/RNBGUExample install --frozen-lockfile - - name: node lint - run: - yarn lint:ci + - name: lint + run: yarn lint:ci + + - name: typecheck + run: yarn typecheck + + - name: js tests + run: yarn test + + - name: setup java + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 17 + + - name: android unit tests + run: | + cd example/RNBGUExample/android + ./gradlew :react-native-background-upload:testDebugUnitTest + + # Host-side unit tests for the pure Swift half of the iOS module. + # They run on macOS with Xcode's Swift toolchain. ios/Package.swift needs + # swift-tools-version 5.9. The macos-15 image ships Xcode 16 or later + # (Swift 6 or later), which meets that. + # + # This job does not build the example app. Do that by hand before a + # release, because a simulator build is too slow for every push: + # cd example/RNBGUExample/ios && pod install + # xcodebuild -workspace RNBGUExample.xcworkspace -scheme RNBGUExample \ + # -sdk iphonesimulator -destination 'generic/platform=iOS Simulator' \ + # -configuration Debug build + ios-host-tests: + runs-on: macos-15 + timeout-minutes: 30 + if: "!contains(github.event.head_commit.message, '[skip ci]')" + + steps: + - name: checkout + uses: actions/checkout@v4 + + - name: swift version + id: swift + run: | + swift --version + echo "version=$(swift --version 2>&1 | head -n 1 | shasum | cut -c1-12)" >> "$GITHUB_OUTPUT" + + - name: cache swift build + uses: actions/cache@v4 + with: + path: ios/.build + key: swiftpm-${{ runner.os }}-${{ steps.swift.outputs.version }}-${{ hashFiles('ios/*.swift', 'ios/Tests/**/*.swift') }} + restore-keys: | + swiftpm-${{ runner.os }}-${{ steps.swift.outputs.version }}- + + - name: ios host tests + run: | + cd ios + swift test diff --git a/.gitignore b/.gitignore index a7e11824..0bd27078 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,3 @@ -# lib build -src/index.js - # VIM temporary files *.swp diff --git a/CHANGELOG.md b/CHANGELOG.md index f6a050d4..5f9b378b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,407 @@ -Since Version > 5.3.0 we follow semantic versioning. +## Unreleased -See the [releases](https://github.com/Vydia/react-native-background-upload/releases) page on GitHub for information regarding each release. +iOS removes staged bodies that no entry claims at launch, once reconcile's +grace waits end. A process killed while `mutate()` was staging a body, before +the entry saved, left that body on disk for good. A chunked file already +moved into an entry-less directory stays, for a same-id `mutate()` to adopt. + +## 11.0.1 + +iOS waits two seconds after the app becomes active before replacing +background-created tasks, and no longer replaces a task that has had a +response, so the daemon's progress and completions from the suspension land +first. See "Not at the system's discretion" in the README for the limits. + +A replaced simple request whose body landed anyway now completes its entry, +even when its replacement failed and the entry moved to a later attempt. +Before, the late success was dropped and the request was sent again. + +A chunked upload saves its entry and the task map once per batch of part +tasks instead of once per part. + +## 11.0.0 + +Breaking: removed `ios.queueAllParts` from the request descriptor. A chunked +upload now always creates every part task at `mutate()`, and `mutate()` with +`ios` rejects as an unknown field. It needs the file's size again in free +space until the parts are sent. + +iOS starts a task created while the app is in the background at its own +discretion and ignores `isDiscretionary = false` for it. The library now +records whether the app was active when it created each task. When the app +becomes active, it replaces the tasks created while it was not that have +not sent a body byte. That covers retries, `mutate()` from a background +wake, and tasks a relaunch rebuilds. Tasks created by earlier versions +count as background tasks, so uploads already deferred by the system start +again the first time the app becomes active after the upgrade. + +## 10.1.0 + +Added `ios.queueAllParts` to the request descriptor. With `parts`, it creates +every part task at `mutate()` instead of 3 at a time. iOS starts a task +created while the app is in the background at its own discretion, so a +chunked upload's later parts could wait for the system. It needs the file's +size again in free space, because each part task has its own part file. +Entries saved before this change keep the window of 3. Android ignores the +field. + +## 10.0.0 + +The library now owns a durable request queue. A consumer describes each +request kind one time with `define()`, enqueues instances with `mutate()`, +and receives every outcome through the definition's handlers, on this launch +or a later one. Outcomes are journaled natively before JS hears about them +and acknowledged only after the handler's promise resolves. See the README's +"Usage" and "Reliable delivery" sections. "Upgrading from v9" there covers +the migration. + +Breaking: +- **`startUpload` and `getAllUploads` are removed.** `define()` + `mutate()` + replace the first; the synchronous `getRequests(filter?)` replaces the + second. +- **The v9 event names are removed.** `addListener` takes `'state'`, + `'progress'`, or `'attempt'`. Terminal outcomes go to the definition's + `onSuccess` / `onError`; a `cancelled` outcome calls no handler. +- **`getUnacknowledgedEvents` and `ackEvents` are internal.** The library + drains the journal after `configure()` and acknowledges after each handler + settles. +- **`cancelUpload` and `removeUpload` fold into `cancel(id)`.** A live entry + settles `cancelled` and is forgotten after its ack; a settled entry is + forgotten now, row and bytes. +- **`wifiOnly` moves from the upload options to the request descriptor.** + `setWifiOnly(enabled)` on the queue, persisted natively, is the value for + entries that omit it. +- **`progress` carries `{ id, bytesSent, totalBytes }`** instead of a + percentage. +- **`configure()` must be called at boot, after every `define()`.** It starts + the replay of journaled outcomes. Its `android` notification options are + unchanged from v9. +- **`ErrorKind` gains `'truncated'`.** With a `response` parser set and a body + over the 1 MB cap, `onError` fires with it instead of `onSuccess`. +- **Removed exports:** `startUpload`, `cancelUpload`, `removeUpload`, + `getAllUploads`, the public `getUnacknowledgedEvents` / `ackEvents`, the + `error` / `completed` / `cancelled` event names, and the + `ProgressData`, `CompletedData`, `ErrorData`, `CancelledData`, `EventData`, + `TerminalEventData`, `JournaledEvent`, `UploadSnapshot`, `UploadOptions`, + `ChunkedUploadOptions`, `StartUploadOptions`, `AndroidOnlyUploadOptions`, + `RawUploadOptions`, and `UploadId` types. The native `startChunkedUpload` is + internal: a descriptor with `parts` routes to it. + +Added: +- **`createUploadClient()`**: builds a client with its own definitions and + settings. The default export is one client. +- **`define({ key, request, response?, onSuccess?, onError? })`**: `vars` + infer from the `request` parameter, the handler data type from the + `response` return. `response(raw, vars)` also receives the entry's `vars`; + a one-argument parser such as `schema.parse` still fits. Without + `response`, `onSuccess` receives the `RawResponse`. A duplicate key + replaces the definition and warns in development. A `request` that takes + no parameter gives a `mutate()` that takes no arguments. +- **`mutate(vars, { id? })`**: runs `request(vars)` once, merges the configured + headers under the descriptor's, validates the descriptor (at most one of + `data` / `form` / `file`; no body on a GET; `parts` only with `file`; parts + must tile the file from 0; no field outside the descriptor shape, with a + "did you mean" hint), defaults `expiresAt` to now + `lifetimeMs`, and + resolves when the entry is durable: after the row and every staged body + copy are on disk, so the caller may delete its source file then. `vars` is + any JSON-serializable object, so generated API request types work as they + are; `mutate()` rejects vars or `data` that do not serialize (a cycle, a + function, a BigInt) and caps `vars` at `configure().maxVarsBytes`, 1 MB by + default. `id` defaults to a UUID. +- **Same id, again.** The body is `data`, `form`, `file`, or `parts`, and a + different `url` or `method` counts as a different body. Same body: resume, + with new headers, `expiresAt`, and `vars`. Different body on an entry that + is not running: replace and reopen; it settles once more. Different body + on a running entry: reject `E_RUNNING`. Cancelled but not yet acknowledged: + a fresh generation. Completed but not yet acknowledged: the journaled + outcome is emitted again and nothing is re-sent. +- **Request bodies**: JSON (`data`), multipart (`form`), whole file (`file`), + and chunked (`file` + `parts`). All under one entry shape and one id. A + bodiless request (a GET, a DELETE, a POST whose meaning is in the URL) sets + none. `data: null` sends the JSON body `null`. +- **`configure()` options**: `lifetimeMs` (default 14 days), `retry` + (backoff base 1 s, max 2 h, jitter 0.2; `terminalHttp.exempt` default + `[404]`), a `headers` provider that runs at `mutate()`, `maxVarsBytes` + (default 1 MB), `enqueueTimeoutMs` (default 10 s: `mutate()` rejects with + a named error and warns when the native write has not settled by then, so + a native bug cannot hang a caller in silence), and the v9 `android` + notification options. +- **Queue control**: `pause(scope?)` / `resume(scope?)`, `cancel(id)`, + `setWifiOnly(enabled)`, and `updateHeaders(patch)` to re-auth entries + parked on 401 or 403. Each `updateHeaders()` call bumps a header + generation, so a 401 from an attempt sent under older headers re-issues at + once instead of parking. +- **Key-scoped pause.** `pause()` and `resume()` take an optional + `{ keys?: string[] }`. No scope is the whole queue; `{ keys }` pauses or + resumes the entries of those definition keys, queued and future. An entry + is paused while the whole queue or its key is paused, so resuming one + scope does not resume an entry the other still pauses. Both states persist + natively. A misspelled scope field, `keys: undefined`, or an empty key + rejects, so a mistake cannot pause the whole queue. The native `pause` and + `resume` now take a scope object, `{}` for the whole queue, because + codegen has no optional arguments. +- **Per-request `wifiOnly`.** `RequestDescriptor.wifiOnly?: boolean` + overrides `setWifiOnly()` for that entry and is persisted with it. An + entry that omits it follows `setWifiOnly()`, including later toggles. + Both are checked before each attempt. +- **`getRequests(filter?)`**: synchronous, from native's in-memory index. + Returns every entry native has not yet forgotten, so completed and + cancelled rows appear until their ack and `error` rows until `cancel()` or + a same-id `mutate()`. `filter` is `{ key?, id? }`. +- **`RequestRow`**: `{ id, key, vars, state, bytesSent, totalBytes, attempts, + updatedAt, nextAttemptAt? }`. `state` is `queued`, `running`, + `awaiting-auth`, `paused`, `completed`, `error`, or `cancelled`. + `nextAttemptAt` (epoch ms) is set while an entry waits out a retry + backoff. `attempts` counts the current generation. +- **`Meta`** for handlers: `{ id, key, at, attempts, requestId?, deliveries }`. + `deliveries` counts deliveries that reached a JS listener: 1 on the first, + +1 per replay. A handler that keeps throwing sees it grow; the library + never gives up on its own, so the app decides a poison policy. +- **Events**: `state` (a full `RequestRow` per transition, plus + `reason: 'unhandled-key'` for an outcome whose key has no definition), + `progress` (byte-weighted across a chunked upload's parts, throttled to + 1 s in the foreground), and `attempt` (one row per HTTP attempt with + `requestId`, `httpCode`, a 4 KB response body, `responseHeaders`, and + `errorKind`). `attempt.outcome` is `completed` or `error`; pause, cancel, + and supersede emit none. `attempt` events are live-only, never journaled. +- **Delivery rules**: dedupe by event id; the outcomes of one id deliver in + order, one handler at a time; an outcome for an id waits for that id's + in-flight `mutate()`; an outcome whose key has no definition stays + unacknowledged and reaches `state` listeners with `reason: 'unhandled-key'`; + a handler that has not settled after 30 s logs a warning and keeps + waiting; a malformed journal entry is dropped with a warning. No ordering + is promised between different ids. +- **Error codes** on the native promises: `E_INVALID` (input native cannot + send: a non-http(s) URL, a header name or value the platform HTTP client + rejects, a GET with a body, parts that do not tile the file), + `E_RUNNING`, `E_FILE_MISSING`, and `E_STORAGE`. `cancel()` rejects with + `E_STORAGE` and changes nothing when the journal (live entry) or the store + (settled entry) cannot be written. A live cancel whose outcome is journaled + resolves, even if the entry save that follows fails. `updateHeaders()` rejects with `E_INVALID` for a bad header name + or value. +- **`X-Request-Id`** on every attempt, minted per attempt. `Meta.requestId` + and `attempt.requestId` carry it. +- **`chunkPlan`** stays a module export and is also on the client. +- **Testing seam**: `createUploadClient({ native })` takes a fake native + module, and `createFakeNative()` in `src/testing` builds one that keeps + rows, journals outcomes, and acks, with `settle()`, `failNext()`, + `seedRows()`, and `seedUnacknowledged()` to script native behavior. The + real validation and delivery run on top of it. + +Native: +- **Android queue.** One durable store under `files/rnbgupload-chunked/`, + one directory per entry (`entry.json` plus the staged body), written + tmp + fsync + rename. The journal of settled outcomes moves to + `files/rnbgupload-settled/`. Queue settings (`wifiOnly`, `paused`, header + generation, retry defaults) persist next to the entries. An in-memory + index serves `getRequests()`. One WorkManager worker per id; a long + backoff waits on a separate wake job so a same-id `mutate()` or + `updateHeaders()` can run the entry at once. The backoff streak is stored + on the entry, so the wait grows toward 2 h across runs. A run stopped by + WorkManager's timeout takes one more backoff step instead of restarting + at once. The global cap of 4 concurrent requests and the window of 3 + parts per chunked upload carry over from v9. A boot sweep repairs a settle + whose entry save was lost, applies a cancel whose save was lost, and + finishes a pause or resume cut short by process death. A settle whose + journal write fails holds the record in memory and retries the write + (5 s, doubling to 10 min). Response bodies are capped at 1 MB while they + stream, counted in UTF-8 bytes and cut on a character boundary. +- **iOS queue.** The v9 chunked store is generalized in place into the queue + store. Every body is staged to a file in the entry directory, because a + background `URLSession` uploads from files only. Store, journal, and task + map writes are tmp + fsync + rename, and the directories are excluded from + backup. Retries are delayed session tasks (`earliestBeginDate`) for simple + entries and for chunked parts, so a backoff keeps running while the app is + suspended or dead. Each attempt owns its task through the task + description, so a stale completion cannot settle a newer attempt. At + relaunch, an entry whose task is gone but whose completion may still be + pending waits up to 10 s for the replay before it re-issues, so a request + the daemon finished while the app was dead is not sent twice; the + background completion handler is released as soon as that replay lands. + Two background sessions, one that allows cellular and one Wi-Fi only, each + with `httpMaximumConnectionsPerHost = 4`. A system cancel, including a + force-quit, is a transient failure: no outcome, the entry retries. A + multipart body always uses its own `Content-Type`, because a caller's has + no boundary. Response bodies are capped at 1 MB while they stream. The v9 + per-part budget of 3 HTTP rejections is gone; the retry policy decides. +- **v9 import, both platforms.** The first v10 launch imports each v9 journal + entry as a read-only settled row with key `legacy`, the v9 upload id, and + 0/0 bytes. Nothing is delivered for them: the app reads them with + `getRequests({ key: 'legacy' })` and calls `cancel(id)`. In-flight v9 work + is cancelled (Android WorkManager rows, iOS session tasks). v9 chunked + manifests and their bytes stay: a same-id `mutate()` with the same parts + resumes from the accepted parts, and one with different parts starts over + on the kept bytes. Wi-Fi only starts off. +- **Platform limits.** Android, API 31 and later: a WorkManager run started + from the background usually cannot start its foreground service and then + has JobScheduler's limit of about 10 minutes; a single body that does not + finish in that time restarts from byte 0 after a backoff, so large bodies + need `parts`. Android: the store and journal hold auth headers and staged + bodies, so the host app sets `android:allowBackup="false"` or excludes the + two directories. iOS: no global in-flight cap, and the AppDelegate + `handleEventsForBackgroundURLSession` hook is required for outcomes to be + journaled when the system relaunches the app. + +Fixed: +- iOS: a chunked part's retry backoff no longer stalls while the app is + dead. v9 ran the cooldown on an in-process timer. +- iOS: task-map keys whose completion never arrives are pruned at the end of + the relaunch grace wait and when their entry is gone. v9 kept them forever. + +## 9.0.1 + +Fixed: +- **iOS no longer crashes when an upload's file is missing.** A background + session raises an Objective-C exception, not an error, when it cannot read + the file, and Swift cannot catch it. A file deleted after the caller checked + it ended the app. `startUpload` now rejects instead, and a chunked part that + cannot be enqueued journals an `error` with `errorKind: 'file'`. +- **Android 15+ no longer kills the app when the `dataSync` foreground-service + budget runs out.** Android gives `dataSync` services six hours per day in the + background, then calls `onTimeout` and expects the service to stop. + WorkManager 2.9 never answered, so the system threw + `ForegroundServiceDidNotStopInTimeException` and the process died mid-upload. + WorkManager 2.12 stops the timed-out workers itself. They re-run without + foreground priority until the user opens the app, which resets the budget. + Bumps `androidx.work` to 2.12.0 (minSdk 24). + +## 9.0.0 + +Chunked uploads move into the library: one file, many part requests, one upload +id and event stream. The consumer authors the parts (URL, headers, byte range) +once; the library owns transport, the bytes, and resume. See the README's +"Chunked uploads" section. + +Breaking: +- **Listeners are global-only.** `addListener(event, uploadId, callback)` is + gone; use `addListener(event, callback)` and discriminate on `data.id`. +- **`acceptStatus: number[]` is replaced by `accept` rules** on all uploads: + `accept: [{ status, bodyIncludes? }]`. `bodyIncludes` narrows by + response-body substring, for statuses that carry several meanings. +- **`customUploadId` is renamed to `id`** on all upload types. +- **`android.maxRetries` is removed.** Retry policy belongs to the library; + chunked uploads are bounded by `expiresAt` instead. +- **`ios.getUploadStatus` is removed.** Its only use was pre-dispatch dedupe, + which idempotent `startUpload` (below) makes unnecessary. +- **Per-upload notification text is removed.** Set it once with `configure()`; + it is persisted natively so a worker relaunched by WorkManager with no JS + running shows the same text. Per-upload `android` options reduce to + `noNotification`. + +Added: +- **Chunked uploads**: `startUpload({ type: 'chunked', id, path, + parts, accept?, expiresAt, wifiOnly? })`. The library takes ownership of the + file at `startUpload` and deletes it only after a `completed` event is + acknowledged; every other terminal outcome keeps the manifest and bytes for + resume or recreate. +- **`startUpload` is idempotent for every upload, always.** Re-calling with a + running id is never an error; for chunked uploads it reconciles — accepted + parts are skipped, the rest continue with the new call's headers (how a fresh + auth token reaches parts stalled on 401). On iOS the guarantee is race-free: + concurrent same-id calls are serialized natively, so they can never enqueue a + duplicate task. +- **Recreate under the same id**: calling `startUpload` with an existing id and + a *different* parts array replaces the parts over the owned bytes — every + part resets to unsent, and the new ranges must tile the same total size. + Accepted only while the upload is not running (stalled on a terminal error, + expired, or cancelled); rejected while it runs. This is how a consumer + re-uploads under a fresh server uploadId after the old one dies. +- **`expiresAt` / `errorKind: 'expired'`**: a chunked upload past its required + deadline journals a terminal `error` and stops, keeping the bytes. Within the + deadline, transient failures retry on backoff with no attempt cap. +- **`removeUpload(uploadId)`** releases a kept manifest and bytes. +- **`configure(options)`** one-time setup (Android notification text/identity). +- **`chunkPlan(sizeBytes, { min?, max? })`**: the deterministic range splitter, + exposed so the part count told to the server and the parts array derive from + one result. +- Concurrency: on Android a hard cap of 4 concurrent requests across all + uploads — every request passes the shared transfer semaphore (previously + fully serial), with a chunked upload's parts windowed inside it. On iOS a + per-session connection-level backstop: `httpMaximumConnectionsPerHost = 4` + (previously 1), with chunked parts bounded by the per-upload window of 3. + +Fixed: +- Android jobs enqueued by a v8 build replay safely after upgrading. WorkManager + can hand a v9 worker a job serialized by v8 (`acceptStatus`, no `accept`); + the worker now normalizes the legacy shape instead of crashing after the file + has fully transmitted and re-sending it on every retry. + +## 8.1.0 + +Added `android.noNotification`, which uploads a file without posting a progress +notification. An app that uploads housekeeping payloads alongside user-visible +ones can now keep the notification shade for the ones a user asked to watch. + +The notification doubles as the upload worker's foreground-service notification, +so a silent upload runs as an ordinary background worker: the OS may defer it, or +stop it mid-flight for WorkManager to re-run. It suits small payloads a restart +costs nothing; media files should keep their notification. Default is `false`, so +existing uploads are unaffected, including jobs enqueued by 8.0.0 and re-run +after the upgrade. + +## 8.0.0 + +Reliability release. Terminal outcomes are now durable and accurately typed, the +iOS module was rewritten in Swift, and the module is a New Architecture +TurboModule. + +Breaking: +- **New Architecture only.** The module is now a codegen TurboModule on both + platforms; the legacy bridge (`RCTEventEmitter` / `RCT_EXTERN_MODULE` on iOS, + `ReactPackage` + `RCTDeviceEventEmitter` on Android) is gone, along with any + reliance on RN's legacy-interop layer. Requires React Native >= 0.84 with the + New Architecture enabled, and React >= 19. +- The iOS background-session handler moved from `RNFileUploader` to + `RNBackgroundUpload`: `[RNBackgroundUpload setBackgroundSessionCompletionHandler: + forIdentifier:]`. `RNFileUploader` is now the TurboModule and is intentionally + unreachable from plain Objective-C (its generated header is Objective-C++ only). + Update the AppDelegate snippet — see README. +- Events are delivered through the codegen event emitters rather than + `DeviceEventEmitter`, so they are no longer visible under the raw + `RNFileUploader-*` device-event names. The `Upload.addListener(...)` API is + unchanged. +- `cancelUpload` on iOS now resolves `false` when no matching in-flight upload was + found (it previously always resolved `true`). Android still always resolves `true`. +- Terminal event payloads are typed as the journal entry they actually are. The + natives emit the journaled entry itself, so `CompletedData` / `ErrorData` / + `CancelledData` now declare the `eventId`, `type` and `timestamp` they were always + sending, plus `responseBodyTruncated`. `eventId` in particular means you can + `ackEvents([eventId])` straight after handling a live event. `JournaledEvent` is + now a union discriminated on `type`. +- `CompletedData.responseCode` and `.responseBody` are optional. They were declared + required but are absent when a task completes without an HTTP response, so reading + them unguarded could throw. +- The `cancelled` payload no longer carries `error` (it used to hold the cancellation + error string). Use `cancelReason` instead. +- iOS `progress` reports `0` instead of `-1` when the total length is unknown, + matching Android and the documented 0-100 range. +- iOS `getAllUploads` reports `cancelled` and `completed` states instead of + collapsing everything non-running into `pending`. +- `completed` fires only for 2xx responses (plus a request's `acceptStatus`, e.g. + `acceptStatus: [409]`). Every other HTTP response now emits an `error` with + `errorKind: 'http'` and the full response attached (previously reported as + `completed`). +- `error` events are typed: `errorKind: 'http' | 'network' | 'file' | 'unknown'`. +- Native module renamed to `RNFileUploader` on both platforms (was + `VydiaRNFileUploader` on iOS); Android package is now `ai.openspace.backgroundupload`. +- iOS AppDelegate must forward `handleEventsForBackgroundURLSession` (see README). +- Removed the committed `lib/` build output; types are served from `src` + (deep imports of `lib/*` break — import from the package root). +- Minimum iOS deployment target is 15.1; minimum Android SDK is 29. Minimum React + Native is 0.84 (New Architecture), minimum React is 19. +- Removed non-functional iOS code paths: multipart, `assets-library://`, and the + `appGroup` option (a no-op even before this — it mutated the session config after + creation, which URLSession ignores; also removed from the TypeScript options). +- Removed the unexposed Android `stopAllUploads`. + +Added: +- Durable native event journal: `getUnacknowledgedEvents()` / `ackEvents(ids)` — + terminal events survive app death and JS reloads (at-least-once delivery). +- `getAllUploads()` on both platforms. +- `cancelled` events carry `cancelReason: 'user' | 'system'`. +- `responseHeaders` on completed events on iOS (was Android-only). +- iOS progress events throttled to 500ms; UUID default upload ids. +- Android `android` options are now optional — sensible notification defaults and + a library-created notification channel. + +Earlier releases: see the [releases](https://github.com/openspacelabs/react-native-background-upload/releases) page. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bde7798c..7dd49910 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,8 +1,5 @@ ## Issues -Along with a bug report, provide a functional example repository which -reproduces the bug you're experiencing. We've made this very easy by providing -a full-stack (React Native + Express.js) example app, which you can fork and -alter to reproduce your bug: - -[ReactNativeBackgroundUploadExample](https://github.com/Vydia/ReactNativeBackgroundUploadExample) +Along with a bug report, provide a functional reproduction using the full-stack +(React Native + Express.js) example app in this repo under `example/` — fork and +alter it to reproduce your bug. diff --git a/README.md b/README.md index 8475adb3..c6834455 100644 --- a/README.md +++ b/README.md @@ -1,365 +1,585 @@ -# react-native-background-upload [![npm version](https://badge.fury.io/js/react-native-background-upload.svg)](https://badge.fury.io/js/react-native-background-upload) ![GitHub Actions status](https://github.com/Vydia/react-native-background-upload/workflows/Test%20iOS%20Example%20App/badge.svg) ![GitHub Actions status](https://github.com/Vydia/react-native-background-upload/workflows/Test%20Android%20Example%20App/badge.svg) [![semantic-release](https://img.shields.io/badge/%20%20%F0%9F%93%A6%F0%9F%9A%80-semantic--release-e10079.svg)](https://github.com/semantic-release/semantic-release) - -The only React Native http post file uploader with android and iOS background support. If you are uploading large files like videos, use this so your users can background your app during a long upload. - -NOTE: Use major version 4 with RN 47.0 and greater. If you have RN less than 47, use 3.0. To view all available versions: -`npm show react-native-background-upload versions` +# react-native-background-upload +OpenSpace's background HTTP file uploader for React Native. On iOS it uses a +background `URLSession`; on Android it uses `WorkManager` (a `CoroutineWorker`) +with OkHttp. Uploads continue while the app is backgrounded and resume after it +is killed. # Installation -## 1. Install package - -`npm install --save react-native-background-upload` - -or - -`yarn add react-native-background-upload` - -Note: if you are installing on React Native < 0.47, use `react-native-background-upload@3.0.0` instead of `react-native-background-upload` - -## 2. Link Native Code - -### Autolinking (React Native >= 0.60) - -##### iOS - -`cd ./ios && pod install && cd ../` - -##### Android - -###### ProGuard -Add this to your ProGuard configuration `(proguard-rules.pro)` : - -`-keep class net.gotev.uploadservice.** { *; }` +**Requires React Native ≥ 0.84 with the New Architecture enabled, and React ≥ 19.** +This is a codegen TurboModule; it does not support the legacy bridge. -### Automatic Native Library Linking (React Native < 0.60) - -`react-native link react-native-background-upload` - -### Or, Manually Link It - -#### iOS - -1. In the XCode's "Project navigator", right click on your project's Libraries folder ➜ `Add Files to <...>` -2. Go to `node_modules` ➜ `react-native-background-upload` ➜ `ios` ➜ select `VydiaRNFileUploader.xcodeproj` -3. Add `VydiaRNFileUploader.a` to `Build Phases -> Link Binary With Libraries` - -#### Android -1. Add the following lines to `android/settings.gradle`: - - ```gradle - include ':react-native-background-upload' - project(':react-native-background-upload').projectDir = new File(settingsDir, '../node_modules/react-native-background-upload/android') - ``` -2. Add the compile and resolutionStrategy line to the dependencies in `android/app/build.gradle`: - - ```gradle - configurations.all { resolutionStrategy.force 'com.squareup.okhttp3:okhttp:3.4.1' } // required by react-native-background-upload until React Native supports okhttp >= okhttp 3.5 - - dependencies { - compile project(':react-native-background-upload') - } - ``` +``` +yarn add react-native-background-upload +cd ios && pod install && cd .. +``` +`pod install` is required after installing. It runs codegen to generate the native +spec this module implements. -3. Add the import and link the package in `MainApplication.java`: +> The package ships TypeScript source with no build step, so it resolves through Metro +> (and `tsc`) but not through plain Node. If you import it from a non-Metro context, +> such as a script or Jest without a transform, add it to your `transformIgnorePatterns` +> allowlist or mock it. - ```java - import com.vydia.RNUploader.UploaderReactPackage; <-- add this import +## iOS: background completion handler (required) - public class MainApplication extends Application implements ReactApplication { - @Override - protected List getPackages() { - return Arrays.asList( - new MainReactPackage(), - new UploaderReactPackage() // <-- add this line - ); - } - } - ``` +So uploads that finish while the app is terminated can relaunch it and be +journaled, add this to your `AppDelegate`: -4. Ensure Android SDK versions. Open your app's `android/app/build.gradle` file. Ensure `compileSdkVersion` and `targetSdkVersion` are 25. Otherwise you'll get compilation errors. +```objc +// AppDelegate.m (Objective-C). Diana uses this form. +@import react_native_background_upload; -## 3. Expo +- (void)application:(UIApplication *)application +handleEventsForBackgroundURLSession:(NSString *)identifier + completionHandler:(void (^)(void))completionHandler { + [RNBackgroundUpload setBackgroundSessionCompletionHandler:completionHandler + forIdentifier:identifier]; +} +``` -To use this library with [Expo](https://expo.io) one must first detach (eject) the project and follow [step 2](#2-link-native-code) instructions. Additionally on iOS there is a must to add a Header Search Path to other dependencies which are managed using Pods. To do so one has to add `$(SRCROOT)/../../../ios/Pods/Headers/Public` to Header Search Path in `VydiaRNFileUploader` module using XCode. +> CocoaPods wires the module map for the app target, so `@import` works in an +> Objective-C `.m` file with the default static-library setup. If your +> AppDelegate is Objective-C++ (`.mm`), `@import` is not available. Add +> `"$(PODS_CONFIGURATION_BUILD_DIR)/react-native-background-upload/Swift Compatibility Header"` +> to the app target's `HEADER_SEARCH_PATHS`, then use +> `#import ` followed by +> `#import "react_native_background_upload-Swift.h"`. The example app's +> `AppDelegate.mm` does this. + +This hook is load-bearing beyond just calling the completion handler: it is what +brings the library's background `URLSession` back to life in a process the system +relaunched with no JS running, so queued completions get journaled. `RNFileUploader` +is the TurboModule and is deliberately not reachable from plain Objective-C, because its +generated header is Objective-C++ only. So the handler lives on `RNBackgroundUpload`. # Usage -```js -import Upload from 'react-native-background-upload' - -const options = { - url: 'https://myservice.com/path/to/post', - path: 'file://path/to/file/on/device', - method: 'POST', - type: 'raw', - maxRetries: 2, // set retry count (Android only). Default 2 - headers: { - 'content-type': 'application/octet-stream', // Customize content-type - 'my-custom-header': 's3headervalueorwhateveryouneed' +The library owns a durable queue of HTTP requests: JSON bodies, multipart +forms, whole files, and chunked files. You describe each request kind one +time with `define()`, enqueue instances with `mutate()`, and receive every +outcome through the definition's handlers. Outcomes survive app death, +because the native side journals them before it tells JS. + +```ts +import { createUploadClient } from 'react-native-background-upload'; + +export const uploads = createUploadClient(); + +// One definition per request kind. `request` runs one time, at mutate(). +// `vars` is any JSON-serializable object, at most 1 MB by default; native +// persists it next to the entry. Generated API request types work as they are. +type AddCommentVars = { siteId: string; noteId: string; comment: string }; +export const addComment = uploads.define({ + key: 'note.comment.add', // persisted with every entry; rename with care + request: ({ siteId, noteId, comment }: AddCommentVars) => ({ + url: `https://api.example.com/sites/${siteId}/notes/${noteId}/comments`, + data: { comment }, // JSON body. Default method is POST. + }), + // Parses the JSON body before onSuccess. Zod users pass schema.parse. The + // parser also receives the entry's vars as a second argument. + response: (raw) => (raw as { content: Comment[] }).content, + onSuccess: (content, { noteId }, meta) => { + // Runs after the server accepted the request, possibly on a later launch. + store.dispatch(commentsLoaded({ noteId, content })); }, - // Below are options only supported on Android - notification: { - enabled: true + onError: (error, vars, meta) => { + // error.errorKind: 'http' | 'network' | 'file' | 'expired' | 'truncated' | 'unknown' }, - useUtf8Charset: true -} - -Upload.startUpload(options).then((uploadId) => { - console.log('Upload started') - Upload.addListener('progress', uploadId, (data) => { - console.log(`Progress: ${data.progress}%`) - }) - Upload.addListener('error', uploadId, (data) => { - console.log(`Error: ${data.error}%`) - }) - Upload.addListener('cancelled', uploadId, (data) => { - console.log(`Cancelled!`) - }) - Upload.addListener('completed', uploadId, (data) => { - // data includes responseCode: number and responseBody: Object - console.log('Completed!') - }) -}).catch((err) => { - console.log('Upload error!', err) -}) +}); + +// At boot, after every define() call. Replay of journaled outcomes starts here. +uploads.configure({ + headers: () => ({ Authorization: `Bearer ${currentToken()}` }), + android: { notificationTitle: 'Uploading', notificationChannel: 'uploads' }, +}); + +// Anywhere. Resolves when the entry and its staged body are on disk, never +// on the network. You may delete a source file after this resolves. +const { id } = await addComment.mutate( + { siteId, noteId, comment }, + { id: localCommentId }, // optional; makes a re-dispatch idempotent +); ``` -## Multipart Uploads - -Just set the `type` option to `multipart` and set the `field` option. Example: - -``` -const options = { - url: 'https://myservice.com/path/to/post', - path: 'file://path/to/file%20on%20device.png', - method: 'POST', - field: 'uploaded_media', - type: 'multipart' -} +Handlers must be idempotent. The library acknowledges an outcome only after +the handler's promise resolves, so a crash before that point redelivers the +outcome at the next launch. + +## The request descriptor + +`request(vars)` returns a plain object. Set at most one body kind. A GET sets +none, and `mutate()` rejects a GET with a body. A DELETE, or a POST whose +meaning is in the URL, may also set none. +A field outside this table makes `mutate()` reject and name the field, because +TypeScript does not flag a misspelled key on an inferred arrow return. + +| Field | Notes | +| --- | --- | +| `url` | Required unless `parts` is set. | +| `method` | `POST` (default), `PUT`, `PATCH`, `DELETE`, `GET`. With `parts` it applies to every part. | +| `headers` | Merged over `configure().headers()`, names matched without regard to case. Every chunked part inherits the result. | +| `data` | JSON body. Any JSON-serializable value. `null` sends the JSON body `null`; omit `data` for no body. | +| `form` | `multipart/form-data`: `[{ name, contentType, string }]` or `[{ name, contentType, path, fileName? }]`. File parts are copied. | +| `file` | Whole file body. Copied. Moved when `parts` is set. | +| `parts` | Chunked over `file`: `[{ url, headers?, range: { start, end } }]`, bytes, end exclusive, tiling the file from 0. | +| `accept` | Non-2xx responses to treat as success: `[{ status, bodyIncludes? }]`. | +| `expiresAt` | Epoch ms. Default now + `lifetimeMs` (14 days). Past it: `error` with `errorKind: 'expired'`. | +| `retry` | Per-request override of the `configure()` retry defaults. | +| `wifiOnly` | `true` waits for Wi-Fi before each attempt; `false` never waits. Overrides `setWifiOnly()` for this entry. Omit it to follow `setWifiOnly()`, including later toggles. | +| `android` | `{ noNotification?: boolean }`. See Silent uploads. | + +`vars` and `data` cross to native as JSON strings, and native parses them. +React Native on iOS drops object keys whose value is `null`, so a string is +the only form in which `{ status: null }` arrives intact. + +### Chunked uploads + +A descriptor with `file` and `parts` sends one file as many part requests but +stays one entry: one id, byte-weighted `progress`, and `completed` only when +every part is accepted. Author the ranges with `chunkPlan` so the part count +you tell your server and the parts the library sends derive from one array. + +```ts +type CaptureFileVars = { path: string; size: number; uploadId: string }; + +const captureFile = uploads.define({ + key: 'capture.file', + request: ({ path, size, uploadId }: CaptureFileVars) => { + const ranges = uploads.chunkPlan(size, { min: 8 * 2 ** 20, max: 20 * 2 ** 20 }); + return { + method: 'PUT', + file: path, // moved into the library directory + // Derive each part URL from its index. The part count you tell the + // server and the parts sent here then come from the one chunkPlan call. + parts: ranges.map((range, i) => ({ + url: partUrl(uploadId, i + 1), // part numbers are 1-indexed + headers: { 'Content-Range': `${range.start}-${range.end - 1}/${size}` }, + range, + })), + accept: [{ status: 409, bodyIncludes: 'already completed' }], + // A 404 on a part means the server-side multipart is gone. Make it + // terminal so onError can recreate under a fresh server upload id. + retry: { terminalHttp: { exempt: [] } }, + }; + }, + onError: (error, vars) => { /* recreate: mutate() again with new parts */ }, +}); ``` -Note the `field` property is required for multipart uploads. +**File ownership.** A chunked `file` is moved into the library's directory at +`mutate()`; a single `file` body and every `form` part path are copied. Bytes +are deleted after a `completed` outcome is acknowledged, or on `cancel()`. +Nothing else deletes them. + +**Same id, again.** The body is `data`, `form`, `file`, or `parts`, and a +different `url` or `method` counts as a different body. `mutate()` with an +id that exists: + +- Same body: resume. New headers, `expiresAt`, and `vars` replace the stored + ones. +- Different body (`data`, `form`, `file`, or `parts`) and the entry is not + running (queued, paused, awaiting-auth, or settled): the descriptor, staged + body, `vars`, headers, and `expiresAt` are replaced and the entry reopens. + It settles once more. +- Different body, entry running: `mutate()` rejects with `E_RUNNING`. +- A cancelled entry whose outcome is not yet acknowledged: a fresh + generation. The old outcome's ack forgets the entry only when the + generation matches. +- A completed entry whose outcome is not yet acknowledged: the journaled + outcome is emitted again. The request does not run again. + +### Silent uploads (Android) + +`android: { noNotification: true }` runs the request without a progress +notification. That notification is also the worker's foreground-service +notification, so a silent request runs as an ordinary background worker and +the OS may defer or restart it. Reserve it for small payloads. + +### Android platform notes + +**Headless time limit.** On API 31 and later, a WorkManager run that starts +from the background usually cannot start its foreground service. The run +then has JobScheduler's limit of about 10 minutes. A single body (`data`, +`form`, `file`) that does not finish in that time starts again from byte 0 +at the next run, after a growing backoff. A body that needs more than +10 minutes headless cannot finish that way. Use `parts` for large bodies: +accepted parts are kept across runs. iOS has no equal limit. + +**Backups.** The queue store (`files/rnbgupload-chunked/`) and the journal +(`files/rnbgupload-settled/`) hold request headers, including auth tokens, +and staged bodies. Set `android:allowBackup="false"` in the host app, or +exclude those two directories in its backup rules. + +### iOS platform notes + +**Bodies are files.** A background `URLSession` uploads from files only, so +the library writes every `data` and `form` body to a file in its own +directory at `mutate()`. The bytes stay there until the entry is forgotten. +The store and journal are excluded from iCloud and iTunes backups. + +**No global cap.** iOS has no hard limit on requests in flight. Each of the +two background sessions (cellular allowed, and Wi-Fi only) sets +`httpMaximumConnectionsPerHost = 4`. That caps connections to one host, not +requests: over HTTP/2, every queued part can be in flight at once. + +**Every part is queued at once.** A chunked upload creates a task for every +part at `mutate()`. Each part task reads its own part file, so a chunked +upload needs the file's size again in free space until its parts are sent. + +**Not at the system's discretion.** Both sessions set +`isDiscretionary = false`, but iOS only honors that for a task created while +the app is in the foreground. It starts a task created in the background at +its own discretion, for example when the device charges. Tasks are often +created in the background: a retry scheduled while the app is there, a +`mutate()` from a background wake, and the tasks a relaunch rebuilds. So the library records whether the app was active when it created +each task. Two seconds after the app becomes active, it replaces every task +created while the app was not active, unless the task has sent a byte of its +body or has a response. The delay lets the system report what the task did +while the app was suspended. A simple request's replacement keeps the +attempt's request id; a part's replacement is a new attempt. Either keeps +what is left of a backoff. A request with an empty body is never replaced, +because it cannot show that it started. Tasks created by an earlier version +of the library count as background tasks, so the first activation after an +upgrade replaces those too. + +Two limits remain. A task created in the background can wait for the system +until the app next becomes active. And if the system has sent a body without +reporting it yet, the replacement sends it again. The server sees the same +`X-Request-Id` for a simple request, and either copy that is accepted +completes the entry. + +**Force-quit.** When the user swipes the app away, iOS cancels the session's +tasks. The library treats that as a transient failure: no outcome is +produced, the entry stays `queued` or `running`, and it is sent again at the +next launch. Accepted chunked parts are kept. A suspension or a system +termination is different: the tasks keep running, and the AppDelegate hook +above lets the library journal their outcomes. + +**Relaunch.** When the app comes back and an entry's task is gone but a +completion may still be in flight from the daemon, the library waits up to +10 s for it before it sends again. This is what stops a request that finished +while the app was dead from being sent twice. + +# Reliable delivery + +1. **Write-ahead.** Entry, descriptor, and staged body persist before any + attempt. `mutate()` resolves after the row and every staged body copy are + durably on disk (temp file plus rename), so the caller may delete its + source file then. A native failure rejects with a code: `E_INVALID`, + `E_RUNNING`, `E_FILE_MISSING`, or `E_STORAGE`. `E_INVALID` is malformed + input native cannot send: a non-http(s) URL, header names or values the + platform HTTP client rejects, a GET with a body, or parts that do not + tile the moved file. +2. **Journal before emit, ack after the handler.** Every terminal outcome is + journaled natively, then delivered. The library acknowledges after the + handler's promise resolves. A rejection, or app death before the ack, + redelivers at the next launch. A handler that has not settled after 30 s + gets a console warning and keeps waiting. `Meta.deliveries` counts + deliveries that reached a JS listener: 1 on the first, +1 per replay. An + outcome journaled while no listener exists starts at 0 and is not emitted + live. A handler that keeps throwing sees the number grow. The library + never gives up on its own; the app decides a poison policy from that + number. +3. **One outcome per settle cycle.** `pause()` produces none, for the whole + queue or for a set of keys. A paused entry past `expiresAt` settles + `error` with `errorKind: 'expired'` when it is resumed. A same-id + `mutate()` on a settled entry reopens it, and it settles once more. +4. **Never before `mutate()` resolves.** Delivery for an id waits for the + caller's promise. +5. **Replay starts after `configure()`.** Outcomes journaled by a dead session + deliver then. Call every `define()` first. +6. **Unknown key is loud.** An outcome whose key has no definition stays + unacknowledged and reaches `state` listeners with `reason: 'unhandled-key'`. +7. **Completed entries are forgotten after ack.** Row and bytes go. An `error` + or `expired` entry keeps both until `cancel()` or a same-id `mutate()`. +8. **Ordering holds per id only.** The outcomes of one id are delivered in + order. There is no ordering guarantee between different ids. +9. **`attempt` events are live-only.** They are never journaled or replayed. + +Retry classes: + +| Attempt outcome | Action | +| --- | --- | +| 2xx, or an `accept` rule matches | settle `completed` | +| network failure, 5xx, 408, 429 | exponential backoff (base 1 s, max 2 h, jitter 0.2) until `expiresAt` | +| 401, 403 | park as `awaiting-auth`; resume on `updateHeaders()` | +| other 4xx not in `retry.terminalHttp.exempt` | settle `error` with `errorKind: 'http'` | +| 4xx in `exempt` (default `[404]`) | as transient | +| payload missing on disk | settle `error` with `errorKind: 'file'` | +| `expiresAt` passed | settle `error` with `errorKind: 'expired'`; bytes kept | +| response body over 1 MB with a `response` parser | `onError` with `errorKind: 'truncated'`; the entry is `completed` | + +Every attempt sends an `X-Request-Id` header, minted per attempt. `Meta.requestId` +carries the last one. # API -## Top Level Functions - -All top-level methods are available as named exports or methods on the default export. - -### startUpload(options) - -The primary method you will use, this starts the upload process. - -Returns a promise with the string ID of the upload. Will reject if there is a connection problem, the file doesn't exist, or there is some other problem. - -`options` is an object with following values: - -*Note: You must provide valid URIs. react-native-background-upload does not escape the values you provide.* - -|Name|Type|Required|Default|Description|Example| -|---|---|---|---|---|---| -|`url`|string|Required||URL to upload to|`https://myservice.com/path/to/post`| -|`path`|string|Required||File path on device|`file://something/coming/from%20the%20device.png`| -|`type`|'raw' or 'multipart'|Optional|`raw`|Primary upload type.|| -|`method`|string|Optional|`POST`|HTTP method|| -|`customUploadId`|string|Optional||`startUpload` returns a Promise that includes the upload ID, which can be used for future status checks. By default, the upload ID is automatically generated. This parameter allows a custom ID to use instead of the default.|| -|`headers`|object|Optional||HTTP headers|`{ 'Accept': 'application/json' }`| -|`field`|string|Required if `type: 'multipart'`||The form field name for the file. Only used when `type: 'multipart`|`uploaded-file`| -|`parameters`|object|Optional||Additional form fields to include in the HTTP request. Only used when `type: 'multipart`|| -|`notification`|Notification object (see below)|Optional||Android only. |`{ enabled: true, onProgressTitle: "Uploading...", autoClear: true }`| -|`useUtf8Charset`|boolean|Optional||Android only. Set to true to use `utf-8` as charset. || -|`appGroup`|string|Optional|iOS only. App group ID needed for share extensions to be able to properly call the library. See: https://developer.apple.com/documentation/foundation/nsfilemanager/1412643-containerurlforsecurityapplicati - -### Notification Object (Android Only) -|Name|Type|Required|Description|Example| -|---|---|---|---|---| -|`enabled`|boolean|Optional|Enable or diasable notifications. Works only on Android version < 8.0 Oreo. On Android versions >= 8.0 Oreo is required by Google's policy to display a notification when a background service run|`{ enabled: true }`| -|`autoClear`|boolean|Optional|Autoclear notification on complete|`{ autoclear: true }`| -|`notificationChannel`|string|Optional|Sets android notificaion channel|`{ notificationChannel: "My-Upload-Service" }`| -|`enableRingTone`|boolean|Optional|Sets whether or not to enable the notification sound when the upload gets completed with success or error|`{ enableRingTone: true }`| -|`onProgressTitle`|string|Optional|Sets notification progress title|`{ onProgressTitle: "Uploading" }`| -|`onProgressMessage`|string|Optional|Sets notification progress message|`{ onProgressMessage: "Uploading new video" }`| -|`onCompleteTitle`|string|Optional|Sets notification complete title|`{ onCompleteTitle: "Upload finished" }`| -|`onCompleteMessage`|string|Optional|Sets notification complete message|`{ onCompleteMessage: "Your video has been uploaded" }`| -|`onErrorTitle`|string|Optional|Sets notification error title|`{ onErrorTitle: "Upload error" }`| -|`onErrorMessage`|string|Optional|Sets notification error message|`{ onErrorMessage: "An error occured while uploading a video" }`| -|`onCancelledTitle`|string|Optional|Sets notification cancelled title|`{ onCancelledTitle: "Upload cancelled" }`| -|`onCancelledMessage`|string|Optional|Sets notification cancelled message|`{ onCancelledMessage: "Video upload was cancelled" }`| - - -### getFileInfo(path) - -Returns some useful information about the file in question. Useful if you want to set a MIME type header. - -`path` is a string, such as `file://path.to.the.file.png` - -Returns a Promise that resolves to an object containing: - -|Name|Type|Required|Description|Example| -|---|---|---|---|---| -|`name`|string|Required|The file name within its directory.|`image2.png`| -|`exists`|boolean|Required|Is there a file matching this path?|| -|`size`|number|If `exists`|File size, in bytes|| -|`extension`|string|If `exists`|File extension|`mov`| -|`mimeType`|string|If `exists`|The MIME type for the file.|`video/mp4`| - -### cancelUpload(uploadId) - -Cancels an upload. - -`uploadId` is the result of the Promise returned from `startUpload` - -Returns a Promise that resolves to an boolean indicating whether the upload was cancelled. +`createUploadClient()` returns a client. The default export is one client; +an app needs one. -### addListener(eventType, uploadId, listener) +### `define(definition): { key, mutate }` -Adds an event listener, possibly confined to a single upload. - -`eventType` Event to listen for. Values: 'progress' | 'error' | 'completed' | 'cancelled' - -`uploadId` The upload ID from `startUpload` to filter events for. If null, this will include all uploads. - -`listener` Function to call when the event occurs. - -Returns an [EventSubscription](https://github.com/facebook/react-native/blob/master/Libraries/vendor/emitter/EmitterSubscription.js). To remove the listener, call `remove()` on the `EventSubscription`. - -## Events - -### progress - -Event Data - -|Name|Type|Required|Description| -|---|---|---|---| -|`id`|string|Required|The ID of the upload.| -|`progress`|0-100|Required|Percentage completed.| - -### error - -Event Data - -|Name|Type|Required|Description| -|---|---|---|---| -|`id`|string|Required|The ID of the upload.| -|`error`|string|Required|Error message.| - -### completed - -Event Data - -|Name|Type|Required|Description| -|---|---|---|---| -|`id`|string|Required|The ID of the upload.| -|`responseCode`|string|Required|HTTP status code received| -|`responseBody`|string|Required|HTTP response body| -|`responseHeaders`|string|Required|HTTP response headers (Android)| - -### cancelled - -Event Data - -|Name|Type|Required|Description| -|---|---|---|---| -|`id`|string|Required|The ID of the upload.| - -# Customizing Android Build Properties -You may want to customize the `compileSdk, buildToolsVersion, and targetSdkVersion` versions used by this package. For that, add this to `android/build.gradle`: - -``` -ext { - targetSdkVersion = 23 - compileSdkVersion = 23 - buildToolsVersion = '23.0.2' -} +```ts +type Definition = + | { + key: string; + request: (vars: V) => RequestDescriptor; + response: (raw: unknown, vars: V) => T; // JSON-parsed body, or undefined when there is none + onSuccess?: (data: T, vars: V, meta: Meta) => void | Promise; + onError?: (error: OutcomeError, vars: V, meta: Meta) => void | Promise; + } + | { + key: string; + request: (vars: V) => RequestDescriptor; + response?: undefined; + onSuccess?: (data: RawResponse, vars: V, meta: Meta) => void | Promise; + onError?: (error: OutcomeError, vars: V, meta: Meta) => void | Promise; + }; ``` -Add it above `allProjects` and you're good. Your `android/build.gradle` might then resemble: +`V` infers from the `request` parameter annotation, `T` from the `response` +return type. `response` also receives the entry's `vars`, for a parser that +needs the request context; a one-argument parser such as `schema.parse` is +assignable as it is. Without `response`, `onSuccess` receives the `RawResponse` +(`{ status?, headers?, body?, bodyTruncated }`), and an `onSuccess` annotated +with any other type is a compile error. `V` is any object or `null`, so a +generated API request type works as it is. `mutate()` rejects vars and +`data` that do not serialize: a cycle, a function, a BigInt, or a value that +`JSON.stringify` turns into a primitive. Methods on a class instance are +dropped. A `request` that declares no parameter gives `V = null`, and +`mutate()` then takes no arguments. When `response` is set and +the body was truncated, `onError` gets `errorKind: 'truncated'`. When +`response` throws, `onError` gets `errorKind: 'unknown'` with the thrown +message; the entry still settles as completed. A key that is already defined +is replaced, with a warning in development. A `cancelled` outcome calls no +handler. + +`Meta` is `{ id, key, at, attempts, requestId?, deliveries }`; `at` is the +native outcome time. `attempts` counts attempts in the current generation; a +same-id `mutate()` that reopens the entry starts a new one. `deliveries` +counts deliveries that reached a JS listener: 1 on the first, +1 per replay. +An outcome journaled while no listener exists starts at 0 and is not emitted +live, so its first delivery, at the boot replay, is 1. + +### `mutate(vars, { id? }): Promise<{ id }>` + +Runs `request(vars)` once, merges `configure().headers()` under the +descriptor's headers, validates the descriptor, defaults `expiresAt`, and +persists the entry. Resolves with the id after the row and every staged body +copy are on disk. Rejects on a malformed descriptor, an unknown descriptor +field, a GET with a body, input native cannot send (`E_INVALID`), a missing +file (`E_FILE_MISSING`), a storage failure (`E_STORAGE`), a running entry +with a different body (`E_RUNNING`), or `vars` over +`maxVarsBytes` (1 MB by default). Only `vars` are capped. `id` defaults to a UUID. For a definition +whose `request` takes no vars, call `mutate()` with no arguments; native stores +`null`. + +### `configure(options): void` + +Call one time at boot, after every `define()`. Starts replay of journaled +outcomes. A second call updates the settings and does not replay again. + +| Option | Notes | +| --- | --- | +| `lifetimeMs` | Default `expiresAt` distance. Default 14 days. | +| `maxVarsBytes` | Cap on the JSON length of `vars`. Default 1 MB. `mutate()` rejects above it. JS-side only. | +| `retry` | `{ backoff?: { baseMs, maxMs, jitter }, terminalHttp?: { exempt } }`. Each of the two objects is optional, but one you give must be complete. Defaults 1 s, 2 h, 0.2, `[404]`. | +| `headers` | `() => Record`, called at `mutate()`. The descriptor merges over it. | +| `enqueueTimeoutMs` | Default 10 s. `mutate()` rejects and warns when native enqueue has not settled by then. Enqueue includes the time to stage a copy of a `file` body and of form `path` parts, so a large file takes longer. A timeout means native did not answer, not that the request failed. A watchdog for a native bug, not a tuning knob. | +| `android` | Notification text and identity: `notificationId/Title/TitleNoWifi/TitleNoInternet/Channel`. Persisted natively. | + +### `pause(scope?): Promise` and `resume(scope?): Promise` +`scope` is `{ keys?: string[] }`. No scope pauses or resumes the whole queue. +`{ keys }` pauses or resumes the entries of those definition keys, queued and +future. An empty `keys` list changes nothing. A scope field other than `keys`, +`keys: undefined`, or a key that is not a non-empty string rejects, so a +mistake cannot pause the whole queue. + +An entry is paused while the whole queue is paused or its key is paused. A +resume of one scope does not resume an entry that the other still pauses: + +```ts +await Upload.pause({ keys: ['capture.upload'] }); // captures wait, notes run +await Upload.pause(); // everything waits +await Upload.resume(); // notes run, captures still wait +await Upload.resume({ keys: ['capture.upload'] }); // captures run ``` -buildscript { - repositories { - jcenter() - } -} - -ext { - targetSdkVersion = 27 - compileSdkVersion = 27 - buildToolsVersion = '23.0.2' -} - -allprojects { +Both states are persisted natively. Each entry that moves emits one `state` +event (`paused`, then `queued`). No outcome is produced and the bytes are +kept. A paused entry past `expiresAt` settles `error` with +`errorKind: 'expired'` when it is resumed. + +### `cancel(id): Promise` +A live entry settles `cancelled` with reason `user` and is forgotten after +its ack. A settled entry is forgotten now: row, bytes, and its +unacknowledged outcomes. An unknown id resolves and does nothing. + +For a live entry, the journaled `cancelled` outcome is the cancel. Once it is +written, `cancel()` resolves, even if the entry save that follows fails: the +work stops and the outcome's ack, a retry, or the next launch finishes it. If +the journal (live entry) or the store (settled entry) cannot be written, +`cancel()` rejects with `E_STORAGE` and changes nothing. The caller may call +again. + +### `setWifiOnly(enabled): Promise` +The queue's Wi-Fi setting. Persisted natively. Applies to queued and future +entries whose descriptor does not set `wifiOnly`. An entry that sets it keeps +its own value. Both are checked before each attempt, so a toggle moves the +queued entries that follow the setting. + +### `updateHeaders(patch): Promise` +Merges the patch into the headers of every entry not yet forgotten and +resumes the entries parked on `awaiting-auth`. The patch also replaces +same-named headers a part carries. This is how a fresh token +reaches requests that stalled on 401. Each call bumps a header generation: a +401 or 403 from an attempt issued under an older generation re-issues at once +instead of parking. Parking emits one `state` event per entry. A header name +or value the platform HTTP client cannot send rejects with `E_INVALID`. + +### `getRequests(filter?): RequestRow[]` +Synchronous, from native's in-memory index, so it works offline. Returns +every entry native has not yet forgotten: `queued`, `running`, +`awaiting-auth`, and `paused` entries; `completed` and `cancelled` entries +until their ack; `error` entries until `cancel()` or a same-id `mutate()`; +and rows imported from a v9 install (key `legacy`, see Upgrading from v9). +`filter` is `{ key?, id? }`. A row is +`{ id, key, vars, state, bytesSent, totalBytes, attempts, updatedAt, nextAttemptAt? }`; +`nextAttemptAt` (epoch ms) is set while the entry waits out a retry backoff. +`vars` is typed `Json`, because a row does not know its definition. Narrow +it before reading a field, for example to cancel every entry of one capture: + +```ts +uploads + .getRequests({ key: 'capture.file' }) + .filter((row) => (row.vars as { captureId?: string }).captureId === captureId) + .forEach((row) => uploads.cancel(row.id)); ``` -# FAQs - -Is there an example/sandbox app to test out this package? - -> Yes, there is a simple react native app that comes with an [express](https://github.com/expressjs/express) server where you can see react-native-background-upload in action and try things out in an isolated local environment. - -[RNBackgroundExample](https://github.com/Vydia/react-native-background-upload/blob/master/example/RNBackgroundExample) - -Does it support iOS camera roll assets? +### `chunkPlan(sizeBytes, { min?, max? }): Array<{ start, end }>` +Splits a byte count into contiguous, end-exclusive ranges: a deterministic +greedy walk of `max`-sized chunks (default 20 MB), with a final remainder +smaller than `min` (default 8 MB) absorbed into the previous chunk. A file +smaller than `min` is a single chunk. Also a module export. + +### `addListener(event, listener): EventSubscription` +See Events. Listeners are global; every event carries the entry's `id`. Call +`.remove()` on the result to unsubscribe. + +### `android.addNotificationListener(listener)` +Fires when the Android progress notification is pressed. No event data. + +# Events + +| Event | Data | +| --- | --- | +| `state` | A full `RequestRow`, one per transition, plus `reason: 'unhandled-key'` for an outcome whose key has no definition. A consumer's reducer is one upsert. | +| `progress` | `{ id, bytesSent, totalBytes }`, byte-weighted across a chunked upload's parts. | +| `attempt` | One HTTP attempt: `{ id, key, requestId, attempt, url, method, partIndex?, outcome, httpCode?, responseBody? (4 KB cap), responseBodyTruncated?, responseHeaders?, errorKind?, errorMessage?, at }`. `outcome` is `completed` for an accepted response (2xx or a matching `accept` rule). Any other HTTP response is `error` with `errorKind: 'http'`; a transport failure is `error` with its own `errorKind`. Pause, cancel, and supersede emit no attempt event. Live-only; never journaled or replayed. | + +Terminal outcomes do not appear here. They go to the definition's handlers. + +# Upgrading from v9 + +The CHANGELOG lists every removed v9 export with its replacement. In short: +`startUpload` becomes a `define()` plus `mutate()`; `getAllUploads` becomes +`getRequests()`; `cancelUpload` and `removeUpload` become `cancel(id)`; the +terminal event names become the definition's handlers; per-upload `wifiOnly` +stays on the descriptor, and `setWifiOnly()` sets the value for entries that +omit it. + +What happens to work a v9 build left behind: + +- **The v9 journal becomes `legacy` rows.** On the first v10 launch, every + v9 journal entry that JS never acknowledged becomes a read-only settled row + with key `legacy`, the v9 upload id, and `0/0` bytes. No handler runs for + them. Read them with `getRequests({ key: 'legacy' })`, reconcile your own + state, then `cancel(id)` each one. The import runs before the first + `getRequests()` answers. +- **In-flight v9 work is cancelled natively.** Nothing runs unowned. +- **v9 chunked uploads resume under the same id.** The v9 manifest and bytes + stay on disk. A `mutate()` with the v9 upload id and the same parts resumes + from the accepted parts; different parts start over on the kept bytes. An + id with no `mutate()` keeps its bytes until `cancel(id)`. +- **Wi-Fi only starts off.** Call `setWifiOnly(true)` again if the app had it + on. + +Each `mutate()` on an id that already exists follows the rules in "Same id, +again" above, so a re-dispatch from persisted app state is safe. + +# Testing your definitions + +`createUploadClient({ native })` takes a fake native module, and +`createFakeNative()` builds one. The real validation, header merge, and +delivery run on top of it; the fake keeps rows, journals outcomes, and acks, +but never sends HTTP. + +Importing the package root resolves the native module at load, so a test must +mock `react-native`'s `TurboModuleRegistry` first, as below. The fake imports +no react-native code at runtime, so the mock factory can require it. + +```ts +jest.mock('react-native', () => { + const { createFakeNative } = jest.requireActual( + 'react-native-background-upload/src/testing', + ); + const fake = createFakeNative(); + return { + TurboModuleRegistry: { getEnforcing: () => fake, get: () => fake }, + }; +}); + +import { createUploadClient } from 'react-native-background-upload'; +import { createFakeNative } from 'react-native-background-upload/src/testing'; + +const native = createFakeNative(); +const uploads = createUploadClient({ native }); +const onSuccess = jest.fn(); +const ping = uploads.define({ + key: 'ping', + request: () => ({ url: 'https://api.test/ping' }), + onSuccess, +}); +uploads.configure({}); + +const { id } = await ping.mutate(); +await native.settle(id, { kind: 'completed', response: { body: '{}' } }); +expect(onSuccess).toHaveBeenCalled(); +``` -> Yes, as of version 4.3.0. +`settle()` resolves after delivery acks the outcome, so an async handler +has finished. It rejects after 2 s (`ackTimeoutMs`) when no ack comes. A +handler that rejects is not acked, so test it with a short timeout: -Does it support multiple file uploads? +```ts +const native = createFakeNative({ ackTimeoutMs: 50 }); +// ... define, configure, and mutate as above ... +await expect(native.settle(id, outcome)).rejects.toThrow(/not acknowledged/); +``` -> Yes and No. It supports multiple concurrent uploads, but only a single upload per request. That should be fine for 90%+ of cases. +If you fire an event with `native.emit.settled()` instead, delivery runs the +handler and the ack later. Flush pending promises (for example +`await new Promise(setImmediate)`) before you assert on +`native.ackedEventIds`. -Why should I use this file uploader instead of others that I've Googled like [react-native-uploader](https://github.com/aroth/react-native-uploader)? +The fake does not model the same-id rules (resume, replace, `E_RUNNING`, +re-emit) or the header merge of `updateHeaders()`; script those with +`failNext()` and `seedUnacknowledged()`. -> This package has two killer features not found anywhere else (as of 12/16/2016). First, it works on both iOS and Android. Others are iOS only. Second, it supports background uploading. This means that users can background your app and the upload will continue. This does not happen with other uploaders. +`native.entries` holds every enqueue as `mutate()` sent it, with `vars` and +`data` parsed back from JSON. `failNext()` makes the next native call reject +with a code such as `E_STORAGE`. `seedUnacknowledged()` and `seedRows()` +model a journal and rows left by a dead session, so `configure()` replays +them. +An app that builds its client at module load can use the fake from the mock +for every client instead. Get it with +`TurboModuleRegistry.getEnforcing('RNFileUploader')`, and call +`native.reset()` in `beforeEach`. # Contributing See [CONTRIBUTING.md](./CONTRIBUTING.md). - -# Common Issues - -## BREAKING CHANGE IN 3.0 -This is for 3.0 only. This does NOT apply to 4.0, as recent React Native versions have upgraded the `okhttp` dependencies. Anyway... - -In 3.0, you need to add -```gradle - configurations.all { resolutionStrategy.force 'com.squareup.okhttp3:okhttp:3.4.1' } -``` -to your app's app's `android/app/build.gradle` file. - -Just add it above (not within) `dependencies` and you'll be fine. - - -## BREAKING CHANGE IN 2.0 -Two big things happened in version 2.0. First, the Android package name had to be changed, as it conflicted with our own internal app. My bad. Second, we updated the android upload service dependency to the latest, but that requires the app have a compileSdkVersion and targetSdkVersion or 25. - -To upgrade: -In `MainApplication.java`: - -Change - - ```java - import com.vydia.UploaderReactPackage; - ``` - -to - - ```java - import com.vydia.RNUploader.UploaderReactPackage; - ``` - -Then open your app's `android/app/build.gradle` file. -Ensure `compileSdkVersion` and `targetSdkVersion` are 25. - -Done! - - -## Gratitude - -Thanks to: -- [android-upload-service](https://github.com/gotev/android-upload-service) It made Android dead simple to support. - -- [MIME type from path on iOS](http://stackoverflow.com/questions/2439020/wheres-the-iphone-mime-type-database) Thanks for the answer! diff --git a/android/build.gradle b/android/build.gradle index f3855a1a..d155f4b5 100755 --- a/android/build.gradle +++ b/android/build.gradle @@ -1,73 +1,55 @@ -buildscript { - ext.safeExtGet = { prop, fallback -> - rootProject.ext.has(prop) ? rootProject.ext.get(prop) : fallback - } - - ext { - ext.kotlinVersion = safeExtGet('kotlinVersion', '1.6.0') - buildToolsVersion = '29.0.2' - compileSdkVersion = 29 - targetSdkVersion = 29 - minSdkVersion = 18 - } - ext.detoxKotlinVersion = ext.kotlinVersion - - repositories { - mavenCentral() - google() - } - dependencies { - classpath 'com.android.tools.build:gradle:3.5.3' - classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlinVersion" - } -} - apply plugin: 'com.android.library' apply plugin: 'kotlin-android' +// Generates NativeRNFileUploaderSpec from src/NativeRNFileUploader.ts, per the +// codegenConfig block in package.json. Required for the TurboModule. +apply plugin: 'com.facebook.react' -def DEFAULT_COMPILE_SDK_VERSION = 28 -def DEFAULT_BUILD_TOOLS_VERSION = "28.0.3" -def DEFAULT_TARGET_SDK_VERSION = 28 - +// Inherit the host app's toolchain; the fallbacks match Diana, the only consumer. def safeExtGet(prop, fallback) { rootProject.ext.has(prop) ? rootProject.ext.get(prop) : fallback } android { - compileSdkVersion safeExtGet('compileSdkVersion', DEFAULT_COMPILE_SDK_VERSION) - buildToolsVersion safeExtGet('buildToolsVersion', DEFAULT_BUILD_TOOLS_VERSION) + namespace 'ai.openspace.backgroundupload' + + compileSdkVersion safeExtGet('compileSdkVersion', 36) defaultConfig { - minSdkVersion 21 - targetSdkVersion safeExtGet('targetSdkVersion', DEFAULT_TARGET_SDK_VERSION) - versionCode 1 - versionName "1.0" - ndk { - abiFilters "armeabi-v7a", "x86" - } + minSdkVersion safeExtGet('minSdkVersion', 29) + targetSdkVersion safeExtGet('targetSdkVersion', 36) + // Keeps the Gson-persisted models intact under the host app's R8. See the + // file for why a minified build breaks silently without it. + consumerProguardFiles 'consumer-rules.pro' } - lintOptions { + + compileOptions { + sourceCompatibility JavaVersion.VERSION_17 + targetCompatibility JavaVersion.VERSION_17 + } + + kotlinOptions { + jvmTarget = '17' + } + + lint { abortOnError false } } repositories { + google() mavenCentral() } -def _ext = ext - -def _kotlinVersion = _ext.has('detoxKotlinVersion') ? _ext.detoxKotlinVersion : '1.3.10' -def _kotlinStdlib = _ext.has('detoxKotlinStdlib') ? _ext.detoxKotlinStdlib : 'kotlin-stdlib-jdk8' - dependencies { - implementation "androidx.core:core-ktx:1.0.1" - - implementation 'com.facebook.react:react-native:+' - - implementation "org.jetbrains.kotlin:$_kotlinStdlib:$_kotlinVersion" + // Version supplied by the React Native Gradle plugin. + implementation 'com.facebook.react:react-android' - implementation 'net.gotev:uploadservice-okhttp:4.9.2' + implementation 'androidx.core:core-ktx:1.13.1' + implementation 'androidx.work:work-runtime-ktx:2.12.0' + implementation 'com.squareup.okhttp3:okhttp:4.12.0' + implementation 'com.google.code.gson:gson:2.11.0' + implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-android:1.8.1' - implementation 'androidx.swiperefreshlayout:swiperefreshlayout:1.0.0' + testImplementation 'junit:junit:4.13.2' } diff --git a/android/consumer-rules.pro b/android/consumer-rules.pro new file mode 100644 index 00000000..d7627d06 --- /dev/null +++ b/android/consumer-rules.pro @@ -0,0 +1,43 @@ +# These rules go to consumers through consumerProguardFiles. Thus a minified +# release build of the host app keeps these guarantees. +# +# Gson persists the queue entry (entry.json), the queue settings +# (settings.json), the settled-outcome journal, NotificationConfig +# (SharedPreferences), and reads the v9 journal and v9 chunked manifests at +# the first v10 launch. The library reads them back later, across app +# restarts AND across app updates. Gson finds fields by name through +# reflection, and it needs the generic Signature attribute to rebuild typed +# collections. Thus, if R8 renames a field or removes Signature, it corrupts +# the persisted state silently: +# +# * Descriptor.accept is a List and Descriptor.parts a +# List. Without Signature, Gson decodes the elements as bare maps. +# Then no accept rule matches, and every chunked part reads as unsent. +# * A record from an older build fails to parse if field names changed. The +# library drops it as malformed. That loses the outcomes the journal +# exists to keep, and the entries the queue exists to run. +# * EntryState is an enum persisted by its @SerializedName wire string. +# +# Debug builds are not minified and round-trip correctly, so neither failure +# is reproducible without R8. Keep these rules. +-keepattributes Signature +-keepattributes *Annotation* + +# v10 queue +-keep class ai.openspace.backgroundupload.QueueEntry { *; } +-keep class ai.openspace.backgroundupload.EntryState { *; } +-keep class ai.openspace.backgroundupload.Descriptor { *; } +-keep class ai.openspace.backgroundupload.FormPart { *; } +-keep class ai.openspace.backgroundupload.RetryOverride { *; } +-keep class ai.openspace.backgroundupload.StagedBody { *; } +-keep class ai.openspace.backgroundupload.Part { *; } +-keep class ai.openspace.backgroundupload.QueueSettings { *; } +-keep class ai.openspace.backgroundupload.RetryDefaults { *; } +-keep class ai.openspace.backgroundupload.EventJournal$SettledRecord { *; } +-keep class ai.openspace.backgroundupload.EventJournal$Response { *; } +-keep class ai.openspace.backgroundupload.UploadOutcome$AcceptRule { *; } +-keep class ai.openspace.backgroundupload.NotificationConfig { *; } + +# v9 files read once at the first v10 launch +-keep class ai.openspace.backgroundupload.LegacyImport$V9Entry { *; } +-keep class ai.openspace.backgroundupload.LegacyManifest { *; } diff --git a/android/src/main/AndroidManifest.xml b/android/src/main/AndroidManifest.xml index 4d977ff9..675d7a34 100644 --- a/android/src/main/AndroidManifest.xml +++ b/android/src/main/AndroidManifest.xml @@ -1,3 +1,16 @@ + + xmlns:tools="http://schemas.android.com/tools"> + + + + + + + + diff --git a/android/src/main/java/ai/openspace/backgroundupload/AtomicFiles.kt b/android/src/main/java/ai/openspace/backgroundupload/AtomicFiles.kt new file mode 100644 index 00000000..bf291b53 --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/AtomicFiles.kt @@ -0,0 +1,53 @@ +package ai.openspace.backgroundupload + +import java.io.File +import java.io.FileOutputStream +import java.io.IOException +import java.nio.channels.FileChannel +import java.nio.file.StandardOpenOption + +/** + * Write-ahead file writes. Every durable file in the library goes through + * [writeAtomically]: write a tmp sibling, fsync it, rename it over the + * target. A crash at any point leaves either the old target or the new one, + * never a partial file. + */ +internal object AtomicFiles { + const val TMP_SUFFIX = ".tmp" + + fun tmpFor(target: File) = File(target.parentFile, target.name + TMP_SUFFIX) + + /** Throws IOException when the target could not be replaced. The old target is then intact. */ + fun writeAtomically(target: File, write: (FileOutputStream) -> Unit) { + val parent = target.parentFile ?: throw IOException("no parent directory for ${target.path}") + if (!parent.isDirectory && !parent.mkdirs()) { + throw IOException("could not create ${parent.path}") + } + val tmp = tmpFor(target) + try { + FileOutputStream(tmp).use { out -> + write(out) + out.flush() + out.fd.sync() + } + if (!tmp.renameTo(target)) throw IOException("could not rename ${tmp.path} to ${target.name}") + } catch (error: Throwable) { + tmp.delete() + throw error + } + syncDirectory(parent) + } + + fun writeText(target: File, text: String) = + writeAtomically(target) { it.write(text.toByteArray(Charsets.UTF_8)) } + + /** + * Makes a rename durable. This works on Linux (Android). Some file systems + * do not allow it, so a failure is ignored: the rename itself is still atomic. + */ + fun syncDirectory(dir: File) { + runCatching { + FileChannel.open(dir.toPath(), StandardOpenOption.READ).use { it.force(true) } + } + } +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/AttemptEvent.kt b/android/src/main/java/ai/openspace/backgroundupload/AttemptEvent.kt new file mode 100644 index 00000000..1274c3d8 --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/AttemptEvent.kt @@ -0,0 +1,88 @@ +package ai.openspace.backgroundupload + +import com.facebook.react.bridge.WritableMap + +/** + * One HTTP attempt, before the library interprets it. Live only: never + * journaled. [outcome] is `completed` when the response is accepted and + * `error` otherwise, so a 401 is `error` with httpCode 401 even though the + * entry parks. A transport failure is `error` with its own errorKind. Pause, + * cancel, supersede, and a system stop emit no attempt event. + */ +data class AttemptEvent( + val id: String, + val key: String, + val requestId: String, + val attempt: Int, + val url: String, + val method: String, + val partIndex: Int?, + val outcome: String, + val httpCode: Int?, + val responseBody: String?, + val responseBodyTruncated: Boolean?, + val responseHeaders: Map?, + val errorKind: String?, + val errorMessage: String?, + val at: Long, +) { + companion object { + fun ofResponse( + entry: QueueEntry, + requestId: String, + url: String, + partIndex: Int?, + response: UploadResponse, + accepted: Boolean, + at: Long, + ): AttemptEvent { + val (body, cut) = BodyCap.cap(response.body, BodyCap.ATTEMPT_MAX_BYTES) + return AttemptEvent( + id = entry.id, key = entry.key, requestId = requestId, attempt = entry.attempts, + url = url, method = entry.descriptor?.method ?: "POST", partIndex = partIndex, + outcome = if (accepted) "completed" else "error", + httpCode = response.code, responseBody = body, responseBodyTruncated = cut || response.truncated, + responseHeaders = response.headers, + errorKind = if (accepted) null else "http", + errorMessage = if (accepted) null else "HTTP ${response.code}", + at = at, + ) + } + + fun ofFailure( + entry: QueueEntry, + requestId: String, + url: String, + partIndex: Int?, + errorKind: String, + message: String, + at: Long, + ) = AttemptEvent( + id = entry.id, key = entry.key, requestId = requestId, attempt = entry.attempts, + url = url, method = entry.descriptor?.method ?: "POST", partIndex = partIndex, + outcome = "error", httpCode = null, responseBody = null, responseBodyTruncated = null, + responseHeaders = null, errorKind = errorKind, errorMessage = message, + at = at, + ) + } + + fun toMap(): Map = LinkedHashMap().apply { + put("id", id) + put("key", key) + put("requestId", requestId) + put("attempt", attempt.toDouble()) + put("url", url) + put("method", method) + partIndex?.let { put("partIndex", it.toDouble()) } + put("outcome", outcome) + httpCode?.let { put("httpCode", it.toDouble()) } + responseBody?.let { put("responseBody", it) } + responseBodyTruncated?.let { put("responseBodyTruncated", it) } + responseHeaders?.let { put("responseHeaders", it) } + errorKind?.let { put("errorKind", it) } + errorMessage?.let { put("errorMessage", it) } + put("at", at.toDouble()) + } + + fun toWritableMap(): WritableMap = JsonBridge.toWritableMap(toMap()) +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/BodyCap.kt b/android/src/main/java/ai/openspace/backgroundupload/BodyCap.kt new file mode 100644 index 00000000..ade8e7df --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/BodyCap.kt @@ -0,0 +1,67 @@ +package ai.openspace.backgroundupload + +import okio.Buffer +import okio.BufferedSource +import java.nio.charset.Charset + +/** + * Response body caps, in UTF-8 bytes. A cut never splits a character: it + * backs off to the last whole one. [read] applies the cap while the body + * streams in, so a huge error page never sits in memory whole. + */ +object BodyCap { + /** 1 MB, the RawResponse cap of a settled outcome. */ + const val SETTLED_MAX_BYTES = 1_048_576 + + /** 4 KB, the body cap of a live attempt event. */ + const val ATTEMPT_MAX_BYTES = 4 * 1024 + + /** A body as text and whether the cap cut it. */ + data class Capped(val text: String, val truncated: Boolean) + + /** + * Reads at most [maxBytes] of [source]. Bytes past the cap are not read. + * A UTF-8 body is cut on a character boundary; another charset is cut at + * the byte cap and decoded as it is. + */ + fun read(source: BufferedSource, maxBytes: Int, charset: Charset = Charsets.UTF_8): Capped { + val buffer = Buffer() + val limit = maxBytes.toLong() + 1 + while (buffer.size < limit) { + if (source.read(buffer, limit - buffer.size) == -1L) break + } + val truncated = buffer.size > maxBytes + val bytes = buffer.readByteArray() + val keep = if (!truncated) bytes.size + else if (charset == Charsets.UTF_8) utf8Boundary(bytes, maxBytes) + else maxBytes + return Capped(String(bytes, 0, keep, charset), truncated) + } + + /** [text] cut to at most [maxBytes] of UTF-8. Null stays null. */ + fun cap(text: String?, maxBytes: Int): Pair { + if (text == null) return null to false + val bytes = text.toByteArray(Charsets.UTF_8) + if (bytes.size <= maxBytes) return text to false + return String(bytes, 0, utf8Boundary(bytes, maxBytes), Charsets.UTF_8) to true + } + + /** + * The longest prefix length of [bytes], at most [max], that does not end + * inside a UTF-8 sequence. A continuation byte is 10xxxxxx. + */ + internal fun utf8Boundary(bytes: ByteArray, max: Int): Int { + if (bytes.size <= max) return bytes.size + var end = max + // bytes[end] is the first byte cut off. While it continues a sequence, + // the sequence started before the cut, so drop its start too. A UTF-8 + // character is at most 4 bytes; past 3 steps the bytes are not UTF-8, + // and the cut stays at max. + var steps = 0 + while (end > 0 && steps < 3 && (bytes[end].toInt() and 0xC0) == 0x80) { + end-- + steps++ + } + return if ((bytes[end].toInt() and 0xC0) == 0x80) max else end + } +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/BodyStaging.kt b/android/src/main/java/ai/openspace/backgroundupload/BodyStaging.kt new file mode 100644 index 00000000..e65d4647 --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/BodyStaging.kt @@ -0,0 +1,212 @@ +package ai.openspace.backgroundupload + +import java.io.BufferedOutputStream +import java.io.File +import java.io.OutputStream +import java.nio.file.Files +import java.nio.file.StandardCopyOption +import java.util.UUID + +/** + * Writes a request body into the entry directory before the entry is saved. + * After enqueue resolves, every byte the request needs is in the library + * directory, so the caller may delete its source. + * + * | Descriptor | Staged file | Content-Type | + * | none | none | none | + * | data | body-.json | application/json unless the caller set one | + * | form | body-.multipart| multipart/form-data; boundary=… (replaces the caller's) | + * | file | file- (copy) | the caller's own | + * | file + parts | blob / blob- (move) | the caller's own | + * + * Each staged file has a name per generation, so a replace writes a new file + * next to the old one. The old body is deleted only after the new entry is + * saved. Every source is checked before anything is written. A chunked blob + * of generation 1 keeps the v9 name `blob`, so v9 blobs are found in place. + */ +object BodyStaging { + const val JSON_CONTENT_TYPE = "application/json" + private const val BUFFER = 64 * 1024 + private val CRLF = "\r\n".toByteArray() + + data class Staged(val body: StagedBody, val headers: Map) + + /** + * @param ownedBlob the chunked blob the entry (or a v9 manifest) already + * owns. The parts run over it when the caller's file is gone (v9 recreate). + * @param keepOwned run over [ownedBlob] even when the caller's file is + * present. Only for the same parts, whose accepted flags carry over. + */ + fun stage( + d: Descriptor, + dir: File, + generation: Int, + ownedBlob: File? = null, + keepOwned: Boolean = false, + ): Staged { + checkSources(d, dir, generation, ownedBlob, keepOwned) + val body = when (d.bodyKind) { + StagedBody.NONE -> StagedBody(StagedBody.NONE, null, null, 0) + StagedBody.JSON -> { + val name = "body-$generation.json" + val target = File(dir, name) + writeJson(d.dataJson!!, target) + StagedBody(StagedBody.JSON, name, null, target.length()) + } + StagedBody.MULTIPART -> { + val name = "body-$generation.multipart" + val target = File(dir, name) + val boundary = newBoundary() + writeMultipart(d.form!!, boundary, target) + StagedBody(StagedBody.MULTIPART, name, boundary, target.length()) + } + StagedBody.FILE -> { + val name = "file-$generation" + val target = File(dir, name) + copyFile(File(d.file!!), target) + StagedBody(StagedBody.FILE, name, null, target.length()) + } + else -> { + val parts = d.parts!! + val source = File(d.file!!) + val runOver = chunkedInput(d, dir, generation, ownedBlob, keepOwned) + // Checked before the move, so a rejected plan moves nothing. + if (!ChunkedParts.tilesExactly(parts, runOver.length())) { + throw QueueException( + QueueException.E_INVALID, + "parts must tile the file exactly: [0, ${runOver.length()})", + ) + } + val blob = if (runOver.path == source.path) { + File(dir, blobName(generation)).also { takeOwnership(source, it) } + } else runOver + StagedBody(StagedBody.CHUNKED, blob.name, null, ChunkedParts.totalBytes(parts)) + } + } + return Staged(body, headersFor(d.headers, body)) + } + + /** The Content-Type rule of the table above. */ + fun headersFor(headers: Map, body: StagedBody): Map = + when (body.kind) { + StagedBody.JSON -> + if (HeaderMap.contains(headers, "Content-Type")) headers + else headers + ("Content-Type" to JSON_CONTENT_TYPE) + StagedBody.MULTIPART -> + HeaderMap.without(headers, "Content-Type") + + ("Content-Type" to "multipart/form-data; boundary=${body.boundary}") + else -> headers + } + + /** The chunked blob name of [generation]. Generation 1 keeps the v9 name. */ + fun blobName(generation: Int) = + if (generation <= 1) QueueStore.BLOB_FILE else "${QueueStore.BLOB_FILE}-$generation" + + /** + * The file a chunked plan runs over, before anything moves: + * 1. [ownedBlob] when [keepOwned] and it exists; + * 2. the caller's file when present (a new file wins over an old blob); + * 3. this generation's blob, left by a crash after the move; + * 4. [ownedBlob], when the caller's file was moved away at an earlier enqueue. + */ + internal fun chunkedInput(d: Descriptor, dir: File, generation: Int, ownedBlob: File?, keepOwned: Boolean): File { + val source = File(d.file!!) + val leftover = File(dir, blobName(generation)) + return when { + keepOwned && ownedBlob != null && ownedBlob.exists() -> ownedBlob + source.isFile -> source + leftover.exists() -> leftover + ownedBlob != null && ownedBlob.exists() -> ownedBlob + else -> throw QueueException(QueueException.E_FILE_MISSING, "file does not exist: ${source.path}") + } + } + + /** Every source must exist and be readable before any write. */ + internal fun checkSources(d: Descriptor, dir: File, generation: Int, ownedBlob: File?, keepOwned: Boolean) { + d.form?.forEach { part -> + part.path?.let { requireReadable(File(it), "form part '${part.name}'") } + } + when (d.bodyKind) { + StagedBody.FILE -> requireReadable(File(d.file!!), "file") + StagedBody.CHUNKED -> chunkedInput(d, dir, generation, ownedBlob, keepOwned) + } + } + + private fun requireReadable(file: File, what: String) { + if (!file.isFile || !file.canRead()) { + throw QueueException(QueueException.E_FILE_MISSING, "$what does not exist or can not be read: ${file.path}") + } + } + + internal fun writeJson(dataJson: String, target: File) = + AtomicFiles.writeText(target, dataJson) + + /** + * RFC 7578. Per part: the boundary line, Content-Disposition with the name + * (and a filename for a file part), Content-Type, a blank line, the bytes, + * CRLF. Then the closing delimiter. File parts stream from disk. + */ + internal fun writeMultipart(form: List, boundary: String, target: File) { + AtomicFiles.writeAtomically(target) { raw -> + val out = BufferedOutputStream(raw, BUFFER) + for (part in form) { + out.ascii("--$boundary") + out.write(CRLF) + val disposition = StringBuilder("Content-Disposition: form-data; name=\"") + .append(escapeQuoted(part.name)).append('"') + if (part.path != null) { + val fileName = part.fileName ?: File(part.path).name + disposition.append("; filename=\"").append(escapeQuoted(fileName)).append('"') + } + out.write(disposition.toString().toByteArray(Charsets.UTF_8)) + out.write(CRLF) + out.write("Content-Type: ${part.contentType}".toByteArray(Charsets.UTF_8)) + out.write(CRLF) + out.write(CRLF) + if (part.path != null) { + File(part.path).inputStream().use { it.copyTo(out, BUFFER) } + } else { + out.write((part.string ?: "").toByteArray(Charsets.UTF_8)) + } + out.write(CRLF) + } + out.ascii("--$boundary--") + out.write(CRLF) + out.flush() + } + } + + /** The WHATWG form encoding of a quoted name: `"`, CR and LF are percent-encoded. */ + internal fun escapeQuoted(value: String): String = + value.replace("\"", "%22").replace("\r", "%0D").replace("\n", "%0A") + + internal fun copyFile(source: File, target: File) { + source.inputStream().use { input -> + AtomicFiles.writeAtomically(target) { out -> input.copyTo(out, BUFFER) } + } + } + + /** + * Moves the caller's file to [blob] (v9 verbatim). A crash between the move + * and the entry save leaves the bytes at the blob path with no entry; a + * retry whose source is gone adopts them ([chunkedInput] step 3). + */ + internal fun takeOwnership(source: File, blob: File) { + if (source.absoluteFile == blob.absoluteFile) return + if (!source.exists()) { + if (blob.exists()) return + throw QueueException(QueueException.E_FILE_MISSING, "file does not exist: ${source.path}") + } + blob.parentFile?.mkdirs() + if (blob.exists()) blob.delete() + if (!source.renameTo(blob)) { + // renameTo can not cross file systems. Files.move falls back to copy + delete. + Files.move(source.toPath(), blob.toPath(), StandardCopyOption.REPLACE_EXISTING) + } + blob.parentFile?.let { AtomicFiles.syncDirectory(it) } + } + + internal fun newBoundary(): String = "----RNBGU" + UUID.randomUUID().toString().replace("-", "") + + private fun OutputStream.ascii(text: String) = write(text.toByteArray(Charsets.US_ASCII)) +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/ChunkedEngine.kt b/android/src/main/java/ai/openspace/backgroundupload/ChunkedEngine.kt new file mode 100644 index 00000000..bbbee474 --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/ChunkedEngine.kt @@ -0,0 +1,38 @@ +package ai.openspace.backgroundupload + +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.Semaphore +import kotlinx.coroutines.sync.withPermit + +/** + * The pure scheduling half of chunked execution: the window. It has no + * Android or OkHttp types, so its two invariants (at most WINDOW parts in + * flight, never two requests for one part index) are unit-testable on a + * plain JVM. The retry decisions moved to [RetryClassifier] in v10. + */ +object ChunkedEngine { + + // Parts of one upload in flight at one time. A library constant, not an option. + const val WINDOW = 3 + + /** + * Runs [executePart] exactly one time per index, with at most [window] + * parts at one time. One coroutine per index is what guarantees no two + * requests for one part are in flight (concurrent PUTs of one partNum are + * unsafe on the server). An executor that throws cancels the other parts, + * and the error propagates. + */ + suspend fun run( + partIndexes: List, + window: Int = WINDOW, + executePart: suspend (Int) -> Unit, + ) { + val gate = Semaphore(window) + coroutineScope { + for (index in partIndexes) { + launch { gate.withPermit { executePart(index) } } + } + } + } +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/ChunkedParts.kt b/android/src/main/java/ai/openspace/backgroundupload/ChunkedParts.kt new file mode 100644 index 00000000..5d8ac2e0 --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/ChunkedParts.kt @@ -0,0 +1,60 @@ +package ai.openspace.backgroundupload + +/** + * One part of a chunked upload, as the caller wrote it. The library sends the + * file bytes [start, end) to [url]. [accepted] is set when the server + * accepted the part. The field names match the v9 manifest, so a v9 + * manifest.json reads into this class unchanged. + */ +data class Part( + val url: String, + val headers: Map, + val start: Long, + val end: Long, // exclusive + val accepted: Boolean = false, +) { + val size get() = end - start +} + +/** Pure rules over a parts list. Moved from the v9 ChunkedManifest with the same meaning. */ +object ChunkedParts { + + /** Whether [parts] cover [0, size) exactly: no gap, no overlap, nothing past the end. Order does not matter. */ + fun tilesExactly(parts: List, size: Long): Boolean { + if (parts.isEmpty()) return false + var cursor = 0L + for (part in parts.sortedBy { it.start }) { + if (part.start != cursor || part.end <= part.start) return false + cursor = part.end + } + return cursor == size + } + + /** + * The same upload: the same count, ranges, and urls, in any order. Headers + * are not compared, because a resume sends fresh headers. + */ + fun sameParts(a: List, b: List): Boolean { + if (a.size != b.size) return false + val x = a.sortedBy { it.start } + val y = b.sortedBy { it.start } + return x.indices.all { i -> + x[i].url == y[i].url && x[i].start == y[i].start && x[i].end == y[i].end + } + } + + /** The [incoming] parts with the accepted flags of [stored]. A flag follows the range start, not the index. */ + fun carryAccepted(stored: List, incoming: List): List { + val acceptedStarts = stored.filter { it.accepted }.map { it.start }.toSet() + return incoming.map { it.copy(accepted = it.start in acceptedStarts) } + } + + fun totalBytes(parts: List): Long = parts.sumOf { it.size } + + fun acceptedBytes(parts: List): Long = parts.filter { it.accepted }.sumOf { it.size } + + fun pendingIndexes(parts: List): List = parts.indices.filter { !parts[it].accepted } + + fun withAccepted(parts: List, index: Int): List = + parts.mapIndexed { i, part -> if (i == index) part.copy(accepted = true) else part } +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/ChunkedUploadWorker.kt b/android/src/main/java/ai/openspace/backgroundupload/ChunkedUploadWorker.kt new file mode 100644 index 00000000..d236ec9a --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/ChunkedUploadWorker.kt @@ -0,0 +1,133 @@ +package ai.openspace.backgroundupload + +import android.content.Context +import androidx.work.WorkerParameters +import java.io.File +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicLong + +/** The WorkManager class for a chunked entry. The run is [EntryWorker]'s. */ +class ChunkedUploadWorker(context: Context, params: WorkerParameters) : EntryWorker(context, params) + +/** + * The parts of one chunked entry, at most [ChunkedEngine.WINDOW] at a time, + * each part through the shared 4-request semaphore. One logical entry has + * one progress stream (byte-weighted) and one outcome: completed only when + * every part is accepted. + * + * Each part runs [EntryRun.attempt] until a verdict ends it. + * Per part: accepted → persist the flag; auth → the whole entry parks (the + * sibling parts stop); transient → a short backoff waits in the part while + * the siblings go on, a long one releases the whole worker (accepted parts + * are kept); terminal → the entry fails with that part's index. + */ +internal class ChunkedTransfer(private val run: EntryRun) { + + /** A terminal part failure. Not a CancellationException, so it stops the sibling parts. */ + private class PartFailed(val settlement: Settlement.Failed) : Exception(settlement.message) + + private val partSent = ConcurrentHashMap() + private val acceptedHere = ConcurrentHashMap.newKeySet() + private val acceptedBytes = AtomicLong(0) + private var total = 0L + + @Volatile + private var lastAcceptedUrl: String? = null + + suspend fun run(start: QueueEntry): Settlement { + val d0 = start.descriptor!! + val parts = d0.parts!! + lastAcceptedUrl = parts.lastOrNull { it.accepted }?.url + val pending = ChunkedParts.pendingIndexes(parts) + // A run over an all-accepted entry that has not settled yet. + if (pending.isEmpty()) return Settlement.Completed(null, lastAcceptedUrl ?: d0.reportUrl, d0.method) + val blob = run.store.bodyFile(start) + if (blob == null || !blob.exists()) { + return Settlement.Failed("file", "the chunked file is missing", null, null, d0.reportUrl, d0.method) + } + total = ChunkedParts.totalBytes(parts) + acceptedBytes.set(ChunkedParts.acceptedBytes(parts)) + run.progressStarted(total, acceptedBytes.get()) + + try { + ChunkedEngine.run(pending) { index -> executePart(index, blob, start.backoffStreak) } + } catch (failed: PartFailed) { + return failed.settlement + } + // Every executor returned, and an executor returns only when its part was + // accepted: the server's auto-publish condition. + return Settlement.Completed(null, lastAcceptedUrl ?: d0.reportUrl, d0.method) + } + + internal suspend fun executePart(index: Int, blob: File, initialStreak: Int) { + var streak = initialStreak + while (true) { + if (index in acceptedHere) return + val latest = run.ops.latest(run.entryId, run.generation) + val stored = latest.descriptor!!.parts!![index] + if (stored.accepted) return + if (RetryClassifier.isExpired(run.host.now(), latest.expiresAt)) throw EntryRun.ExpiredException() + + // A range past EOF can never be sent. length() is 0 for a missing file; + // that case falls through to the attempt, which classifies it as file. + val blobLength = runCatching { blob.length() }.getOrDefault(0L) + if (blobLength > 0L && stored.end > blobLength) { + throw PartFailed( + Settlement.Failed( + "file", + "part $index range [${stored.start}, ${stored.end}) exceeds the file size $blobLength", + null, index, stored.url, latest.descriptor.method, + ), + ) + } + run.waitForNetwork() + + val a = run.attempt( + partIndex = index, + body = { _, part -> rangeRequestBody(blob, part!!.start, part.end) }, + onProgress = { sent -> onPartProgress(index, sent) }, + fileExists = { blob.exists() }, + ) + when (val r = a.result) { + is EntryRun.AttemptResult.Accepted -> { + markAccepted(index, a.entry.descriptor!!.parts!![index]) + return + } + is EntryRun.AttemptResult.Auth -> { + if (!r.reissue) throw EntryRun.ParkException(r.headerGeneration) + streak = 0 + } + EntryRun.AttemptResult.Transient -> { + onPartProgress(index, 0L) + streak++ + run.backoffOrRelease(a.policy, streak, a.entry.expiresAt) + } + is EntryRun.AttemptResult.Terminal -> { + onPartProgress(index, 0L) + val message = r.response?.let { "HTTP ${it.code} on part $index" } ?: r.message + throw PartFailed(Settlement.Failed(r.errorKind, message, r.response, index, a.url, a.method)) + } + } + } + } + + private fun markAccepted(index: Int, part: Part) { + // Remembered here too, so a lost flag write does not re-send the part in this run. + acceptedHere += index + run.ops.markAccepted(run.entryId, run.generation, index) + acceptedBytes.addAndGet(part.size) + lastAcceptedUrl = part.url + partSent.remove(index) + report() + } + + private fun onPartProgress(index: Int, sent: Long) { + if (sent == 0L) partSent.remove(index) else partSent[index] = sent + report() + } + + private fun report() { + val sent = (acceptedBytes.get() + partSent.values.sum()).coerceAtMost(total) + run.reportProgress(sent, total) + } +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/Diag.kt b/android/src/main/java/ai/openspace/backgroundupload/Diag.kt new file mode 100644 index 00000000..14382887 --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/Diag.kt @@ -0,0 +1,19 @@ +package ai.openspace.backgroundupload + +import android.util.Log + +/** + * Logging that is safe in the JVM unit tests. There, android.util.Log is a + * stub that throws, so every call is wrapped. + */ +internal object Diag { + const val TAG = "RNFileUploader" + + fun warn(message: String, error: Throwable? = null) { + runCatching { Log.w(TAG, message, error) } + } + + fun error(message: String, error: Throwable? = null) { + runCatching { Log.e(TAG, message, error) } + } +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/EnqueueRules.kt b/android/src/main/java/ai/openspace/backgroundupload/EnqueueRules.kt new file mode 100644 index 00000000..5fd1ac15 --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/EnqueueRules.kt @@ -0,0 +1,179 @@ +package ai.openspace.backgroundupload + +/** + * The same-id rules of enqueue() (plan 5.4), as pure functions. [decide] + * picks the action; the builders make the next entry from the staged body. + * + * | Stored entry | Action | + * | none, no v9 manifest | Create | + * | none, v9 manifest | AdoptV9: keep the blob and accepted parts | + * | legacy row, v9 manifest | AdoptV9 (generation + 1) | + * | legacy row, no manifest | Replace (generation + 1) | + * | same body, completed, record present | ReEmit: deliveries + 1, no re-run | + * | same body, completed, record gone | Replace (it was acked) | + * | same body, any other state | Resume (a settled one reopens: gen + 1, | + * | | attempts 0) | + * | different body, running | RejectRunning (E_RUNNING) | + * | different body, otherwise | Replace (generation + 1, attempts 0) | + */ +object EnqueueRules { + + sealed class Action { + object Create : Action() + /** [generation] is 1, or the legacy row's + 1. */ + data class AdoptV9(val manifest: LegacyManifest, val generation: Int) : Action() + data class ReEmit(val eventId: String) : Action() + object Resume : Action() + object Replace : Action() + object RejectRunning : Action() + } + + fun decide( + existing: QueueEntry?, + v9: LegacyManifest?, + incoming: Descriptor, + hasRecord: (eventId: String) -> Boolean, + ): Action { + if (existing == null) return if (v9 != null) Action.AdoptV9(v9, 1) else Action.Create + // An imported v9 outcome row. Its v9 chunked manifest, when present, is + // the upload's progress: adopt it, as with no row at all. + if (existing.legacy) return if (v9 != null) Action.AdoptV9(v9, existing.generation + 1) else Action.Replace + if (existing.sameBodyAs(incoming)) { + if (existing.state == EntryState.COMPLETED) { + val eventId = existing.settledEventId + return if (eventId != null && hasRecord(eventId)) Action.ReEmit(eventId) else Action.Replace + } + return Action.Resume + } + return if (existing.state == EntryState.RUNNING) Action.RejectRunning else Action.Replace + } + + /** + * The generation to stage a body for before the store lock is taken, or + * null to stage under the lock. Only a copied body (JSON, multipart, + * file), and only when no worker can change the decision meanwhile: a new + * id, or a replace of an entry that a worker can not take (not queued, + * not running). A chunked body is a move, which is fast, over a blob that + * a worker may be reading, so it always stages under the lock. + */ + fun preStageGeneration(existing: QueueEntry?, action: Action, incoming: Descriptor): Int? { + val copied = incoming.bodyKind.let { + it == StagedBody.JSON || it == StagedBody.MULTIPART || it == StagedBody.FILE + } + if (!copied) return null + return when (action) { + Action.Create -> 1 + Action.Replace -> existing + ?.takeIf { it.state != EntryState.QUEUED && it.state != EntryState.RUNNING } + ?.let { it.generation + 1 } + else -> null + } + } + + /** The parts an adopted v9 manifest runs with: its accepted flags when the parts are the same. */ + fun adoptedParts(v9: LegacyManifest, incoming: List): List = + if (ChunkedParts.sameParts(v9.parts, incoming)) ChunkedParts.carryAccepted(v9.parts, incoming) + else incoming + + private fun initialState(paused: Boolean) = if (paused) EntryState.PAUSED else EntryState.QUEUED + + /** A new entry (Create, AdoptV9). [parts] carries adopted accepted flags. */ + fun created( + p: EntryParsing.Parsed, + staged: BodyStaging.Staged, + parts: List?, + paused: Boolean, + headerGeneration: Int, + now: Long, + ): QueueEntry { + val runParts = parts ?: p.descriptor.parts + return QueueEntry( + id = p.id, + key = p.key, + varsJson = p.varsJson, + descriptor = p.descriptor.copy(headers = staged.headers, parts = runParts), + body = staged.body, + state = initialState(paused), + attempts = 0, + bytesSent = runParts?.let { ChunkedParts.acceptedBytes(it) } ?: 0L, + totalBytes = staged.body.totalBytes, + expiresAt = p.expiresAt, + createdAt = now, + updatedAt = now, + headerGeneration = headerGeneration, + generation = 1, + ) + } + + /** AdoptV9: a new entry over the v9 blob, at [generation]; over a legacy row it keeps the row's createdAt. */ + fun adopted( + p: EntryParsing.Parsed, + staged: BodyStaging.Staged, + parts: List?, + legacyRow: QueueEntry?, + generation: Int, + paused: Boolean, + headerGeneration: Int, + now: Long, + ): QueueEntry = created(p, staged, parts, paused, headerGeneration, now).copy( + createdAt = legacyRow?.createdAt ?: now, + generation = generation, + ) + + /** + * Same body. New headers, expiresAt, vars, accept, retry, wifiOnly, and + * notification flag replace the stored ones; the body and accepted parts + * stay. A settled entry reopens with a fresh generation and attempts 0 + * (attempts count the current generation). A running one stays running + * (the worker reads the new headers before its next attempt). + */ + fun resumed( + existing: QueueEntry, + p: EntryParsing.Parsed, + paused: Boolean, + headerGeneration: Int, + now: Long, + ): QueueEntry { + val stored = existing.descriptor!! + val body = existing.body!! + val parts = stored.parts?.let { ChunkedParts.carryAccepted(it, p.descriptor.parts!!) } + val running = existing.state == EntryState.RUNNING + val reopen = existing.isSettled + return existing.copy( + key = p.key, + varsJson = p.varsJson, + descriptor = stored.copy( + headers = BodyStaging.headersFor(p.descriptor.headers, body), + parts = parts, + accept = p.descriptor.accept, + retry = p.descriptor.retry, + noNotification = p.descriptor.noNotification, + wifiOnly = p.descriptor.wifiOnly, + ), + state = if (running) EntryState.RUNNING else initialState(paused), + attempts = if (reopen) 0 else existing.attempts, + bytesSent = parts?.let { ChunkedParts.acceptedBytes(it) } ?: if (running) existing.bytesSent else 0L, + expiresAt = p.expiresAt, + updatedAt = now, + nextAttemptAt = null, + backoffStreak = 0, + headerGeneration = headerGeneration, + parkedGeneration = null, + generation = if (reopen) existing.generation + 1 else existing.generation, + settledEventId = if (reopen) null else existing.settledEventId, + ) + } + + /** Different body (or over a legacy or acked row). A new life over the same id. */ + fun replaced( + existing: QueueEntry, + p: EntryParsing.Parsed, + staged: BodyStaging.Staged, + paused: Boolean, + headerGeneration: Int, + now: Long, + ): QueueEntry = created(p, staged, null, paused, headerGeneration, now).copy( + createdAt = existing.createdAt, + generation = existing.generation + 1, + ) +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/EntryParsing.kt b/android/src/main/java/ai/openspace/backgroundupload/EntryParsing.kt new file mode 100644 index 00000000..5992de45 --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/EntryParsing.kt @@ -0,0 +1,226 @@ +package ai.openspace.backgroundupload + +import com.facebook.react.bridge.ReadableArray +import com.facebook.react.bridge.ReadableMap +import com.facebook.react.bridge.ReadableType +import okhttp3.Headers +import okhttp3.HttpUrl.Companion.toHttpUrlOrNull +import java.net.URI + +/** + * Turns the EnqueueEntry `{ id, key, varsJson, descriptor }` into Kotlin + * values. JS has already validated the descriptor. Native checks only what + * it needs to run, and rejects anything else with E_INVALID. + * + * `varsJson` and `descriptor.dataJson` are JSON text. They cross as strings + * because React Native on iOS drops object keys whose value is null. Native + * keeps the text as it came: it is the body that goes out. A dataJson of + * "null" is the JSON body null, a real body. + */ +object EntryParsing { + class InvalidEntryException(message: String) : IllegalArgumentException(message) + + data class Parsed( + val id: String, + val key: String, + val varsJson: String, + val descriptor: Descriptor, + val expiresAt: Long, + ) + + private val METHODS = setOf("POST", "PUT", "PATCH", "DELETE", "GET") + + fun parse(entry: ReadableMap): Parsed { + val id = entry.string("id")?.takeIf { it.isNotEmpty() } ?: invalid("id is required") + val key = entry.string("key")?.takeIf { it.isNotEmpty() } ?: invalid("key is required") + val varsJson = entry.string("varsJson") ?: invalid("varsJson is required") + if (!JsonBridge.isJson(varsJson)) invalid("varsJson is not JSON text") + val d = entry.map("descriptor") ?: invalid("descriptor is required") + val expiresAt = d.number("expiresAt")?.toLong() ?: invalid("descriptor.expiresAt is required") + return Parsed(id, key, varsJson, descriptor(d), expiresAt) + } + + fun descriptor(d: ReadableMap): Descriptor { + val method = (d.string("method") ?: "POST").uppercase() + if (method !in METHODS) invalid("method $method is not supported") + + val parts = d.array("parts")?.let { parseParts(it) } + val url = d.string("url") + if (url == null && parts == null) invalid("url is required unless parts is set") + url?.let { requireHttpUrl(it, "url") } + + // An old JS layer would send `data`. Ignoring it would send no body. + if (d.isSet("data")) invalid("data crosses as dataJson") + val dataJson = if (d.isSet("dataJson")) { + val text = d.string("dataJson") ?: invalid("dataJson must be a string") + if (!JsonBridge.isJson(text)) invalid("dataJson is not JSON text") + text + } else null + val form = d.array("form")?.let { parseForm(it) } + val file = d.string("file")?.let { stripFileScheme(it) } + val kinds = listOfNotNull(dataJson?.let { "data" }, form?.let { "form" }, file?.let { "file" }) + if (kinds.size > 1) invalid("at most one of data, form, file; got ${kinds.joinToString()}") + if (parts != null && file == null) invalid("parts requires file") + if (method == "GET" && kinds.isNotEmpty()) invalid("a GET request can not carry a body") + + val headers = parseHeaderMap(d.map("headers")) + requireValidHeaders(headers, "headers") + if (d.isSet("wifiOnly") && d.bool("wifiOnly") == null) invalid("wifiOnly must be a boolean") + + return Descriptor( + url = url, + method = method, + headers = headers, + dataJson = dataJson, + form = form, + file = file, + parts = parts, + accept = parseAcceptRules(d.array("accept")), + retry = d.map("retry")?.let { parseRetry(it) }, + noNotification = d.map("android")?.bool("noNotification") ?: false, + wifiOnly = d.bool("wifiOnly"), + ) + } + + /** + * pause/resume scope `{ keys?: string[] }`. A null scope or a scope with + * no keys is the whole queue (null comes from a caller that sent no + * argument). A keys field that is not a list of non-empty strings is + * refused: dropping it would widen the scope to the whole queue. + */ + fun scopeKeys(scope: ReadableMap?): List? { + if (scope == null || !scope.hasKey("keys")) return null + val arr = scope.array("keys") ?: invalid("scope.keys must be an array of strings") + return (0 until arr.size()).map { i -> + if (arr.getType(i) != ReadableType.String) invalid("scope.keys[$i] must be a string") + arr.getString(i)?.takeIf { it.isNotEmpty() } ?: invalid("scope.keys[$i] must be a non-empty string") + } + } + + /** updateHeaders(patch): the header map, checked the same way as a descriptor's. */ + fun headerPatch(patch: ReadableMap): Map = + parseHeaderMap(patch).also { requireValidHeaders(it, "updateHeaders") } + + /** `file:///a%20b` → `/a b`. A plain path is returned as it is. */ + fun stripFileScheme(path: String): String { + if (!path.startsWith("file://")) return path + return runCatching { URI(path).path }.getOrNull() ?: path.removePrefix("file://") + } + + private fun parseParts(arr: ReadableArray): List { + if (arr.size() == 0) invalid("parts must be a non-empty array") + return (0 until arr.size()).map { i -> + val p = arr.getMap(i) ?: invalid("parts[$i] must be an object") + val url = p.string("url") ?: invalid("parts[$i].url is required") + requireHttpUrl(url, "parts[$i].url") + val range = p.map("range") ?: invalid("parts[$i].range is required") + val start = range.number("start")?.toLong() ?: invalid("parts[$i].range.start is required") + val end = range.number("end")?.toLong() ?: invalid("parts[$i].range.end is required") + if (start < 0 || end <= start) invalid("parts[$i].range must satisfy 0 <= start < end") + val headers = parseHeaderMap(p.map("headers")) + requireValidHeaders(headers, "parts[$i].headers") + Part(url = url, headers = headers, start = start, end = end) + } + } + + private fun parseForm(arr: ReadableArray): List { + if (arr.size() == 0) invalid("form must be a non-empty array") + return (0 until arr.size()).map { i -> + val p = arr.getMap(i) ?: invalid("form[$i] must be an object") + val name = p.string("name") ?: invalid("form[$i].name is required") + val contentType = p.string("contentType") ?: invalid("form[$i].contentType is required") + val string = p.string("string") + val path = p.string("path")?.let { stripFileScheme(it) } + if ((string == null) == (path == null)) invalid("form[$i] must set exactly one of string, path") + FormPart(name, contentType, string, path, p.string("fileName")) + } + } + + private fun parseRetry(r: ReadableMap): RetryOverride { + val backoff = r.map("backoff") + val exempt = r.map("terminalHttp")?.array("exempt")?.let { arr -> + (0 until arr.size()).mapNotNull { i -> + if (arr.getType(i) == ReadableType.Number) arr.getDouble(i).toInt() else null + } + } + return RetryOverride( + baseMs = backoff?.number("baseMs")?.toLong(), + maxMs = backoff?.number("maxMs")?.toLong(), + jitter = backoff?.number("jitter"), + exempt = exempt, + ) + } + + internal fun parseAcceptRules(arr: ReadableArray?): List { + if (arr == null) return listOf() + return (0 until arr.size()).mapNotNull { i -> + val rule = arr.getMap(i) ?: return@mapNotNull null + val status = rule.number("status") ?: return@mapNotNull null + UploadOutcome.AcceptRule(status.toInt(), rule.string("bodyIncludes")) + } + } + + /** Header values keep their text. A number is written as JSON would write it. */ + internal fun parseHeaderMap(map: ReadableMap?): Map { + if (map == null) return mapOf() + val out = LinkedHashMap() + JsonBridge.fromReadable(map).forEach { (k, v) -> + when (v) { + null -> Unit + is Double -> out[k] = JsonBridge.numberText(v) + else -> out[k] = v.toString() + } + } + return out + } + + private fun requireHttpUrl(url: String, where: String) { + if (url.toHttpUrlOrNull() == null) invalid("$where is not an http(s) url: $url") + } + + /** + * OkHttp throws on a header name or value it can not send. Check it here, + * so the error is an enqueue rejection and not a failure at attempt time. + * The message names the header and the offset only: a value can be a + * credential, and OkHttp's own message would print it. + */ + internal fun requireValidHeaders(headers: Map, where: String) { + headers.forEach { (name, value) -> + // OkHttp's rules: a name is 1+ chars in 0x21..0x7e; a value is tab or 0x20..0x7e. + if (name.isEmpty()) invalid("$where: a header name is empty") + name.indexOfFirst { it !in '\u0021'..'\u007e' }.takeIf { it >= 0 }?.let { i -> + // Only the valid part before the offset: the rest could be a value + // pasted into the name. + invalid("$where: the header name that starts '${name.take(i)}' has an invalid character at offset $i") + } + value.indexOfFirst { it != '\t' && it !in '\u0020'..'\u007e' }.takeIf { it >= 0 }?.let { i -> + invalid("$where: the value of header '$name' has an invalid character at offset $i") + } + // A backstop for any rule OkHttp adds later. Its message is not used. + try { + Headers.Builder().add(name, value) + } catch (e: IllegalArgumentException) { + invalid("$where: the HTTP client does not accept header '$name'") + } + } + } + + private fun invalid(message: String): Nothing = throw InvalidEntryException(message) + + private fun ReadableMap.isSet(key: String) = hasKey(key) && getType(key) != ReadableType.Null + + private fun ReadableMap.string(key: String): String? = + if (hasKey(key) && getType(key) == ReadableType.String) getString(key) else null + + private fun ReadableMap.number(key: String): Double? = + if (hasKey(key) && getType(key) == ReadableType.Number) getDouble(key) else null + + private fun ReadableMap.bool(key: String): Boolean? = + if (hasKey(key) && getType(key) == ReadableType.Boolean) getBoolean(key) else null + + private fun ReadableMap.map(key: String): ReadableMap? = + if (hasKey(key) && getType(key) == ReadableType.Map) getMap(key) else null + + private fun ReadableMap.array(key: String): ReadableArray? = + if (hasKey(key) && getType(key) == ReadableType.Array) getArray(key) else null +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/EntryRun.kt b/android/src/main/java/ai/openspace/backgroundupload/EntryRun.kt new file mode 100644 index 00000000..aa130380 --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/EntryRun.kt @@ -0,0 +1,326 @@ +package ai.openspace.backgroundupload + +import kotlinx.coroutines.CancellationException +import okhttp3.RequestBody +import java.io.IOException +import java.util.UUID +import kotlin.math.max +import kotlin.math.min + +/** + * The run of one queue entry, and the one attempt step both transfers use. + * No Android types: [EntryWorker] is the shell that holds the per-id gate + * and is the real [TransferHost]. + * + * The run: take the entry (queued → running), run the transfer, then + * settle, park, or release. The body kind at run time picks the transfer, + * [SimpleTransfer] or [ChunkedTransfer], so a kind change under a queued run + * is safe. The entry is read from the store at start and again before every + * attempt, so fresh headers and a new expiresAt reach a running run. + */ +internal class EntryRun( + val entryId: String, + val store: QueueStore, + val ops: WorkerOps, + val host: TransferHost, +) { + /** A 401/403 under the headers of [headerGeneration] (the entry's value the attempt sent). */ + class ParkException(val headerGeneration: Int) : Exception("awaiting auth") + + /** A backoff too long to wait here. */ + class BackoffException(val nextAttemptAt: Long, val streak: Int) : Exception("released for backoff") + + class ExpiredException : Exception("expired before completion") + + /** The entry was paused (whole queue or its key) between the module's pause and the work cancel reaching us. */ + class PausedException : Exception("paused") + + /** How one attempt ended, after the retry table. */ + sealed class AttemptResult { + data class Accepted(val response: UploadResponse) : AttemptResult() + + /** A 401/403. [reissue]: newer headers arrived while it was in flight, so send again now. */ + data class Auth(val headerGeneration: Int, val reissue: Boolean) : AttemptResult() + + object Transient : AttemptResult() + + data class Terminal(val errorKind: String, val message: String, val response: UploadResponse?) : AttemptResult() + } + + /** One attempt: the entry it ran under, where it went, the retry policy, and how it ended. */ + class Attempt( + val entry: QueueEntry, + val url: String, + val policy: RetryClassifier.Policy, + val result: AttemptResult, + ) { + val method: String get() = entry.descriptor!!.method + } + + companion object { + /** The poll while the network is unusable (offline, or waiting for wifi). */ + const val CONNECTIVITY_POLL_MS = 10_000L + + /** The poll while a short backoff remainder runs out before the run starts. */ + private const val WAIT_POLL_MS = 10_000L + + /** The descriptor's headers, the part's over them, and X-Request-Id over all. */ + fun headersFor(d: Descriptor, part: Part?, requestId: String): Map { + val withPart = if (part == null) d.headers else HeaderMap.merge(d.headers, part.headers) + return HeaderMap.merge(withPart, mapOf("X-Request-Id" to requestId)) + } + } + + var generation = 0 + private set + + /** The bytes of the current attempt, as last reported. A failed simple entry settles with them. */ + @Volatile + var liveBytes = 0L + private set + + /** + * Returns when the run is over. Throws a CancellationException after it + * handled a stop, or an IOException after a store write failed (the + * caller returns retry; nothing settled). + */ + suspend fun run() { + val initial = store.load(entryId) ?: return // forgotten while queued + if (initial.legacy) return + if (initial.state == EntryState.AWAITING_AUTH) { + // The expiry wake of a parked entry. No attempt ran here, so the + // stored bytes stand. + if (RetryClassifier.isExpired(host.now(), initial.expiresAt)) { + ops.settle(entryId, initial.generation, expired(initial).copy(bytesSent = null)) + } + return + } + if (initial.state != EntryState.QUEUED && initial.state != EntryState.RUNNING) return + if (ops.settings().isPaused(initial.key)) return + if (!waitUntilDue(initial)) return + val entry = ops.begin(entryId) ?: return + generation = entry.generation + + var current = entry + var first = true + while (true) { + try { + if (!first) current = ops.latest(entryId, generation) + first = false + host.foreground(current) + val settlement = transfer(current) + endProgress(completed = settlement is Settlement.Completed) + ops.settle(entryId, generation, settlement) + return + } catch (park: ParkException) { + endProgress(completed = false) + // REISSUE: updateHeaders() landed while this attempt was in flight. + if (ops.park(entryId, generation, park.headerGeneration) != WorkerOps.ParkResult.REISSUE) return + } catch (backoff: BackoffException) { + endProgress(completed = false) + ops.release(entryId, generation, backoff.nextAttemptAt, backoff.streak) + return + } catch (error: ExpiredException) { + endProgress(completed = false) + ops.settle(entryId, generation, expired(current)) + return + } catch (error: NotOwnedException) { + endProgress(completed = false) + return + } catch (error: PausedException) { + endProgress(completed = false) + return + } catch (error: CancellationException) { + // A system stop moves a running entry back to queued. A pause or a + // cancel already moved it; then this does nothing. + endProgress(completed = false) + if (host.stoppedByTimeout()) releaseAfterTimeout() else ops.stopped(entryId, generation) + throw error + } catch (error: IOException) { + // A store write failed (disk full, directory briefly unwritable). + // The attempt step classifies every network IOException itself, so + // one that lands here is storage: transient, no outcome. Back to + // queued; the caller returns retry. + endProgress(completed = false) + ops.stopped(entryId, generation) + throw error + } catch (error: Throwable) { + endProgress(completed = false) + val d = current.descriptor + ops.settle( + entryId, + generation, + Settlement.Failed( + errorKind = "unknown", + message = error.message ?: error.javaClass.simpleName, + response = null, + partIndex = null, + url = d?.reportUrl ?: "", + method = d?.method ?: "POST", + bytesSent = liveBytesOf(current), + ), + ) + return + } + } + } + + private suspend fun transfer(entry: QueueEntry): Settlement = + if (entry.body?.kind == StagedBody.CHUNKED) ChunkedTransfer(this).run(entry) + else SimpleTransfer(this).run(entry) + + private fun liveBytesOf(entry: QueueEntry): Long? = + if (entry.body?.kind == StagedBody.CHUNKED) null else liveBytes + + private fun expired(entry: QueueEntry) = Settlement.Failed( + errorKind = "expired", + message = "expired before completion", + response = null, + partIndex = null, + url = entry.descriptor?.reportUrl ?: "", + method = entry.descriptor?.method ?: "POST", + bytesSent = liveBytesOf(entry), + ) + + /** + * The system stopped the run at its time limit. That happens to a + * headless run whose foreground start was denied (API 31+), after about + * 10 minutes. Starting again at once would send the body from byte 0 on + * every run, so this is one more transient failure: the next backoff + * step, then a wake. + */ + private fun releaseAfterTimeout() { + runCatching { + val e = store.load(entryId) + if (!EntryTransitions.isOwnedRun(e, generation)) return + val streak = e!!.backoffStreak + 1 + val now = host.now() + val backoff = RetryClassifier.backoffMs(policy(e), streak) + ops.release(entryId, generation, RetryClassifier.nextAttemptAt(now, backoff, e.expiresAt), streak) + }.onFailure { Diag.error("could not release '$entryId' after a timeout stop", it) } + } + + /** + * Sleeps out a short backoff remainder. False when the wait is long (the + * wake run comes back for it) or the entry is no longer queued. + */ + private suspend fun waitUntilDue(initial: QueueEntry): Boolean { + var e = initial + while (true) { + val at = e.nextAttemptAt ?: return true + val remaining = at - host.now() + if (remaining <= 0) return true + if (remaining > RetryClassifier.IN_WORKER_BACKOFF_MAX_MS) return false + host.sleep(min(remaining, WAIT_POLL_MS)) + e = store.load(entryId) ?: return false + if (e.state != EntryState.QUEUED && e.state != EntryState.RUNNING) return false + } + } + + // MARK: - helpers for the transfers + + /** + * Waits until the network fits the entry's Wi-Fi rule: its own wifiOnly, + * else the queue's setting. Re-reads the settings and the entry at every + * poll, so a setWifiOnly() toggle reaches an entry that follows it. + */ + suspend fun waitForNetwork() { + while (true) { + val s = ops.settings() + val entry = ops.latest(entryId, generation) + if (s.isPaused(entry.key)) throw PausedException() + if (RetryClassifier.isExpired(host.now(), entry.expiresAt)) throw ExpiredException() + if (host.connectivity(s.wifiOnlyFor(entry)) == Connectivity.Ok) return + host.sleep(CONNECTIVITY_POLL_MS) + } + } + + fun policy(entry: QueueEntry) = + RetryClassifier.policy(ops.settings().retry, entry.descriptor?.retry) + + /** + * One HTTP attempt, the step both transfers share: write the attempt + * ahead (attempts + 1, its X-Request-Id), send, emit the attempt event, + * and classify. [partIndex] is null for a simple entry. [body] builds the + * request body from the entry the attempt runs under. [fileExists] tells a + * missing payload from a network failure. + */ + suspend fun attempt( + partIndex: Int?, + body: (Descriptor, Part?) -> RequestBody?, + onProgress: (Long) -> Unit, + fileExists: () -> Boolean, + ): Attempt { + val requestId = UUID.randomUUID().toString() + val entry = ops.recordAttempt(entryId, generation, requestId) + // The generation of the headers this attempt sends: both come from the same entry. + val headerGeneration = entry.headerGeneration + val d = entry.descriptor!! + val part = partIndex?.let { d.parts!![it] } + val url = part?.url ?: d.url!! + val policy = policy(entry) + + val response = try { + host.send(TransferRequest(url, d.method, headersFor(d, part, requestId), body(d, part)), onProgress) + } catch (error: CancellationException) { + throw error + } catch (error: Throwable) { + // A failed probe reads as present, so a flaky check is a retryable + // network error and not a terminal "file gone". + val verdict = RetryClassifier.classifyFailure(error, runCatching(fileExists).getOrDefault(true)) + host.attempt( + AttemptEvent.ofFailure( + entry, requestId, url, partIndex, RetryClassifier.failureKind(verdict), + error.message ?: error.javaClass.simpleName, host.now(), + ), + ) + val result = if (verdict is RetryClassifier.Verdict.Terminal) { + AttemptResult.Terminal(verdict.errorKind, verdict.message, null) + } else AttemptResult.Transient + return Attempt(entry, url, policy, result) + } + + val verdict = RetryClassifier.classifyResponse(response.code, response.body, d.accept, policy.exempt) + host.attempt( + AttemptEvent.ofResponse( + entry, requestId, url, partIndex, response, verdict == RetryClassifier.Verdict.Accepted, host.now(), + ), + ) + val result = when (verdict) { + RetryClassifier.Verdict.Accepted -> AttemptResult.Accepted(response) + RetryClassifier.Verdict.Auth -> + AttemptResult.Auth(headerGeneration, ops.hasNewerHeaders(entryId, generation, headerGeneration)) + RetryClassifier.Verdict.Transient -> AttemptResult.Transient + is RetryClassifier.Verdict.Terminal -> AttemptResult.Terminal("http", verdict.message, response) + } + return Attempt(entry, url, policy, result) + } + + /** + * A short backoff waits here, with the row still running and showing + * nextAttemptAt; a long one throws [BackoffException] to release the run. + */ + suspend fun backoffOrRelease(policy: RetryClassifier.Policy, streak: Int, expiresAt: Long) { + val backoff = RetryClassifier.backoffMs(policy, streak) + val now = host.now() + if (backoff <= RetryClassifier.IN_WORKER_BACKOFF_MAX_MS) { + val wait = min(backoff, max(0L, expiresAt - now)) + ops.backingOff(entryId, generation, now + wait) + host.sleep(wait) + return + } + throw BackoffException(RetryClassifier.nextAttemptAt(now, backoff, expiresAt), streak) + } + + fun progressStarted(total: Long, sent: Long) { + liveBytes = sent + host.progressStarted(entryId, total, sent) + } + + fun reportProgress(sent: Long, total: Long) { + liveBytes = sent + host.progress(entryId, sent, total) + } + + private fun endProgress(completed: Boolean) = host.progressEnded(entryId, completed) +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/EntryTransitions.kt b/android/src/main/java/ai/openspace/backgroundupload/EntryTransitions.kt new file mode 100644 index 00000000..d099d8df --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/EntryTransitions.kt @@ -0,0 +1,114 @@ +package ai.openspace.backgroundupload + +/** + * The state changes of one entry, as pure functions. [QueueController] and + * [WorkerOps] apply them inside `QueueStore.compute`, so each one is atomic + * against the other side. + */ +object EntryTransitions { + + /** A worker takes a queued entry. */ + fun toRunning(e: QueueEntry, now: Long) = + e.copy(state = EntryState.RUNNING, nextAttemptAt = null, updatedAt = now) + + /** A 401/403 under the current header generation. No backoff. */ + fun toParked(e: QueueEntry, headerGeneration: Int, now: Long) = e.copy( + state = EntryState.AWAITING_AUTH, + parkedGeneration = headerGeneration, + backoffStreak = 0, + updatedAt = now, + ) + + /** + * A short backoff that the worker waits out itself. The row stays running + * and shows when the next attempt is due. + */ + fun toBackingOff(e: QueueEntry, nextAttemptAt: Long, now: Long) = + e.copy(nextAttemptAt = nextAttemptAt, updatedAt = now) + + /** One attempt starts: attempts + 1, its X-Request-Id, and no pending backoff. */ + fun toAttempt(e: QueueEntry, requestId: String, now: Long) = e.copy( + attempts = e.attempts + 1, + lastRequestId = requestId, + nextAttemptAt = null, + updatedAt = now, + ) + + /** A backoff too long to wait inside the worker. The streak is kept so the next wait keeps growing. */ + fun toReleased(e: QueueEntry, nextAttemptAt: Long, streak: Int, now: Long) = e.copy( + state = EntryState.QUEUED, + nextAttemptAt = nextAttemptAt, + backoffStreak = streak, + updatedAt = now, + ) + + fun toSettled(e: QueueEntry, state: EntryState, eventId: String, bytesSent: Long, now: Long) = e.copy( + state = state, + settledEventId = eventId, + bytesSent = bytesSent, + nextAttemptAt = null, + parkedGeneration = null, + updatedAt = now, + ) + + /** + * pause(). parkedGeneration is kept so resume() can return the entry to + * awaiting-auth. nextAttemptAt is cleared: resume() retries at once. + */ + fun toPaused(e: QueueEntry, now: Long) = + e.copy(state = EntryState.PAUSED, nextAttemptAt = null, updatedAt = now) + + /** resume(): back to awaiting-auth only when no updateHeaders() came in between. */ + fun toResumed(e: QueueEntry, headerGeneration: Int, now: Long): QueueEntry = + if (e.parkedGeneration != null && e.parkedGeneration == headerGeneration) { + e.copy(state = EntryState.AWAITING_AUTH, updatedAt = now) + } else { + e.copy(state = EntryState.QUEUED, parkedGeneration = null, updatedAt = now) + } + + /** A system stop of a running worker. WorkManager runs the row again. No outcome. */ + fun toStopped(e: QueueEntry, now: Long) = e.copy(state = EntryState.QUEUED, updatedAt = now) + + /** updateHeaders() on a parked entry. */ + fun toUnparked(e: QueueEntry, paused: Boolean, now: Long) = e.copy( + state = if (paused) EntryState.PAUSED else EntryState.QUEUED, + parkedGeneration = null, + updatedAt = now, + ) + + /** + * Whether a worker of [generation] may still settle [e]. Not after a + * cancel, a replace, or a settle. Under pause only an [accepted] response + * settles: the server already took it. A failure waits for resume, which + * runs the entry again. + */ + fun canSettle(e: QueueEntry?, generation: Int, accepted: Boolean) = + e != null && e.generation == generation && e.isLive && (accepted || e.state != EntryState.PAUSED) + + /** The entry state a journal record puts its entry in. */ + fun stateOf(record: EventJournal.SettledRecord): EntryState = + EntryState.values().firstOrNull { it.wire == record.state } ?: EntryState.ERROR + + /** A settled entry, and the other records of its life to ack. */ + data class Journaled(val entry: QueueEntry, val extraEventIds: List) + + /** + * A live entry with a journal record of its own generation: the settle + * journaled, then its store write was lost (a process death, a failed + * save). Apply the newest record; do not run the request again. Null when + * there is no such record. The boot sweep and a worker's begin share it. + */ + fun journaledSettle(e: QueueEntry, records: List, now: Long): Journaled? { + if (!e.isLive || e.legacy) return null + val own = records.filter { it.id == e.id && it.generation == e.generation } + val latest = own.maxByOrNull { it.at } ?: return null + return Journaled( + toSettled(e, stateOf(latest), latest.eventId, latest.bytesSent, now), + own.filter { it !== latest }.map { it.eventId }, + ) + } + + /** Whether a worker of [generation] still owns the running entry. */ + fun isOwnedRun(e: QueueEntry?, generation: Int) = + e != null && e.generation == generation && e.state == EntryState.RUNNING +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/EntryWorker.kt b/android/src/main/java/ai/openspace/backgroundupload/EntryWorker.kt new file mode 100644 index 00000000..d967862b --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/EntryWorker.kt @@ -0,0 +1,147 @@ +package ai.openspace.backgroundupload + +import android.app.NotificationManager +import android.content.Context +import androidx.work.CoroutineWorker +import androidx.work.ForegroundInfo +import androidx.work.WorkInfo +import androidx.work.WorkerParameters +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.delay +import kotlinx.coroutines.sync.withPermit +import kotlinx.coroutines.withContext + +/** + * The WorkManager shell of one entry's run. The input data holds only the + * entry id. The worker acquires the per-id gate, then [EntryRun] does the + * run; this class is its [TransferHost]: OkHttp, the clock, progress, and + * the notification. + * + * [UploadWorker] and [ChunkedUploadWorker] are the two class names + * WorkManager knows; both run this same code. Every run returns success + * (see [WorkManagerScheduler] for why), except a store failure before the + * run could settle, which returns retry. A v9 row, which has no entry id, + * exits at once in silence. + */ +open class EntryWorker(protected val context: Context, params: WorkerParameters) : + CoroutineWorker(context, params) { + + companion object { + private const val GATE_POLL_MS = 100L + } + + private val store by lazy { QueueStore.get(context) } + private val ops by lazy { + WorkerOps( + store, + EventJournal.get(context), + QueueSettingsStore.get(context), + EventReporter, + WorkManagerScheduler(context), + ) + } + private val config by lazy { NotificationConfig.load(context) } + private val notificationManager = + context.getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager + + @Volatile + private var connectivity = Connectivity.Ok + + @Volatile + private var showsNotification = false + + final override suspend fun doWork(): Result = withContext(Dispatchers.IO) { + val id = inputData.getString(WorkManagerScheduler.ENTRY_ID_KEY) ?: return@withContext Result.success() + // Acquire before the first store read: a cancel-then-enqueue can start + // this run while the old one still winds down. + while (!WorkerGate.tryAcquire(id, this@EntryWorker)) delay(GATE_POLL_MS) + try { + EntryRun(id, store, ops, host).run() + Result.success() + } catch (error: CancellationException) { + throw error + } catch (error: Throwable) { + // A store failure (disk full at start or during an attempt). Nothing + // was settled; try the run again later. + Diag.error("run of '$id' failed before it could settle; retrying", error) + Result.retry() + } finally { + WorkerGate.release(id, this@EntryWorker) + } + } + + private val host = object : TransferHost { + override fun now() = System.currentTimeMillis() + + override suspend fun sleep(ms: Long) = delay(ms) + + override suspend fun send(request: TransferRequest, onProgress: (Long) -> Unit): UploadResponse = + transferSemaphore.withPermit { okhttpSend(uploadHttpClient, request, onProgress) } + + override fun connectivity(wifiOnly: Boolean): Connectivity { + connectivity = validateConnectivity(context, wifiOnly) + updateNotification() + return connectivity + } + + override suspend fun foreground(entry: QueueEntry) { + showsNotification = entry.descriptor?.noNotification == false + startForeground() + } + + override fun progressStarted(id: String, total: Long, sent: Long) { + UploadProgress.add(id, total) + UploadProgress.set(id, sent) + } + + override fun progress(id: String, sent: Long, total: Long) { + UploadProgress.set(id, sent) + EventReporter.progress(id, sent, total) + updateNotification() + } + + override fun progressEnded(id: String, completed: Boolean) { + if (completed) UploadProgress.complete(id) else UploadProgress.remove(id) + EventReporter.flushProgress(id) + EventReporter.dropProgress(id) + } + + override fun attempt(event: AttemptEvent) = EventReporter.attempt(event) + + override fun stoppedByTimeout() = stopReason == WorkInfo.STOP_REASON_TIMEOUT + } + + // MARK: - notification + + // v9 rules. A suppressed notification means no foreground mode. A denied + // foreground start (API 31+, app in the background: the usual case for a + // WorkManager relaunch) is not a failure; the transfer runs without + // foreground priority, under JobScheduler's time limit (see + // [EntryRun.releaseAfterTimeout]). Any other failure is logged and the run + // goes on. + private suspend fun startForeground() { + if (!showsNotification) return + try { + ensureNotificationChannel(notificationManager, config) + setForeground(getForegroundInfo()) + } catch (error: CancellationException) { + throw error + } catch (error: Throwable) { + if (!isForegroundStartDenied(error)) Diag.warn("foreground start failed; running without it", error) + } + } + + private fun updateNotification() { + if (!showsNotification) return + runCatching { + notificationManager.notify( + config.systemNotificationId, + buildUploadNotification(context, config, connectivity), + ) + } + } + + override suspend fun getForegroundInfo(): ForegroundInfo = + uploadForegroundInfo(config, buildUploadNotification(context, config, connectivity)) +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/EventJournal.kt b/android/src/main/java/ai/openspace/backgroundupload/EventJournal.kt new file mode 100644 index 00000000..3f560947 --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/EventJournal.kt @@ -0,0 +1,360 @@ +package ai.openspace.backgroundupload + +import android.content.Context +import com.facebook.react.bridge.WritableMap +import com.google.gson.Gson +import java.io.File +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit + +/** + * The durable record of settled outcomes (completed, error, cancelled). A + * record is written BEFORE the outcome is emitted to JS and deleted only when + * JS acknowledges it. One JSON file per record, `.json`, written + * with tmp + fsync + rename, so a crash mid-write can not corrupt another + * record. + * + * [maxEntries] is a runaway guard: if nothing ever acknowledges, the oldest + * records are dropped, except those a row still names. It only fires in + * that broken case. + */ +class EventJournal( + private val dir: File, + private val maxEntries: Int = MAX_ENTRIES, + /** Runs a task after a delay. Tests pass a manual one. */ + private val retryLater: (delayMs: Long, task: () -> Unit) -> Unit = ::onTimer, +) { + + /** RawResponse. [status] is null for a chunked completion. */ + data class Response( + val status: Int?, + val headers: Map?, + val body: String?, + val bodyTruncated: Boolean, + ) { + fun toMap(): Map = LinkedHashMap().apply { + status?.let { put("status", it.toDouble()) } + headers?.let { put("headers", it) } + body?.let { put("body", it) } + put("bodyTruncated", bodyTruncated) + } + + companion object { + fun of(response: UploadResponse): Response { + val (body, cut) = BodyCap.cap(response.body, BodyCap.SETTLED_MAX_BYTES) + return Response(response.code, response.headers, body, response.truncated || cut) + } + + /** A chunked completion: N parts, no one response. */ + val NONE = Response(null, null, null, false) + } + } + + /** One settled outcome, in the SettledEvent shape plus [generation]. */ + data class SettledRecord( + val eventId: String, + val id: String, + val key: String, + val varsJson: String, + val at: Long, + val attempts: Int, + val requestId: String?, + /** + * How many times this outcome reached a JS listener: 1 after a live + * emit, 0 when it was journaled with no listener; +1 per later delivery + * (replay, re-emit). [append] sets it. + */ + val deliveries: Int, + /** The entry state this outcome puts it in. */ + val state: String, + val bytesSent: Long, + val totalBytes: Long, + val url: String, + val method: String, + val partIndex: Int?, + /** completed | error | cancelled */ + val kind: String, + /** completed; also error with errorKind http. */ + val response: Response?, + val errorKind: String?, + val message: String?, + val cancelReason: String?, + /** The entry life this belongs to (same-id rule 6). */ + val generation: Int, + ) { + fun toMap(): Map = LinkedHashMap().apply { + put("eventId", eventId) + put("id", id) + put("key", key) + put("vars", runCatching { JsonBridge.parse(varsJson) }.getOrNull()) + put("at", at.toDouble()) + put("attempts", attempts.toDouble()) + requestId?.let { put("requestId", it) } + put("deliveries", deliveries.toDouble()) + put("state", state) + put("bytesSent", bytesSent.toDouble()) + put("totalBytes", totalBytes.toDouble()) + put("url", url) + put("method", method) + partIndex?.let { put("partIndex", it.toDouble()) } + put("kind", kind) + when (kind) { + KIND_COMPLETED -> put("response", (response ?: Response.NONE).toMap()) + KIND_ERROR -> put("error", LinkedHashMap().apply { + put("errorKind", errorKind ?: "unknown") + put("message", message ?: "") + response?.let { put("response", it.toMap()) } + partIndex?.let { put("partIndex", it.toDouble()) } + }) + KIND_CANCELLED -> put("cancelReason", cancelReason ?: "user") + } + } + + fun toWritableMap(): WritableMap = JsonBridge.toWritableMap(toMap()) + } + + companion object { + const val KIND_COMPLETED = "completed" + const val KIND_ERROR = "error" + const val KIND_CANCELLED = "cancelled" + + const val MAX_ENTRIES = 1000 + + /** The first wait before a held record is written again. It doubles up to [RETRY_MAX_MS]. */ + const val RETRY_MS = 5_000L + const val RETRY_MAX_MS = 600_000L + private val gson = Gson() + + // Event ids are UUIDs that native mints. ackEvents takes ids from JS, and + // an id is a file name here, so anything else is ignored. + private val EVENT_ID = Regex("^[A-Za-z0-9-]{1,64}$") + + fun isValidEventId(id: String) = EVENT_ID.matches(id) + + private val timer by lazy { + Executors.newSingleThreadScheduledExecutor { r -> + Thread(r, "RNFileUploader.journal").apply { isDaemon = true } + } + } + + private fun onTimer(delayMs: Long, task: () -> Unit) { + timer.schedule(task, delayMs, TimeUnit.MILLISECONDS) + } + + @Volatile + private var instance: EventJournal? = null + + /** v10 records live in `rnbgupload-settled`. The v9 `rnbgupload-events` is read once by [LegacyImport]. */ + fun get(context: Context): EventJournal = + instance ?: synchronized(this) { + instance ?: EventJournal(File(context.filesDir, "rnbgupload-settled")).also { instance = it } + } + } + + init { + dir.mkdirs() + } + + /** + * The JS listener, set by [drain] and cleared by [stopListening]. While it + * is set, a new record starts at 1 delivery and the caller emits it live. + * While it is null, a record starts at 0 and the next drain delivers it. + * Both decisions take this object's lock, so a record is either in the + * drain or emitted live, never both and never neither. + */ + private var listener: Any? = null + + /** + * Records whose file write failed, by eventId. They count as journaled in + * every read and ack, and a timer writes them once the disk allows. They + * live only in memory: a process death loses them. + */ + private val held = LinkedHashMap() + private var retryScheduled = false + + private fun fileFor(eventId: String) = File(dir, "$eventId.json") + + /** + * Writes [record] with deliveries 1 when a listener is set, else 0, and + * returns it as written. The caller emits it only when deliveries > 0. + * Throws IOException when the write failed; nothing is kept then. + * + * [keep] names the records the prune must not delete (every eventId a row + * names). It runs only when the journal is over its cap, under this lock. + * Callers hold the store lock (lock order: store, then journal). + */ + @Synchronized + fun append(record: SettledRecord, keep: () -> Set = { emptySet() }): SettledRecord { + val stamped = stamp(record) + AtomicFiles.writeText(fileFor(stamped.eventId), gson.toJson(stamped)) + pruneToMax(keep) + return stamped + } + + /** + * As [append], but never throws. A failed write holds the record in + * memory and a timer tries it again. For a worker's settle: the request + * already ran, and a thrown error would send it again. + */ + @Synchronized + fun appendOrHold(record: SettledRecord, keep: () -> Set = { emptySet() }): SettledRecord = + try { + append(record, keep) + } catch (t: Throwable) { + Diag.error("journal append failed for ${record.eventId}; held in memory", t) + val stamped = stamp(record) + held[stamped.eventId] = stamped + scheduleRetry(RETRY_MS) + stamped + } + + private fun stamp(record: SettledRecord): SettledRecord { + val response = record.response?.let { r -> + val (body, cut) = BodyCap.cap(r.body, BodyCap.SETTLED_MAX_BYTES) + if (cut) r.copy(body = body, bodyTruncated = true) else r + } + return record.copy(deliveries = if (listener != null) 1 else 0, response = response) + } + + /** Whether [eventId] is held in memory, not yet on disk. */ + @Synchronized + fun isHeld(eventId: String) = eventId in held + + /** Writes every held record. Returns true when none is left. */ + @Synchronized + fun writeHeld(): Boolean { + val written = held.values.filter { r -> + runCatching { AtomicFiles.writeText(fileFor(r.eventId), gson.toJson(r)) }.isSuccess + } + written.forEach { held.remove(it.eventId) } + return held.isEmpty() + } + + private fun scheduleRetry(delayMs: Long) { + if (retryScheduled) return + retryScheduled = true + retryLater(delayMs) { + synchronized(this) { + retryScheduled = false + if (!writeHeld()) scheduleRetry(minOf(delayMs * 2, RETRY_MAX_MS)) + } + } + } + + // Prunes by file time (no parsing), sparing the records rows name. Never + // throws: it only runs in the broken case where nothing acknowledges. + private fun pruneToMax(keep: () -> Set) { + try { + dir.listFiles { f -> f.name.endsWith(AtomicFiles.TMP_SUFFIX) }?.forEach { it.delete() } + val files = dir.listFiles { f -> f.extension == "json" } ?: return + if (files.size <= maxEntries) return + val named = keep() + files.filter { it.nameWithoutExtension !in named } + .sortedBy { it.lastModified() } + .take(files.size - maxEntries) + .forEach { it.delete() } + } catch (t: Throwable) { + Diag.error("journal prune failed", t) + } + } + + /** + * getUnacknowledgedEvents(): sets [owner] as the listener and returns + * every record, oldest first, each counted as one more delivery. One lock + * spans both, see [listener]. + */ + @Synchronized + fun drain(owner: Any, isActive: () -> Boolean = { true }): List { + // [isActive] is read under this lock. A module torn down before its + // queued drain runs does not become the listener, and counts nothing. + if (!isActive()) return emptyList() + listener = owner + return unacknowledged().mapNotNull { incrementDeliveries(it.eventId) } + } + + /** Clears the listener, only when [owner] set it: a reload builds the next module before it tears down this one. */ + @Synchronized + fun stopListening(owner: Any) { + if (listener === owner) listener = null + } + + @Synchronized + fun isListening() = listener != null + + /** The listener a live settled event goes to: the module whose JS drained last. */ + @Synchronized + fun listener(): Any? = listener + + /** + * A re-emit of a journaled outcome (same-id rule 7): one more delivery, + * returned for a live emit. Null when no listener is set (the next drain + * delivers it) or the record is gone. + */ + @Synchronized + fun redeliver(eventId: String): SettledRecord? = + if (listener == null) null else incrementDeliveries(eventId) + + /** Every record, oldest first, the held ones included. Corrupt files are skipped. */ + @Synchronized + fun unacknowledged(): List = + ((dir.listFiles { f -> f.extension == "json" } ?: emptyArray()).mapNotNull { read(it) } + held.values) + .sortedWith(compareBy { it.at }.thenBy { it.eventId }) + + @Synchronized + fun find(eventId: String): SettledRecord? = + if (isValidEventId(eventId)) held[eventId] ?: read(fileFor(eventId)) else null + + @Synchronized + fun forEntry(id: String): List = unacknowledged().filter { it.id == id } + + /** + * One more delivery of [eventId]: rewrites the record with deliveries + 1 + * and returns it. Null when the record is gone. When the rewrite fails the + * incremented record is still returned, so the delivery goes ahead. + */ + @Synchronized + fun incrementDeliveries(eventId: String): SettledRecord? { + val record = find(eventId) ?: return null + val next = record.copy(deliveries = record.deliveries + 1) + if (eventId in held) { + held[eventId] = next + return next + } + try { + AtomicFiles.writeText(fileFor(eventId), gson.toJson(next)) + } catch (t: Throwable) { + Diag.error("journal deliveries update failed for $eventId", t) + } + return next + } + + /** Idempotent. Unknown and malformed ids are ignored. */ + @Synchronized + fun ack(eventIds: List) { + eventIds.filter { isValidEventId(it) }.forEach { + held.remove(it) + fileFor(it).delete() + } + } + + /** Acks every record of entry [id]: cancel() of a settled entry forgets its outcomes. */ + @Synchronized + fun ackEntry(id: String) { + ack(forEntry(id).map { it.eventId }) + } + + @Suppress("SENSELESS_COMPARISON", "USELESS_ELVIS") + private fun read(file: File): SettledRecord? { + if (!file.exists()) return null + val r = runCatching { gson.fromJson(file.readText(), SettledRecord::class.java) }.getOrNull() + ?: return null + // Gson does not run constructors. Check every field JS relies on. + if (r.eventId == null || r.id == null || r.key == null || r.kind == null || r.state == null) return null + return r.copy( + varsJson = r.varsJson ?: "null", + url = r.url ?: "", + method = r.method ?: "POST", + deliveries = r.deliveries.coerceAtLeast(0), + ) + } +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/EventReporter.kt b/android/src/main/java/ai/openspace/backgroundupload/EventReporter.kt new file mode 100644 index 00000000..ea3cc62a --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/EventReporter.kt @@ -0,0 +1,69 @@ +package ai.openspace.backgroundupload + +import com.facebook.react.bridge.Arguments + +/** The events that queue transitions produce. [EventReporter] sends them to JS; tests record them. */ +interface QueueEvents { + fun state(row: RequestRow) + + /** + * The caller journaled [record] first, and emits it only when its + * deliveries is above 0: [EventJournal] decides that under its lock. + * [listener] is [EventJournal.listener]: the module whose JS drained, so + * the delivery the journal counted goes to that JS. + */ + fun settled(record: EventJournal.SettledRecord, listener: Any) +} + +/** + * Sends live events to JS through the module's codegen emitters. With no + * module (a headless worker, a reload) an event is dropped. Settled outcomes + * are journaled before they reach here, so a dropped one is replayed from + * the journal. + */ +object EventReporter : QueueEvents { + + private val throttle = ProgressThrottle { id, sent, total -> + val module = UploaderModule.instance ?: return@ProgressThrottle + module.emitProgress(Arguments.createMap().apply { + putString("id", id) + putDouble("bytesSent", sent.toDouble()) + putDouble("totalBytes", total.toDouble()) + }) + } + + override fun state(row: RequestRow) { + val module = UploaderModule.instance ?: return + module.emitState(JsonBridge.toWritableMap(row.toMap())) + } + + // Not UploaderModule.instance: a reload sets that before the new JS + // subscribes, and the journal counted this delivery for the listener. + override fun settled(record: EventJournal.SettledRecord, listener: Any) { + val module = listener as? UploaderModule ?: return + module.emitSettled(record.toWritableMap()) + } + + /** Moves the row's bytesSent in memory and emits through the throttle. */ + fun progress(id: String, sent: Long, total: Long) { + RequestIndex.shared.setBytes(id, sent) + throttle.offer(id, sent, total, isForeground()) + } + + fun flushProgress(id: String) = throttle.flush(id) + + fun dropProgress(id: String) = throttle.drop(id) + + fun attempt(event: AttemptEvent) { + val module = UploaderModule.instance ?: return + module.emitAttempt(event.toWritableMap()) + } + + fun notification() { + val module = UploaderModule.instance ?: return + module.emitNotification(Arguments.createMap()) + } + + private fun isForeground(): Boolean = + runCatching { UploaderModule.instance?.isForeground() == true }.getOrDefault(false) +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/JsonBridge.kt b/android/src/main/java/ai/openspace/backgroundupload/JsonBridge.kt new file mode 100644 index 00000000..fe84fbe7 --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/JsonBridge.kt @@ -0,0 +1,164 @@ +package ai.openspace.backgroundupload + +import com.facebook.react.bridge.Arguments +import com.facebook.react.bridge.ReadableArray +import com.facebook.react.bridge.ReadableMap +import com.facebook.react.bridge.ReadableType +import com.facebook.react.bridge.WritableArray +import com.facebook.react.bridge.WritableMap +import com.google.gson.Gson +import com.google.gson.JsonElement +import com.google.gson.JsonParser +import com.google.gson.JsonPrimitive +import com.google.gson.Strictness +import com.google.gson.stream.JsonReader +import com.google.gson.stream.JsonToken +import java.io.StringReader +import kotlin.math.abs +import kotlin.math.floor + +/** + * Moves values between the bridge (ReadableMap, WritableMap), plain Kotlin + * values (Map, List, String, Double, Boolean, null), and JSON text. `vars` + * and `data` cross from JS as JSON text and are stored as it came; native + * parses them back only to hand objects to JS. + * + * Numbers: RN gives every JS number to Kotlin as a Double. [numberText] + * writes a Double with no fraction as an integer, so a header value 5 is + * "5", as JSON.stringify writes it, not "5.0". + */ +object JsonBridge { + private val gson = Gson() + + // 2^53. Above this a Double can not hold every integer, so it keeps the + // Double form. + private const val MAX_SAFE_INTEGER = 9_007_199_254_740_992.0 + + fun fromReadable(map: ReadableMap): Map { + val out = LinkedHashMap() + val keys = map.keySetIterator() + while (keys.hasNextKey()) { + val key = keys.nextKey() + out[key] = valueOf(map, key) + } + return out + } + + fun fromReadableArray(array: ReadableArray): List = + (0 until array.size()).map { i -> + when (array.getType(i)) { + ReadableType.Null -> null + ReadableType.Boolean -> array.getBoolean(i) + ReadableType.Number -> array.getDouble(i) + ReadableType.String -> array.getString(i) + ReadableType.Map -> array.getMap(i)?.let { fromReadable(it) } + ReadableType.Array -> array.getArray(i)?.let { fromReadableArray(it) } + } + } + + /** The plain value at [key]. Null for an absent key. */ + fun valueOf(map: ReadableMap, key: String): Any? { + if (!map.hasKey(key)) return null + return when (map.getType(key)) { + ReadableType.Null -> null + ReadableType.Boolean -> map.getBoolean(key) + ReadableType.Number -> map.getDouble(key) + ReadableType.String -> map.getString(key) + ReadableType.Map -> map.getMap(key)?.let { fromReadable(it) } + ReadableType.Array -> map.getArray(key)?.let { fromReadableArray(it) } + } + } + + /** JSON text to plain values. Throws on malformed text. Numbers come back as Double. */ + fun parse(json: String): Any? = fromElement(JsonParser.parseString(json)) + + /** + * Whether [text] is one strict JSON value, as JSON.stringify writes it. + * Any top-level value counts, so "null" is valid. Lenient forms (single + * quotes, bare keys, trailing text) are not. + */ + fun isJson(text: String): Boolean = runCatching { + val reader = JsonReader(StringReader(text)).apply { setStrictness(Strictness.STRICT) } + gson.getAdapter(JsonElement::class.java).read(reader) + reader.peek() == JsonToken.END_DOCUMENT + }.getOrDefault(false) + + /** A number as JSON would print it: an integer when it has no fraction. */ + fun numberText(d: Double): String = gson.toJson(number(d)) + + private fun number(d: Double): JsonPrimitive = + if (d.isFinite() && d == floor(d) && abs(d) < MAX_SAFE_INTEGER) JsonPrimitive(d.toLong()) + else JsonPrimitive(d) + + private fun fromElement(element: JsonElement): Any? = when { + element.isJsonNull -> null + element.isJsonObject -> LinkedHashMap().apply { + element.asJsonObject.entrySet().forEach { (k, v) -> put(k, fromElement(v)) } + } + element.isJsonArray -> element.asJsonArray.map { fromElement(it) } + else -> { + val p = element.asJsonPrimitive + when { + p.isBoolean -> p.asBoolean + p.isNumber -> p.asDouble + else -> p.asString + } + } + } + + /** + * Plain values to the bridge. The factories default to the native ones. The + * JVM tests pass JavaOnlyMap and JavaOnlyArray, because the native ones need + * the React Native C++ library. + */ + fun toWritableMap( + map: Map, + newMap: () -> WritableMap = Arguments::createMap, + newArray: () -> WritableArray = Arguments::createArray, + ): WritableMap { + val out = newMap() + map.forEach { (k, v) -> putValue(out, k, v, newMap, newArray) } + return out + } + + fun toWritableArray( + list: List, + newMap: () -> WritableMap = Arguments::createMap, + newArray: () -> WritableArray = Arguments::createArray, + ): WritableArray { + val out = newArray() + list.forEach { v -> + when (v) { + null -> out.pushNull() + is Boolean -> out.pushBoolean(v) + is Number -> out.pushDouble(v.toDouble()) + is String -> out.pushString(v) + is Map<*, *> -> out.pushMap(toWritableMap(stringKeys(v), newMap, newArray)) + is List<*> -> out.pushArray(toWritableArray(v, newMap, newArray)) + else -> out.pushString(v.toString()) + } + } + return out + } + + fun putValue( + map: WritableMap, + key: String, + value: Any?, + newMap: () -> WritableMap = Arguments::createMap, + newArray: () -> WritableArray = Arguments::createArray, + ) { + when (value) { + null -> map.putNull(key) + is Boolean -> map.putBoolean(key, value) + is Number -> map.putDouble(key, value.toDouble()) + is String -> map.putString(key, value) + is Map<*, *> -> map.putMap(key, toWritableMap(stringKeys(value), newMap, newArray)) + is List<*> -> map.putArray(key, toWritableArray(value, newMap, newArray)) + else -> map.putString(key, value.toString()) + } + } + + private fun stringKeys(map: Map<*, *>): Map = + map.entries.associate { (k, v) -> k.toString() to v } +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/LegacyImport.kt b/android/src/main/java/ai/openspace/backgroundupload/LegacyImport.kt new file mode 100644 index 00000000..da5b19ab --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/LegacyImport.kt @@ -0,0 +1,104 @@ +package ai.openspace.backgroundupload + +import android.content.Context +import com.google.gson.Gson +import java.io.File + +/** + * First v10 launch. Each v9 journal entry becomes a read-only settled row + * with key `legacy` and the v9 upload id. Nothing is delivered; the app + * reads the rows with getRequests() and cancels them. v9 chunked manifests + * are left in place: a same-id enqueue adopts them. + * + * The caller cancels the v9 WorkManager rows after this returns true. The + * order is safe: under v10 code a v9 row exits at once (it has no entry + * id), so nothing can write a v9 journal file after the import. + * + * Runs once, guarded by a marker file. A failed row save leaves the marker + * unwritten, so the next launch tries again. + */ +object LegacyImport { + const val MARKER = "v9-imported" + const val KEY = "legacy" + const val V9_JOURNAL_DIR = "rnbgupload-events" + + /** The v9 journal Entry, every field nullable: Gson reads whatever is there. */ + data class V9Entry( + val eventId: String?, + val uploadId: String?, + val type: String?, + val timestamp: Long?, + ) + + private val gson = Gson() + + /** Returns true when the import ran at this launch (no marker yet). */ + fun runOnce(context: Context, store: QueueStore): Boolean = + runOnce(File(QueueStore.rootDir(context), MARKER), File(context.filesDir, V9_JOURNAL_DIR), store) + + /** [runOnce] over plain files, for the JVM tests. */ + internal fun runOnce(marker: File, v9Dir: File, store: QueueStore): Boolean { + if (marker.exists()) return false + val complete = import(v9Dir, store) + if (complete) { + runCatching { AtomicFiles.writeText(marker, "1") } + .onFailure { Diag.error("could not write the v9 import marker", it) } + } + return true + } + + /** + * Imports every v9 journal entry in [v9Dir]. The newest entry per upload + * id wins. Returns false when a row could not be saved (its file is kept). + */ + internal fun import(v9Dir: File, store: QueueStore): Boolean { + val files = v9Dir.listFiles { f -> f.extension == "json" } ?: return true + val read = files.mapNotNull { f -> + val entry = runCatching { gson.fromJson(f.readText(), V9Entry::class.java) }.getOrNull() + if (entry == null) { + Diag.warn("v9 journal file unreadable, skipped: ${f.name}") + f.delete() + null + } else f to entry + } + var complete = true + read.groupBy { it.second.uploadId }.forEach { (uploadId, group) -> + val newest = group.maxByOrNull { it.second.timestamp ?: 0L }!!.second + val row = if (uploadId == null) null else legacyRow(newest) + val saved = when { + row == null -> true + store.load(row.id) != null -> true // a v10 entry already owns the id + else -> runCatching { store.save(row) }.isSuccess + } + if (saved) group.forEach { it.first.delete() } else complete = false + } + return complete + } + + internal fun legacyRow(entry: V9Entry): QueueEntry? { + val id = entry.uploadId ?: return null + val state = when (entry.type) { + "completed" -> EntryState.COMPLETED + "error" -> EntryState.ERROR + "cancelled" -> EntryState.CANCELLED + else -> return null + } + val at = entry.timestamp ?: 0L + return QueueEntry( + id = id, + key = KEY, + varsJson = "null", + descriptor = null, + body = null, + state = state, + attempts = 0, + bytesSent = 0, + totalBytes = 0, + expiresAt = at, + createdAt = at, + updatedAt = at, + generation = 1, + legacy = true, + ) + } +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/NotificationConfig.kt b/android/src/main/java/ai/openspace/backgroundupload/NotificationConfig.kt new file mode 100644 index 00000000..ec947bfd --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/NotificationConfig.kt @@ -0,0 +1,83 @@ +package ai.openspace.backgroundupload + +import android.content.Context +import com.facebook.react.bridge.ReadableMap +import com.google.gson.Gson + +/** + * The text and the identity of the upload progress notification. JS sets it + * one time with `configure()`. The class writes it to SharedPreferences. Thus a + * worker that WorkManager relaunches — with no JS, possibly in a process where + * React never initialized — shows the same text. Each field falls back to a + * library default. Thus uploads work when `configure()` was never called. + */ +data class NotificationConfig( + val notificationId: String, + val notificationTitle: String, + val notificationTitleNoInternet: String, + val notificationTitleNoWifi: String, + val notificationChannel: String, +) { + // The id given to NotificationManager. All uploads share the configured id. + // Thus they share one notification, and its progress bar is the total. + val systemNotificationId get() = notificationId.hashCode() + + companion object { + const val DEFAULT_NOTIFICATION_CHANNEL = "background-upload" + + val DEFAULTS = NotificationConfig( + notificationId = DEFAULT_NOTIFICATION_CHANNEL, + notificationTitle = "Uploading…", + notificationTitleNoInternet = "Waiting for connection…", + notificationTitleNoWifi = "Waiting for Wi-Fi…", + notificationChannel = DEFAULT_NOTIFICATION_CHANNEL, + ) + + private const val PREFS_NAME = "rnbgupload-config" + private const val PREFS_KEY = "notificationConfig" + private val gson = Gson() + + fun fromReadableMap(map: ReadableMap) = NotificationConfig( + notificationId = map.getString(NotificationConfig::notificationId.name) + ?: DEFAULTS.notificationId, + notificationTitle = map.getString(NotificationConfig::notificationTitle.name) + ?: DEFAULTS.notificationTitle, + notificationTitleNoInternet = map.getString(NotificationConfig::notificationTitleNoInternet.name) + ?: DEFAULTS.notificationTitleNoInternet, + notificationTitleNoWifi = map.getString(NotificationConfig::notificationTitleNoWifi.name) + ?: DEFAULTS.notificationTitleNoWifi, + notificationChannel = map.getString(NotificationConfig::notificationChannel.name) + ?: DEFAULTS.notificationChannel, + ) + + // Gson does not use the constructor. Thus a blob from a build with + // different fields, or a corrupt blob, can make a non-null field null. + // Each field falls back alone. A bad blob must give default text. It must + // never crash a worker. + @Suppress("SENSELESS_COMPARISON") + fun fromJson(json: String?): NotificationConfig { + val parsed = json?.let { + runCatching { gson.fromJson(it, NotificationConfig::class.java) }.getOrNull() + } ?: return DEFAULTS + return NotificationConfig( + notificationId = parsed.notificationId ?: DEFAULTS.notificationId, + notificationTitle = parsed.notificationTitle ?: DEFAULTS.notificationTitle, + notificationTitleNoInternet = parsed.notificationTitleNoInternet + ?: DEFAULTS.notificationTitleNoInternet, + notificationTitleNoWifi = parsed.notificationTitleNoWifi + ?: DEFAULTS.notificationTitleNoWifi, + notificationChannel = parsed.notificationChannel ?: DEFAULTS.notificationChannel, + ) + } + + fun save(context: Context, config: NotificationConfig) { + prefs(context).edit().putString(PREFS_KEY, gson.toJson(config)).apply() + } + + fun load(context: Context): NotificationConfig = + fromJson(prefs(context).getString(PREFS_KEY, null)) + + private fun prefs(context: Context) = + context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE) + } +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/NotificationReceiver.kt b/android/src/main/java/ai/openspace/backgroundupload/NotificationReceiver.kt new file mode 100644 index 00000000..945d59e8 --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/NotificationReceiver.kt @@ -0,0 +1,17 @@ +package ai.openspace.backgroundupload + +import android.content.BroadcastReceiver +import android.content.Context +import android.content.Intent + +// there's no way to directly open the app from the notification without reloading it, +// so we use a BroadcastReceiver to listen to the notification intent +class NotificationReceiver : BroadcastReceiver() { + override fun onReceive(context: Context?, intent: Intent?) { + context ?: return + val packageName = context.packageName + val launchIntent = context.packageManager.getLaunchIntentForPackage(packageName) + context.startActivity(launchIntent) + EventReporter.notification() + } +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/ProgressThrottle.kt b/android/src/main/java/ai/openspace/backgroundupload/ProgressThrottle.kt new file mode 100644 index 00000000..f4032d20 --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/ProgressThrottle.kt @@ -0,0 +1,55 @@ +package ai.openspace.backgroundupload + +/** + * Limits progress events per id: at most one per second while the app is in + * the foreground, one per 10 minutes in the background. A value held back is + * kept as pending; [flush] sends it (the trailing edge on settle, park, + * release, and stop). + */ +class ProgressThrottle( + private val clock: () -> Long = System::currentTimeMillis, + private val emit: (id: String, sent: Long, total: Long) -> Unit, +) { + companion object { + const val FOREGROUND_MS = 1_000L + const val BACKGROUND_MS = 600_000L + } + + private class Slot(var lastEmitAt: Long?, var pending: Pair?) + + private val slots = HashMap() + + fun offer(id: String, sent: Long, total: Long, foreground: Boolean) { + val interval = if (foreground) FOREGROUND_MS else BACKGROUND_MS + val now = clock() + val send = synchronized(slots) { + val slot = slots.getOrPut(id) { Slot(null, null) } + val last = slot.lastEmitAt + if (last == null || now - last >= interval) { + slot.lastEmitAt = now + slot.pending = null + true + } else { + slot.pending = sent to total + false + } + } + if (send) emit(id, sent, total) + } + + /** Sends the held-back value once, if there is one. */ + fun flush(id: String) { + val pending = synchronized(slots) { + val slot = slots[id] ?: return + val p = slot.pending ?: return + slot.pending = null + slot.lastEmitAt = clock() + p + } + emit(id, pending.first, pending.second) + } + + fun drop(id: String) { + synchronized(slots) { slots.remove(id) } + } +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/QueueController.kt b/android/src/main/java/ai/openspace/backgroundupload/QueueController.kt new file mode 100644 index 00000000..f8a9ed69 --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/QueueController.kt @@ -0,0 +1,483 @@ +package ai.openspace.backgroundupload + +import java.io.IOException +import java.util.UUID + +/** + * Every transition that JS causes: enqueue, pause, resume, cancel, + * setWifiOnly, updateHeaders, ack, and the boot sweep. [UploaderModule] calls + * it from one single-thread executor, so these calls never overlap each other. + * Workers change entries at the same time; every change here is inside the + * store lock, so each one is atomic against them. + * + * Order of a change: journal (when there is an outcome), store, work + * schedule, then events. Events go out after the store lock is released. + */ +class QueueController( + private val store: QueueStore, + private val journal: EventJournal, + private val settings: QueueSettingsStore, + private val events: QueueEvents, + private val scheduler: WorkScheduler, + private val isWorkerRunning: (id: String) -> Boolean = WorkerGate::isRunning, + private val clock: () -> Long = System::currentTimeMillis, +) { + + // MARK: - enqueue + + private class Enqueued(val entry: QueueEntry?, val reEmit: EventJournal.SettledRecord?) + + /** A body staged before the store lock was taken, and the generation it was staged for. */ + private class PreStaged(val generation: Int, val staged: BodyStaging.Staged) + + /** Test seam: runs between the staging outside the lock and the commit under it. */ + internal var afterPreStage: () -> Unit = {} + + /** + * Persists the entry and every staged byte, then schedules it. Returns the + * id. Throws [QueueException] with E_RUNNING, E_FILE_MISSING, E_STORAGE, or + * E_INVALID. + * + * A copied body is staged before the store lock when the decision can not + * change meanwhile (see [EnqueueRules.preStageGeneration]), so a large + * copy does not block every worker transition. Under the lock the + * decision is made again; a staged body that no longer fits is deleted + * and the body is staged again under the lock. + */ + fun enqueue(p: EntryParsing.Parsed): String { + val pre = preStage(p) + var preUsed = false + fun preFor(generation: Int): BodyStaging.Staged? = + pre?.takeIf { it.generation == generation }?.staged?.also { preUsed = true } + val result = try { + store.locked { decideAndCommit(p, ::preFor) } + } finally { + if (pre != null && !preUsed) discard(p.id, pre) + } + result.reEmit?.let { record -> journal.listener()?.let { events.settled(record, it) } } + result.entry?.let { entry -> + scheduleRun(entry) + events.state(entry.toRow()) + } + return p.id + } + + private fun preStage(p: EntryParsing.Parsed): PreStaged? { + val generation = store.locked { + val existing = store.load(p.id) + val v9 = v9ManifestFor(existing, p.id) + val action = EnqueueRules.decide(existing, v9, p.descriptor) { journal.find(it) != null } + EnqueueRules.preStageGeneration(existing, action, p.descriptor) + } ?: return null + val staged = stageOrThrow(p.descriptor, store.entryDir(p.id), generation) + afterPreStage() + return PreStaged(generation, staged) + } + + /** Deletes a pre-staged body that the commit did not use, unless the stored entry points at it. */ + private fun discard(id: String, pre: PreStaged) { + val file = pre.staged.body.fileName?.let { java.io.File(store.entryDir(id), it) } ?: return + val current = store.load(id) + if (current == null || store.bodyFile(current) != file) file.delete() + } + + /** Runs under the store lock. [preStaged] returns the body staged outside the lock for a generation, if any. */ + private fun decideAndCommit(p: EntryParsing.Parsed, preStaged: (generation: Int) -> BodyStaging.Staged?): Enqueued { + val existing = store.load(p.id) + val v9 = v9ManifestFor(existing, p.id) + val s = settings.load() + val now = clock() + val dir = store.entryDir(p.id) + return when (val action = EnqueueRules.decide(existing, v9, p.descriptor) { journal.find(it) != null }) { + EnqueueRules.Action.RejectRunning -> throw QueueException( + QueueException.E_RUNNING, + "entry '${p.id}' is running; a different body is accepted once it stops", + ) + // With no listener yet, the next drain delivers it. + is EnqueueRules.Action.ReEmit -> Enqueued(null, journal.redeliver(action.eventId)) + EnqueueRules.Action.Resume -> { + val next = EnqueueRules.resumed(existing!!, p, s.isPaused(p.key), s.headerGeneration, now) + saveOrThrow(next) + Enqueued(next, null) + } + EnqueueRules.Action.Create -> { + val staged = preStaged(1) ?: stageOrThrow(p.descriptor, dir, 1) + commit(EnqueueRules.created(p, staged, null, s.isPaused(p.key), s.headerGeneration, now)) + } + is EnqueueRules.Action.AdoptV9 -> { + val incoming = p.descriptor.parts + val parts = incoming?.let { EnqueueRules.adoptedParts(action.manifest, it) } + // The same parts resume over the v9 blob, as a same-body enqueue does. + val keepOwned = incoming != null && ChunkedParts.sameParts(action.manifest.parts, incoming) + val staged = stageOrThrow(p.descriptor.copy(parts = parts), dir, action.generation, store.blobFile(p.id), keepOwned) + commit(EnqueueRules.adopted(p, staged, parts, existing, action.generation, s.isPaused(p.key), s.headerGeneration, now)) + } + EnqueueRules.Action.Replace -> { + val old = existing!! + // Different parts: a present caller file wins; the old blob is the fallback. + val ownedBlob = if (old.body?.kind == StagedBody.CHUNKED) store.bodyFile(old) else null + val staged = preStaged(old.generation + 1) + ?: stageOrThrow(p.descriptor, dir, old.generation + 1, ownedBlob) + commit(EnqueueRules.replaced(old, p, staged, s.isPaused(p.key), s.headerGeneration, now)) + } + } + } + + /** A v9 manifest counts only where no v10 entry owns the id, or the owner is its legacy row. */ + private fun v9ManifestFor(existing: QueueEntry?, id: String): LegacyManifest? = + if (existing == null || existing.legacy) store.legacyManifest(id) else null + + private fun stageOrThrow( + d: Descriptor, + dir: java.io.File, + generation: Int, + ownedBlob: java.io.File? = null, + keepOwned: Boolean = false, + ): BodyStaging.Staged = + try { + BodyStaging.stage(d, dir, generation, ownedBlob, keepOwned) + } catch (e: QueueException) { + throw e + } catch (e: IOException) { + throw QueueException(QueueException.E_STORAGE, "could not stage the body: ${e.message}") + } + + /** Saves a newly staged entry. On failure the new body file is removed; the old entry stays as it was. */ + private fun commit(next: QueueEntry): Enqueued { + try { + store.save(next) + } catch (e: IOException) { + if (next.body?.kind != StagedBody.CHUNKED) store.bodyFile(next)?.delete() + throw QueueException(QueueException.E_STORAGE, "could not save the entry: ${e.message}") + } + store.pruneUnreferenced(next) + return Enqueued(next, null) + } + + private fun saveOrThrow(next: QueueEntry) { + try { + store.save(next) + } catch (e: IOException) { + throw QueueException(QueueException.E_STORAGE, "could not save the entry: ${e.message}") + } + } + + // MARK: - queue control + + /** + * [keys] null is the whole-queue gate; a list adds those keys to the + * paused set, and an empty list changes nothing. Each live row that is now + * paused (gate on, or its key in the set) moves to paused and its work + * stops. No outcome. + */ + fun pause(keys: List? = null) { + if (keys != null && keys.isEmpty()) return + val s = settings.update { + if (keys == null) it.copy(paused = true) else it.copy(pausedKeys = it.pausedKeys + keys) + } + val now = clock() + val paused = transformAll { e -> + if (e.isLive && e.state != EntryState.PAUSED && s.isPaused(e.key)) EntryTransitions.toPaused(e, now) else e + } + paused.forEach { scheduler.cancel(it.id) } + paused.forEach { events.state(it.toRow()) } + } + + /** + * Undoes the pause of the same scope: [keys] null turns the gate off, a + * list removes those keys from the set. A row comes back only when no + * scope still pauses it, so the gate on + its key resumed stays paused. + */ + fun resume(keys: List? = null) { + if (keys != null && keys.isEmpty()) return + val s = settings.update { + if (keys == null) it.copy(paused = false) else it.copy(pausedKeys = it.pausedKeys - keys.toSet()) + } + val now = clock() + val resumed = transformAll { e -> + if (e.state == EntryState.PAUSED && !s.isPaused(e.key)) EntryTransitions.toResumed(e, s.headerGeneration, now) else e + } + resumed.forEach { events.state(it.toRow()) } + // Every queued entry, not only the resumed ones: a run is idempotent. + store.all().forEach { scheduleRun(it) } + } + + /** + * Live: journal a 'cancelled' (user) outcome, then forget after its ack. + * Settled (or legacy): forget now, row, bytes, and its unacknowledged + * outcomes. Unknown: no-op. + * + * A failed journal write rejects E_STORAGE and changes nothing: the entry + * goes on, and JS can call cancel() again. + * + * A failed entry save after the journal write also rejects E_STORAGE, but + * the cancel is durable: the work is stopped and the record is emitted. + * Its ack, the next cancel(), a worker's begin or settle, or the boot + * sweep applies it. A live entry that already has a record of its own + * generation gets that record applied, not a second outcome. + */ + fun cancel(id: String) { + var stop = false + var journaled: EventJournal.SettledRecord? = null + var saved: QueueEntry? = null + try { + store.locked { + stop = true // unknown or settled: stop any stray work, as before + val e = store.load(id) ?: return@locked + if (!e.isLive || e.legacy) { + journal.ackEntry(id) + store.remove(id) + return@locked + } + stop = false // a live entry: only once an outcome is journaled + val now = clock() + EntryTransitions.journaledSettle(e, journal.forEntry(id), now)?.let { + stop = true + saveOrThrow(it.entry) + journal.ack(it.extraEventIds) + saved = it.entry + return@locked + } + val record = cancelledRecord(e, now) + journaled = try { + journal.append(record) { store.referencedEventIds() + record.eventId } + } catch (error: IOException) { + throw QueueException(QueueException.E_STORAGE, "could not journal the cancel: ${error.message}") + } + stop = true + // The journaled record is the cancel. A failed save does not undo + // it: the ack, the next cancel(), or the boot sweep applies the + // record. So cancel resolves, as on iOS; it rejects only when + // nothing changed. + val next = EntryTransitions.toSettled(e, EntryState.CANCELLED, record.eventId, e.bytesSent, now) + try { + store.save(next) + saved = next + } catch (error: IOException) { + Diag.error("cancel journaled '$id' but could not save it; the record is applied later", error) + } + } + } finally { + // Once an outcome is journaled, the worker must stop even when the + // save failed: a running request would settle a second outcome. + if (stop) scheduler.cancel(id) + journaled?.let { record -> + if (record.deliveries > 0) journal.listener()?.let { events.settled(record, it) } + } + saved?.let { events.state(it.toRow()) } + } + } + + private fun cancelledRecord(e: QueueEntry, now: Long) = EventJournal.SettledRecord( + eventId = UUID.randomUUID().toString(), + id = e.id, + key = e.key, + varsJson = e.varsJson, + at = now, + attempts = e.attempts, + requestId = e.lastRequestId, + deliveries = 0, // the journal sets it + state = EntryState.CANCELLED.wire, + bytesSent = e.bytesSent, + totalBytes = e.totalBytes, + url = e.descriptor?.reportUrl ?: "", + method = e.descriptor?.method ?: "POST", + partIndex = null, + kind = EventJournal.KIND_CANCELLED, + response = null, + errorKind = null, + message = null, + cancelReason = "user", + generation = e.generation, + ) + + fun setWifiOnly(enabled: Boolean) { + settings.update { it.copy(wifiOnly = enabled) } + } + + fun configureRetry(defaults: RetryDefaults) { + settings.update { it.copy(retry = defaults) } + } + + /** + * Bumps the header generation, merges [patch] into every entry not yet + * forgotten, and requeues the parked ones. Workers compare each entry's + * own headerGeneration, which changes here under the store lock together + * with its headers. So a worker that gets a 401 either sees the patched + * entry and re-issues, or parks first and is requeued here. + */ + fun updateHeaders(patch: Map) { + val s = settings.update { it.copy(headerGeneration = it.headerGeneration + 1) } + val now = clock() + val unparkedIds = mutableSetOf() + val changed = transformAll { e -> + val patched = e.withHeadersPatched(patch, s.headerGeneration) + if (patched.state != EntryState.AWAITING_AUTH) return@transformAll patched + unparkedIds += e.id + EntryTransitions.toUnparked(patched, s.isPaused(patched.key), now) + } + changed.filter { it.id in unparkedIds }.forEach { entry -> + scheduleRun(entry) + events.state(entry.toRow()) + } + } + + // MARK: - journal + + /** + * getUnacknowledgedEvents(): [listener] becomes the JS listener, and every + * unacknowledged outcome is returned, each counted as one more delivery. + */ + fun unacknowledged(listener: Any, isActive: () -> Boolean = { true }): List = + journal.drain(listener, isActive) + + /** + * Removes the records. An acked completed or cancelled outcome of the + * entry's current life forgets the entry: row and bytes. An error keeps + * the row until cancel() or a same-id enqueue. Unknown ids are ignored. + * + * A record of the entry's current life on a live entry means the settle + * journaled and emitted, but its store write failed. The record is applied + * first, as the boot sweep would; without that, the sweep finds no record + * after this ack and runs the finished request again. When that save fails + * too, the record stays unacked so the sweep can apply it later. + */ + fun ack(eventIds: List) { + val forgotten = mutableListOf() + val repaired = mutableListOf() + store.locked { + eventIds.forEach { eventId -> + val record = journal.find(eventId) ?: return@forEach + var e = store.load(record.id) + if (e != null && e.isLive && !e.legacy && e.generation == record.generation) { + val next = EntryTransitions.toSettled(e, EntryTransitions.stateOf(record), record.eventId, record.bytesSent, clock()) + if (!trySave(next)) return@forEach + repaired += next + e = next + } + journal.ack(listOf(eventId)) + if (record.kind == EventJournal.KIND_ERROR || e == null) return@forEach + if (e.generation == record.generation && e.settledEventId == record.eventId) { + store.remove(record.id) + forgotten += record.id + } + } + } + forgotten.forEach { scheduler.cancel(it) } + repaired.filter { it.id !in forgotten }.forEach { events.state(it.toRow()) } + } + + // MARK: - boot sweep + + /** + * Repairs what a process death can leave, then schedules queued work. An + * entry whose worker runs in this process is skipped: that worker finishes + * its own transition. Run at module init, after [LegacyImport]. + * + * 1. A live entry with a journal record of its own generation: the process + * died between the journal append and the store transition. Apply it. + * 2. A running entry with no worker: a process death mid-run. Queue it. + * A live row that disagrees with the paused settings, the gate or its + * key (a death partway through pause() or resume()): make it agree. + * 3. A settled entry with records of its generation other than its own: + * orphans from a cancel race. Ack them. + * 4. A completed or cancelled entry whose own record is gone: the ack + * landed but the forget did not. Forget it. + * 5. Schedule every queued entry, and the expiry wake of every parked one. + */ + fun sweep() { + val now = clock() + val changed = mutableListOf() + val toForget = mutableListOf() + val toSchedule = mutableListOf() + val s = settings.load() + store.locked { + val records = journal.unacknowledged().groupBy { it.id } + for (e in store.all()) { + if (e.legacy || isWorkerRunning(e.id)) continue + val own = records[e.id].orEmpty().filter { it.generation == e.generation } + if (e.isLive) { + val journaled = EntryTransitions.journaledSettle(e, own, now) + if (journaled != null) { + if (trySave(journaled.entry)) { + journal.ack(journaled.extraEventIds) + changed += journaled.entry + } + continue + } + var cur = e + if (e.state == EntryState.RUNNING) { + cur = EntryTransitions.toStopped(e, now) + } + // A process death partway through pause() or resume() leaves rows + // that disagree with the settings (the gate or the paused keys). + val paused = s.isPaused(e.key) + if (paused && cur.state != EntryState.PAUSED) { + cur = EntryTransitions.toPaused(cur, now) + } else if (!paused && cur.state == EntryState.PAUSED) { + cur = EntryTransitions.toResumed(cur, s.headerGeneration, now) + } + if (cur !== e) { + if (trySave(cur)) changed += cur else continue + } + if (!paused || cur.state == EntryState.AWAITING_AUTH) toSchedule += cur + } else { + val orphans = own.filter { it.eventId != e.settledEventId } + if (orphans.isNotEmpty()) journal.ack(orphans.map { it.eventId }) + val forgettable = e.state == EntryState.COMPLETED || e.state == EntryState.CANCELLED + if (forgettable && own.none { it.eventId == e.settledEventId }) { + store.remove(e.id) + toForget += e.id + } + } + } + } + toForget.forEach { scheduler.cancel(it) } + toSchedule.forEach { scheduleRun(it, keepWake = true) } + changed.forEach { events.state(it.toRow()) } + } + + private fun trySave(entry: QueueEntry): Boolean = try { + store.save(entry) + true + } catch (e: IOException) { + Diag.error("could not save '${entry.id}'", e) + false + } + + // MARK: - helpers + + /** + * Runs [entry] when it is queued. A backoff longer than a worker waits goes + * to the wake; a parked entry gets a wake at its expiry, so it settles + * 'expired' on time. + */ + private fun scheduleRun(entry: QueueEntry, keepWake: Boolean = false) { + val now = clock() + when (entry.state) { + EntryState.QUEUED -> { + val at = entry.nextAttemptAt + if (at != null && at - now > RetryClassifier.IN_WORKER_BACKOFF_MAX_MS) { + scheduler.scheduleWake(entry, at, replace = !keepWake) + } else { + scheduler.schedule(entry) + } + } + EntryState.AWAITING_AUTH -> scheduler.scheduleWake(entry, entry.expiresAt, replace = !keepWake) + else -> Unit + } + } + + /** Applies [transform] to every non-legacy entry under the store lock. Returns the ones it changed. */ + private fun transformAll(transform: (QueueEntry) -> QueueEntry): List { + val changed = mutableListOf() + store.locked { + store.all().filter { !it.legacy }.forEach { e -> + store.compute(e.id) { cur -> + if (cur == null) null else transform(cur).also { if (it !== cur) changed += it } + } + } + } + return changed + } +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/QueueEntry.kt b/android/src/main/java/ai/openspace/backgroundupload/QueueEntry.kt new file mode 100644 index 00000000..09f1b364 --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/QueueEntry.kt @@ -0,0 +1,230 @@ +package ai.openspace.backgroundupload + +import com.google.gson.annotations.SerializedName + +/** The row states. [wire] is the RequestState string JS sees. */ +enum class EntryState(val wire: String) { + @SerializedName("queued") QUEUED("queued"), + @SerializedName("running") RUNNING("running"), + @SerializedName("awaiting-auth") AWAITING_AUTH("awaiting-auth"), + @SerializedName("paused") PAUSED("paused"), + @SerializedName("completed") COMPLETED("completed"), + @SerializedName("error") ERROR("error"), + @SerializedName("cancelled") CANCELLED("cancelled"); + + val isLive get() = this == QUEUED || this == RUNNING || this == AWAITING_AUTH || this == PAUSED +} + +/** One multipart field. [path] is the caller's file, read only at staging. */ +data class FormPart( + val name: String, + val contentType: String, + val string: String?, + val path: String?, + val fileName: String?, +) + +/** A request's `retry`, as a partial override of the configure() defaults. */ +data class RetryOverride( + val baseMs: Long?, + val maxMs: Long?, + val jitter: Double?, + val exempt: List?, +) + +/** What request(vars) returned, as native needs it to run. */ +data class Descriptor( + /** Null only with parts. */ + val url: String?, + val method: String, + /** The merged headers, plus the Content-Type that staging sets. */ + val headers: Map, + /** JSON text of `data`. It is also the bytes of the staged body. */ + val dataJson: String?, + val form: List?, + /** The caller's path, with any file:// prefix removed. */ + val file: String?, + /** The accepted flags live here. */ + val parts: List?, + val accept: List, + val retry: RetryOverride?, + val noNotification: Boolean, + /** The request's own Wi-Fi rule. Null follows the queue's setWifiOnly(). */ + val wifiOnly: Boolean? = null, +) { + val bodyKind: String + get() = when { + parts != null -> StagedBody.CHUNKED + file != null -> StagedBody.FILE + form != null -> StagedBody.MULTIPART + dataJson != null -> StagedBody.JSON + else -> StagedBody.NONE + } + + /** The url to report for this request: the descriptor's, or the last part's. */ + val reportUrl: String get() = url ?: parts?.lastOrNull()?.url ?: "" +} + +/** + * Where the request body is on disk. [fileName] is relative to the entry + * directory, so a moved app data directory does not break it. + */ +data class StagedBody( + val kind: String, + val fileName: String?, + val boundary: String?, + val totalBytes: Long, +) { + companion object { + const val NONE = "none" + const val JSON = "json" + const val MULTIPART = "multipart" + const val FILE = "file" + const val CHUNKED = "chunked" + } +} + +/** One queue entry. [QueueStore] persists it as `entry.json` with Gson. */ +data class QueueEntry( + val id: String, + val key: String, + /** "null" for null vars. */ + val varsJson: String, + /** Null only for a legacy row. */ + val descriptor: Descriptor?, + /** Null only for a legacy row. */ + val body: StagedBody?, + val state: EntryState, + /** Attempts in this life. Chunked: across every part. */ + val attempts: Int, + val bytesSent: Long, + val totalBytes: Long, + val expiresAt: Long, + val createdAt: Long, + val updatedAt: Long, + /** Set while the entry waits out a backoff: queued for a long one, running for a short one. */ + val nextAttemptAt: Long? = null, + /** The consecutive transient failures before the last release. The next backoff continues from it. */ + val backoffStreak: Int = 0, + /** The settings header generation the headers were last merged at. */ + val headerGeneration: Int = 0, + /** Set while awaiting-auth (and kept under pause): the header generation it parked under. */ + val parkedGeneration: Int? = null, + /** +1 each time a settled entry reopens, and on a different-body replace. Journal records carry it. */ + val generation: Int = 1, + /** The journal record of this life's outcome. */ + val settledEventId: String? = null, + /** The X-Request-Id of the last attempt. */ + val lastRequestId: String? = null, + val legacy: Boolean = false, +) { + val isLive get() = state.isLive + val isSettled get() = !state.isLive + + fun toRow() = RequestRow( + id = id, + key = key, + varsJson = varsJson, + state = state.wire, + bytesSent = bytesSent, + totalBytes = totalBytes, + attempts = attempts, + updatedAt = updatedAt, + nextAttemptAt = nextAttemptAt, + createdAt = createdAt, + ) + + /** + * Whether [incoming] carries the same body. A different body kind, a + * different url or method, or different content is a different body. + * Chunked compares the parts (the path is ignored: the owned blob is the + * truth, as in v9). + */ + fun sameBodyAs(incoming: Descriptor): Boolean { + val stored = descriptor ?: return false + if (stored.bodyKind != incoming.bodyKind) return false + if (stored.method != incoming.method) return false + return when (stored.bodyKind) { + StagedBody.CHUNKED -> ChunkedParts.sameParts(stored.parts!!, incoming.parts!!) + StagedBody.FILE -> stored.url == incoming.url && stored.file == incoming.file + StagedBody.MULTIPART -> stored.url == incoming.url && stored.form == incoming.form + StagedBody.JSON -> stored.url == incoming.url && stored.dataJson == incoming.dataJson + else -> stored.url == incoming.url + } + } + + /** + * updateHeaders(): the patch replaces same-named headers (any case) and adds + * the rest. A part that carries its own copy of a patched header gets the + * new value too, because a stale per-part Authorization would shadow the + * fresh one. + */ + fun withHeadersPatched(patch: Map, generation: Int): QueueEntry { + val d = descriptor ?: return copy(headerGeneration = generation) + return copy( + descriptor = d.copy( + headers = HeaderMap.merge(d.headers, patch), + parts = d.parts?.map { part -> + val shared = patch.filterKeys { name -> HeaderMap.contains(part.headers, name) } + if (shared.isEmpty()) part else part.copy(headers = HeaderMap.merge(part.headers, shared)) + }, + ), + headerGeneration = generation, + ) + } +} + +/** One row of getRequests() and of a state event. */ +class RequestRow( + val id: String, + val key: String, + val varsJson: String, + val state: String, + val bytesSent: Long, + val totalBytes: Long, + val attempts: Int, + val updatedAt: Long, + val nextAttemptAt: Long?, + /** Sort order only; not sent to JS. */ + val createdAt: Long, +) { + /** vars parsed back to an object, once. A malformed text reads as null. */ + val vars: Any? by lazy { runCatching { JsonBridge.parse(varsJson) }.getOrNull() } + + fun withBytes(sent: Long) = RequestRow( + id, key, varsJson, state, sent, totalBytes, attempts, updatedAt, nextAttemptAt, createdAt, + ) + + /** The RequestRow shape. nextAttemptAt only when set. */ + fun toMap(): Map = LinkedHashMap().apply { + put("id", id) + put("key", key) + put("vars", vars) + put("state", state) + put("bytesSent", bytesSent.toDouble()) + put("totalBytes", totalBytes.toDouble()) + put("attempts", attempts.toDouble()) + put("updatedAt", updatedAt.toDouble()) + nextAttemptAt?.let { put("nextAttemptAt", it.toDouble()) } + } +} + +/** Header maps whose names match without regard to case. */ +object HeaderMap { + fun contains(headers: Map, name: String) = + headers.keys.any { it.equals(name, ignoreCase = true) } + + fun get(headers: Map, name: String): String? = + headers.entries.firstOrNull { it.key.equals(name, ignoreCase = true) }?.value + + /** [over] replaces same-named entries of [base] (any case); its spelling is kept. */ + fun merge(base: Map, over: Map): Map { + val out = LinkedHashMap() + base.forEach { (k, v) -> if (!contains(over, k)) out[k] = v } + out.putAll(over) + return out + } + + fun without(headers: Map, name: String): Map = + headers.filterKeys { !it.equals(name, ignoreCase = true) } +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/QueueSettings.kt b/android/src/main/java/ai/openspace/backgroundupload/QueueSettings.kt new file mode 100644 index 00000000..1adda7fe --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/QueueSettings.kt @@ -0,0 +1,108 @@ +package ai.openspace.backgroundupload + +import android.content.Context +import com.google.gson.Gson +import java.io.File + +/** The configure() retry defaults. A request's `retry` overrides them field by field. */ +data class RetryDefaults( + val baseMs: Long = 1_000, + val maxMs: Long = 7_200_000, + val jitter: Double = 0.2, + val exempt: List = listOf(404), +) + +/** Queue-wide settings. They live next to the entries, in `settings.json`. */ +data class QueueSettings( + /** The queue's Wi-Fi setting. An entry whose descriptor sets wifiOnly ignores it. */ + val wifiOnly: Boolean = false, + /** The whole-queue pause gate. */ + val paused: Boolean = false, + /** The keys paused by pause({ keys }). An entry is paused under the gate or its key. */ + val pausedKeys: Set = emptySet(), + /** +1 per updateHeaders(). A 401 from an attempt sent under an older value re-issues at once. */ + val headerGeneration: Int = 0, + val retry: RetryDefaults = RetryDefaults(), +) { + /** Whether an entry of [key] is paused: the gate is on, or its key is in the set. */ + fun isPaused(key: String) = paused || key in pausedKeys + + /** The Wi-Fi rule of one attempt: the entry's own value, else the queue's. */ + fun wifiOnlyFor(entry: QueueEntry) = entry.descriptor?.wifiOnly ?: wifiOnly +} + +/** + * Reads and writes [QueueSettings]. The value is cached after the first read. + * Workers read it before every attempt, so the cache matters. A corrupt file + * reads as the defaults. + */ +class QueueSettingsStore(private val file: File) { + + companion object { + private val gson = Gson() + + @Volatile + private var instance: QueueSettingsStore? = null + + fun get(context: Context): QueueSettingsStore = + instance ?: synchronized(this) { + instance ?: QueueSettingsStore(File(QueueStore.rootDir(context), "settings.json")) + .also { instance = it } + } + + /** configure().retry → defaults. Absent fields keep the library defaults. */ + fun retryDefaults(retry: Map?): RetryDefaults { + val d = RetryDefaults() + if (retry == null) return d + val backoff = retry["backoff"] as? Map<*, *> + val terminal = retry["terminalHttp"] as? Map<*, *> + return RetryDefaults( + baseMs = (backoff?.get("baseMs") as? Number)?.toLong() ?: d.baseMs, + maxMs = (backoff?.get("maxMs") as? Number)?.toLong() ?: d.maxMs, + jitter = (backoff?.get("jitter") as? Number)?.toDouble() ?: d.jitter, + exempt = (terminal?.get("exempt") as? List<*>)?.mapNotNull { (it as? Number)?.toInt() } + ?: d.exempt, + ) + } + } + + private var cached: QueueSettings? = null + + @Synchronized + fun load(): QueueSettings { + cached?.let { return it } + val read = if (file.exists()) { + runCatching { gson.fromJson(file.readText(), QueueSettings::class.java) }.getOrNull() + } else null + return validated(read).also { cached = it } + } + + /** Throws IOException when the write fails. The cache then keeps the old value. */ + @Synchronized + fun update(transform: (QueueSettings) -> QueueSettings): QueueSettings { + val next = transform(load()) + AtomicFiles.writeText(file, gson.toJson(next)) + cached = next + return next + } + + // Gson does not run constructors, so absent fields read as null or 0. + @Suppress("SENSELESS_COMPARISON", "USELESS_ELVIS", "UNNECESSARY_SAFE_CALL") + private fun validated(s: QueueSettings?): QueueSettings { + if (s == null) return QueueSettings() + val d = RetryDefaults() + val r = s.retry + return QueueSettings( + wifiOnly = s.wifiOnly, + paused = s.paused, + pausedKeys = s.pausedKeys?.toSet() ?: emptySet(), + headerGeneration = s.headerGeneration, + retry = if (r == null) d else RetryDefaults( + baseMs = if (r.baseMs > 0) r.baseMs else d.baseMs, + maxMs = if (r.maxMs > 0) r.maxMs else d.maxMs, + jitter = r.jitter, + exempt = r.exempt ?: d.exempt, + ), + ) + } +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/QueueStore.kt b/android/src/main/java/ai/openspace/backgroundupload/QueueStore.kt new file mode 100644 index 00000000..3db76532 --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/QueueStore.kt @@ -0,0 +1,221 @@ +package ai.openspace.backgroundupload + +import android.content.Context +import com.google.gson.Gson +import java.io.File +import java.io.IOException +import java.util.Base64 + +/** + * The durable queue: one directory per entry id. This is the v9 chunked + * manifest store, generalized in place. The root directory keeps its v9 name + * (`rnbgupload-chunked`) so v9 blobs and manifests are found without a move. + * + * A v10 directory holds `entry.json` and at most one staged body file (see + * [StagedBody.fileName]). A v9 directory holds `manifest.json` and `blob` + * until a same-id enqueue adopts it. + * + * Every write is tmp + fsync + rename ([AtomicFiles]). Bodies are staged + * before `entry.json` is saved, so an `entry.json` on disk means its body is + * durable. The store keeps [RequestIndex] current on every save and remove. + * It is reachable from a bare Context, because a worker can run in a process + * where React never started. + */ +class QueueStore(private val dir: File, private val index: RequestIndex = RequestIndex()) { + + companion object { + const val ENTRY_FILE = "entry.json" + const val V9_MANIFEST_FILE = "manifest.json" + const val BLOB_FILE = "blob" + + private val gson = Gson() + + @Volatile + private var instance: QueueStore? = null + + fun rootDir(context: Context) = File(context.filesDir, "rnbgupload-chunked") + + /** The process-wide store. The first call loads every row into [RequestIndex.shared]. */ + fun get(context: Context): QueueStore = + instance ?: synchronized(this) { + instance ?: QueueStore(rootDir(context), RequestIndex.shared) + .also { it.loadIndex() } + .also { instance = it } + } + } + + init { + dir.mkdirs() + } + + /** Rebuilds the index from disk. */ + @Synchronized + fun loadIndex() { + index.replaceAll(all().map { it.toRow() }) + } + + // Ids come from the caller and can hold path separators, so the directory + // name is an encoding of the id. The id is read back from the file. + fun entryDir(id: String) = + File(dir, Base64.getUrlEncoder().withoutPadding().encodeToString(id.toByteArray())) + + fun blobFile(id: String) = File(entryDir(id), BLOB_FILE) + + /** The staged body file of [entry], or null for a bodiless request. */ + fun bodyFile(entry: QueueEntry): File? = + entry.body?.fileName?.let { File(entryDir(entry.id), it) } + + private fun entryFile(id: String) = File(entryDir(id), ENTRY_FILE) + + /** Runs [block] under the store lock. For work that spans several store calls. */ + fun locked(block: () -> T): T = synchronized(this) { block() } + + @Synchronized + fun load(id: String): QueueEntry? = read(entryFile(id)) + + /** Throws IOException when the entry did not persist. */ + @Synchronized + fun save(entry: QueueEntry) { + AtomicFiles.writeText(entryFile(entry.id), gson.toJson(entry)) + index.put(entry.toRow()) + } + + /** + * An atomic read-modify-write. The lock spans load, [transform], and save, + * so nothing can write between them and be erased. A result that is the + * same object as the input writes nothing. A null result writes nothing: + * forgetting an entry is always an explicit [remove]. A throwing + * transform or a failed write propagates. + */ + @Synchronized + fun compute(id: String, transform: (QueueEntry?) -> QueueEntry?): QueueEntry? { + val current = load(id) + val next = transform(current) + if (next != null && next !== current) save(next) + return next + } + + /** Best effort, for a worker that can go on from memory: null when the entry is gone or the write failed. */ + @Synchronized + fun update(id: String, transform: (QueueEntry) -> QueueEntry): QueueEntry? = + runCatching { + val current = load(id) ?: return null + val next = transform(current) + if (next !== current) save(next) + next + }.getOrNull() + + /** + * Forgets the entry: row and bytes. `entry.json` goes first, so a partial + * delete never leaves a row that points at missing bytes. + */ + @Synchronized + fun remove(id: String) { + entryFile(id).delete() + entryDir(id).deleteRecursively() + index.remove(id) + } + + /** Every v10 entry. A directory with only a v9 manifest is not a row. */ + @Synchronized + fun all(): List = + (dir.listFiles { f -> f.isDirectory } ?: emptyArray()) + .mapNotNull { d -> File(d, ENTRY_FILE).takeIf { it.exists() }?.let { read(it) } } + + /** Every eventId a row names. The journal prune spares them. */ + @Synchronized + fun referencedEventIds(): Set = all().mapNotNull { it.settledEventId }.toSet() + + /** + * The v9 chunked manifest in [id]'s directory. The caller asks only when + * there is no v10 entry or the entry is a legacy row: a v10 entry + * prunes the manifest when it adopts it. + */ + @Synchronized + fun legacyManifest(id: String): LegacyManifest? { + val file = File(entryDir(id), V9_MANIFEST_FILE) + if (!file.exists()) return null + val parsed = runCatching { gson.fromJson(file.readText(), LegacyManifest::class.java) }.getOrNull() + return LegacyManifest.validated(parsed) + } + + /** + * Deletes every file in the entry directory that [entry] does not use: + * an older body, a v9 manifest it adopted, tmp files from a crash. + */ + @Synchronized + fun pruneUnreferenced(entry: QueueEntry) { + val keep = setOfNotNull(ENTRY_FILE, entry.body?.fileName) + entryDir(entry.id).listFiles()?.forEach { f -> + if (f.name !in keep) f.deleteRecursively() + } + } + + private fun read(file: File): QueueEntry? { + if (!file.exists()) return null + val parsed = try { + gson.fromJson(file.readText(), QueueEntry::class.java) + } catch (error: Throwable) { + Diag.warn("queue entry unreadable, skipped: ${file.parentFile?.name}", error) + return null + } + return validated(parsed).also { + if (it == null) Diag.warn("queue entry incomplete, skipped: ${file.parentFile?.name}") + } + } + + // Gson does not run constructors. A corrupt file, or one from an older + // build, can hold null in a non-null field. Reject what the engine relies + // on; normalize what has a safe default. + @Suppress("SENSELESS_COMPARISON", "USELESS_ELVIS") + private fun validated(e: QueueEntry?): QueueEntry? { + if (e == null || e.id == null || e.key == null || e.state == null) return null + if (!e.legacy && (e.descriptor == null || e.body == null || e.body.kind == null)) return null + val d = e.descriptor?.let { d -> + if (d.parts != null && d.parts.any { it == null || it.url == null }) return null + d.copy( + method = d.method ?: "POST", + headers = d.headers ?: emptyMap(), + accept = d.accept ?: emptyList(), + parts = d.parts?.map { if (it.headers == null) it.copy(headers = emptyMap()) else it }, + ) + } + return e.copy( + varsJson = e.varsJson ?: "null", + descriptor = d, + generation = if (e.generation <= 0) 1 else e.generation, + ) + } +} + +/** + * A v9 chunked manifest, only the fields v10 reads. The field names are the + * v9 ones; Gson skips the rest of the file. + */ +data class LegacyManifest( + val id: String, + val parts: List, + val accept: List, +) { + companion object { + @Suppress("SENSELESS_COMPARISON") + fun validated(m: LegacyManifest?): LegacyManifest? { + if (m == null || m.id == null || m.parts == null || m.parts.isEmpty()) return null + if (m.parts.any { it == null || it.url == null }) return null + return m.copy( + parts = m.parts.map { if (it.headers == null) it.copy(headers = emptyMap()) else it }, + accept = m.accept ?: emptyList(), + ) + } + } +} + +/** A rejection that reaches JS as `promise.reject(code, message)`. */ +class QueueException(val code: String, message: String) : IOException(message) { + companion object { + const val E_RUNNING = "E_RUNNING" + const val E_FILE_MISSING = "E_FILE_MISSING" + const val E_STORAGE = "E_STORAGE" + const val E_INVALID = "E_INVALID" + } +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/RequestIndex.kt b/android/src/main/java/ai/openspace/backgroundupload/RequestIndex.kt new file mode 100644 index 00000000..be3b7b0d --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/RequestIndex.kt @@ -0,0 +1,47 @@ +package ai.openspace.backgroundupload + +import java.util.concurrent.ConcurrentHashMap + +/** + * The in-memory rows behind the synchronous getRequests(). [QueueStore] + * keeps it current: it calls [put] after every save and [remove] after every + * remove. Progress ticks move [RequestRow.bytesSent] here only. + */ +class RequestIndex { + companion object { + val shared = RequestIndex() + } + + private val rows = ConcurrentHashMap() + + fun replaceAll(all: Collection) { + rows.clear() + all.forEach { rows[it.id] = it } + } + + /** + * A save of a running entry keeps the larger bytesSent. The stored value + * lags the in-memory progress, and a save for an attempt must not move the + * row backwards. + */ + fun put(row: RequestRow) { + rows.compute(row.id) { _, old -> + if (old != null && old.state == row.state && row.state == EntryState.RUNNING.wire && + old.bytesSent > row.bytesSent && old.bytesSent <= row.totalBytes + ) row.withBytes(old.bytesSent) else row + } + } + + fun remove(id: String) { + rows.remove(id) + } + + /** Oldest first, then by id. */ + fun snapshot(): List = + rows.values.sortedWith(compareBy { it.createdAt }.thenBy { it.id }) + + /** A progress tick. A missing id is ignored. */ + fun setBytes(id: String, bytesSent: Long) { + rows.computeIfPresent(id) { _, row -> row.withBytes(bytesSent) } + } +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/RetryClassifier.kt b/android/src/main/java/ai/openspace/backgroundupload/RetryClassifier.kt new file mode 100644 index 00000000..d570ec61 --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/RetryClassifier.kt @@ -0,0 +1,85 @@ +package ai.openspace.backgroundupload + +import java.io.IOException +import kotlin.math.min +import kotlin.random.Random + +/** + * The retry table (plan section 6.1), as pure functions. + * + * | Response or failure | Verdict | + * | 2xx, or an accept rule matches | Accepted | + * | 401, 403 | Auth (park) | + * | 408, 429, 5xx | Transient | + * | other 4xx in `exempt` (default [404]) | Transient | + * | other 4xx | Terminal http | + * | any other status (1xx, a final 3xx) | Terminal http | + * | IOException, payload file missing | Terminal file | + * | IOException | Transient | + * | anything else | Terminal unknown | + */ +object RetryClassifier { + + sealed class Verdict { + object Accepted : Verdict() + object Transient : Verdict() + object Auth : Verdict() + data class Terminal(val errorKind: String, val message: String) : Verdict() + } + + data class Policy(val baseMs: Long, val maxMs: Long, val jitter: Double, val exempt: List) + + /** A wait up to this long happens inside the worker. A longer one releases the worker. */ + const val IN_WORKER_BACKOFF_MAX_MS = 30_000L + + fun policy(defaults: RetryDefaults, override: RetryOverride?): Policy = Policy( + baseMs = override?.baseMs ?: defaults.baseMs, + maxMs = override?.maxMs ?: defaults.maxMs, + jitter = override?.jitter ?: defaults.jitter, + exempt = override?.exempt ?: defaults.exempt, + ) + + fun classifyResponse( + code: Int, + body: String?, + accept: List, + exempt: List, + ): Verdict = when { + UploadOutcome.isAccepted(code, body, accept) -> Verdict.Accepted + code == 401 || code == 403 -> Verdict.Auth + code == 408 || code == 429 || code in 500..599 -> Verdict.Transient + code in 400..499 && code in exempt -> Verdict.Transient + else -> Verdict.Terminal("http", "HTTP $code") + } + + /** A CancellationException is never classified; the caller rethrows it first. */ + fun classifyFailure(error: Throwable, fileExists: Boolean): Verdict = when { + error is IOException && !fileExists -> + Verdict.Terminal("file", "request body file is missing: ${error.message ?: error.javaClass.simpleName}") + error is IOException -> Verdict.Transient + else -> Verdict.Terminal("unknown", error.message ?: error.javaClass.simpleName) + } + + /** + * The live attempt's errorKind for a transport failure, from its + * [classifyFailure] verdict: file, unknown, or network for a transient one. + */ + fun failureKind(verdict: Verdict): String = (verdict as? Verdict.Terminal)?.errorKind ?: "network" + + fun isExpired(now: Long, expiresAt: Long) = now >= expiresAt + + /** + * base * 2^(streak-1), capped at maxMs, then spread by ± jitter and capped + * again. streak 1 is baseMs. + */ + fun backoffMs(policy: Policy, streak: Int, random: Random = Random.Default): Long { + val exponent = (streak.coerceAtLeast(1) - 1).coerceAtMost(40) + val raw = min(policy.baseMs.toDouble() * Math.pow(2.0, exponent.toDouble()), policy.maxMs.toDouble()) + val jitter = policy.jitter.coerceIn(0.0, 1.0) + val spread = raw * (1.0 + jitter * (2.0 * random.nextDouble() - 1.0)) + return min(spread, policy.maxMs.toDouble()).toLong().coerceAtLeast(0L) + } + + /** The wake time, never later than expiresAt, so an entry expires on time. */ + fun nextAttemptAt(now: Long, backoffMs: Long, expiresAt: Long): Long = min(now + backoffMs, expiresAt) +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/Scheduler.kt b/android/src/main/java/ai/openspace/backgroundupload/Scheduler.kt new file mode 100644 index 00000000..872dd2a3 --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/Scheduler.kt @@ -0,0 +1,111 @@ +package ai.openspace.backgroundupload + +import android.content.Context +import androidx.work.ExistingWorkPolicy +import androidx.work.OneTimeWorkRequest +import androidx.work.OneTimeWorkRequestBuilder +import androidx.work.WorkInfo +import androidx.work.WorkManager +import androidx.work.workDataOf +import java.util.concurrent.TimeUnit + +/** + * Starts and stops worker runs for entries. A run only carries the entry id; + * the worker reads everything else from the store. So an extra run is always + * harmless: it finds nothing to do and exits. + */ +interface WorkScheduler { + /** A run as soon as possible, on the entry's main chain. */ + fun schedule(entry: QueueEntry) + + /** + * A run at [at] under a second unique name. Used for long backoffs and for + * the expiry of a parked entry. [replace] false keeps a wake that already + * exists (the boot sweep). + */ + fun scheduleWake(entry: QueueEntry, at: Long, replace: Boolean) + + /** Cancels the main chain and the wake. */ + fun cancel(id: String) +} + +/** + * WorkManager unique work per entry id, APPEND_OR_REPLACE, as v9. + * + * Why APPEND_OR_REPLACE: a worker settles before doWork returns, so a same-id + * enqueue can arrive while the row is still RUNNING. KEEP would drop it and + * REPLACE would kill the running worker. APPEND runs it after; OR_REPLACE + * starts a fresh chain after a CANCELLED or FAILED one. Workers always return + * success, because WorkManager fails the dependents of a FAILED row without a + * run. + * + * Long waits do not go on the main chain. A delayed row there would hold + * back every later "run now" appended behind it. They use the wake name + * (`#wake`) with an initial delay instead. + * + * No WorkManager Constraints: connectivity and wifi-only are checked inside + * the worker, as v9, because the constraint path was unreliable. + */ +class WorkManagerScheduler(context: Context) : WorkScheduler { + companion object { + /** v9 rows carry the tag "RNFileUploader"; this one is new so the v9 cancel does not touch v10 work. */ + const val WORK_TAG = "RNFileUploader.v10" + const val ID_TAG_PREFIX = "RNFileUploaderId:" + /** A string literal, because WorkManager persists it across builds. */ + const val ENTRY_ID_KEY = "entryId" + const val V9_WORK_TAG = "RNFileUploader" + + fun wakeName(id: String) = "$id#wake" + + /** Milliseconds from [now] until [at]; never negative. */ + fun initialDelayMs(at: Long?, now: Long): Long = if (at == null) 0L else (at - now).coerceAtLeast(0L) + + /** An unfinished row that is not RUNNING: a queued run that did not start yet. */ + fun hasQueuedSuccessor(states: List): Boolean = + states.any { !it.isFinished && it != WorkInfo.State.RUNNING } + } + + private val workManager = WorkManager.getInstance(context) + + override fun schedule(entry: QueueEntry) { + // A queued successor already guarantees a run after the current one. + val states = workManager.getWorkInfosForUniqueWork(entry.id).get().map { it.state } + if (hasQueuedSuccessor(states)) return + workManager + .beginUniqueWork(entry.id, ExistingWorkPolicy.APPEND_OR_REPLACE, request(entry, 0L)) + .enqueue() + } + + override fun scheduleWake(entry: QueueEntry, at: Long, replace: Boolean) { + val delay = initialDelayMs(at, System.currentTimeMillis()) + workManager.enqueueUniqueWork( + wakeName(entry.id), + if (replace) ExistingWorkPolicy.REPLACE else ExistingWorkPolicy.KEEP, + request(entry, delay), + ) + } + + override fun cancel(id: String) { + workManager.cancelUniqueWork(id) + workManager.cancelUniqueWork(wakeName(id)) + } + + /** First v10 launch: the v9 rows. Their workers are gone. */ + fun cancelV9Work() { + workManager.cancelAllWorkByTag(V9_WORK_TAG) + } + + private fun request(entry: QueueEntry, delayMs: Long): OneTimeWorkRequest { + val builder = if (entry.body?.kind == StagedBody.CHUNKED) { + OneTimeWorkRequestBuilder() + } else { + OneTimeWorkRequestBuilder() + } + return builder + .addTag(WORK_TAG) + .addTag(ID_TAG_PREFIX + entry.id) + .setInputData(workDataOf(ENTRY_ID_KEY to entry.id)) + .setInitialDelay(delayMs, TimeUnit.MILLISECONDS) + .build() + } +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/TransferHost.kt b/android/src/main/java/ai/openspace/backgroundupload/TransferHost.kt new file mode 100644 index 00000000..d7556a3d --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/TransferHost.kt @@ -0,0 +1,34 @@ +package ai.openspace.backgroundupload + +/** + * What a run needs from the platform: time, the network, progress, the + * notification, and live attempt events. [EntryWorker] is the real one. The + * JVM tests pass a fake with a scripted [send] and a manual clock, so every + * branch of [EntryRun] runs with no device. + */ +internal interface TransferHost { + fun now(): Long + + suspend fun sleep(ms: Long) + + /** One request through the library-wide cap of 4. Throws on a transport failure. */ + suspend fun send(request: TransferRequest, onProgress: (Long) -> Unit): UploadResponse + + /** The network state for the queue's wifi-only setting. The notification shows it. */ + fun connectivity(wifiOnly: Boolean): Connectivity + + /** Foreground mode for an entry that shows the notification. Never throws. */ + suspend fun foreground(entry: QueueEntry) + + fun progressStarted(id: String, total: Long, sent: Long) + + fun progress(id: String, sent: Long, total: Long) + + /** The trailing progress event, then the progress state is dropped. */ + fun progressEnded(id: String, completed: Boolean) + + fun attempt(event: AttemptEvent) + + /** Whether the system stopped this run at its time limit (JobScheduler, about 10 minutes). */ + fun stoppedByTimeout(): Boolean +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/UploadNotification.kt b/android/src/main/java/ai/openspace/backgroundupload/UploadNotification.kt new file mode 100644 index 00000000..f1b6b90b --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/UploadNotification.kt @@ -0,0 +1,135 @@ +package ai.openspace.backgroundupload + +import android.app.Notification +import android.app.NotificationChannel +import android.app.NotificationManager +import android.app.PendingIntent +import android.content.Context +import android.content.Intent +import android.content.pm.ServiceInfo +import android.net.ConnectivityManager +import android.net.NetworkCapabilities.NET_CAPABILITY_VALIDATED +import android.net.NetworkCapabilities.TRANSPORT_WIFI +import android.os.Build +import android.widget.RemoteViews +import androidx.core.app.NotificationCompat +import androidx.work.ForegroundInfo + +// This file builds the progress notification and probes connectivity. The +// simple and chunked workers share it. All uploads share one notification, +// identified by the NotificationConfig that configure() saved. Its progress +// bar is the total across the uploads. + +internal enum class Connectivity { NoWifi, NoInternet, Ok } + +// This is synchronized to ensure consistent status across workers +@Synchronized +internal fun validateConnectivity(context: Context, wifiOnly: Boolean): Connectivity { + val manager = context.getSystemService(Context.CONNECTIVITY_SERVICE) as ConnectivityManager + val network = manager.activeNetwork + val capabilities = manager.getNetworkCapabilities(network) + + val hasInternet = capabilities?.hasCapability(NET_CAPABILITY_VALIDATED) == true + + // not wifiOnly, return early + if (!wifiOnly) return if (hasInternet) Connectivity.Ok else Connectivity.NoInternet + + // handle wifiOnly + return if (hasInternet && capabilities?.hasTransport(TRANSPORT_WIFI) == true) + Connectivity.Ok + else + Connectivity.NoWifi // don't return NoInternet here, more direct to request to join wifi +} + +// Makes sure that the channel for the foreground notification exists. It makes +// the channel only when the channel is absent. Thus a channel that the consumer +// registered, with their own name and importance, always wins. When configure() +// never set a channel, we use a default LOW-importance channel, and no notifee +// setup is necessary. +internal fun ensureNotificationChannel(manager: NotificationManager, config: NotificationConfig) { + // minSdk is 29, so NotificationChannel (API 26) is always available. + if (manager.getNotificationChannel(config.notificationChannel) != null) return + val channel = NotificationChannel( + config.notificationChannel, + "Uploads", + NotificationManager.IMPORTANCE_LOW, + ) + manager.createNotificationChannel(channel) +} + +// builds the notification required to enable Foreground mode +internal fun buildUploadNotification( + context: Context, + config: NotificationConfig, + connectivity: Connectivity, +): Notification { + val channel = config.notificationChannel + val progress = UploadProgress.total() + val progress2Decimals = "%.2f".format(progress) + val title = when (connectivity) { + Connectivity.NoWifi -> config.notificationTitleNoWifi + Connectivity.NoInternet -> config.notificationTitleNoInternet + Connectivity.Ok -> config.notificationTitle + } + + // Custom layout for progress notification. + // The default hides the % text. This one shows it on the right, + // like most examples in various docs. + val content = RemoteViews(context.packageName, R.layout.notification) + content.setTextViewText(R.id.notification_title, title) + content.setTextViewText(R.id.notification_progress, "${progress2Decimals}%") + content.setProgressBar(R.id.notification_progress_bar, 100, progress.toInt(), false) + + return NotificationCompat.Builder(context, channel).run { + // Starting Android 12, the notification shows up with a confusing delay of 10s. + // This fixes that delay. + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) + foregroundServiceBehavior = Notification.FOREGROUND_SERVICE_IMMEDIATE + + // Required by android. Here we use the system's default upload icon + setSmallIcon(android.R.drawable.stat_sys_upload) + // These prevent the notification from being force-dismissed or dismissed when pressed + setOngoing(true) + setAutoCancel(false) + // These help show the same custom content when the notification collapses and expands + setCustomContentView(content) + setCustomBigContentView(content) + // opens the app when the notification is pressed + setContentIntent(openAppIntent(context)) + build() + } +} + +internal fun uploadForegroundInfo(config: NotificationConfig, notification: Notification): ForegroundInfo { + val id = config.systemNotificationId + // Starting Android 14, FOREGROUND_SERVICE_TYPE_DATA_SYNC is mandatory, otherwise app will crash + return if (Build.VERSION.SDK_INT > Build.VERSION_CODES.TIRAMISU) + ForegroundInfo(id, notification, ServiceInfo.FOREGROUND_SERVICE_TYPE_DATA_SYNC) + else + ForegroundInfo(id, notification) +} + +/** + * Whether [error] means that the system refused to let this worker enter + * foreground mode, because the app is in the background. Android 12 (API 31) + * restricts foreground-service starts from the background, and WorkManager + * relaunches workers exactly there (a reboot, or a quota resume). The Android + * 15 dataSync time-limit denial surfaces as the same exception. The transfer + * itself needs no foreground mode. It only loses process-priority protection. + * Thus callers continue without it. They do not fail an upload that can run. + * The function walks the causes, because setForeground can wrap the platform + * exception. + */ +internal fun isForegroundStartDenied(error: Throwable): Boolean { + if (Build.VERSION.SDK_INT < 31) return false + // ServiceStartNotAllowedException (API 31) covers the two variants: + // Foreground- and BackgroundServiceStartNotAllowedException. + return generateSequence(error) { it.cause } + .any { it is android.app.ServiceStartNotAllowedException } +} + +private fun openAppIntent(context: Context): PendingIntent? { + val intent = Intent(context, NotificationReceiver::class.java) + val flags = PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE + return PendingIntent.getBroadcast(context, "RNFileUpload-notification".hashCode(), intent, flags) +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/UploadOutcome.kt b/android/src/main/java/ai/openspace/backgroundupload/UploadOutcome.kt new file mode 100644 index 00000000..f1edc40a --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/UploadOutcome.kt @@ -0,0 +1,27 @@ +package ai.openspace.backgroundupload + +// Pure classification of terminal upload outcomes. Kept free of Android/React +// types so it can be unit-tested on a plain JVM — this is the highest-consequence +// logic in the uploader (it decides success vs failure), so it's covered directly. +object UploadOutcome { + + /** + * A non-2xx response to treat as success. `bodyIncludes` narrows the rule by + * a response-body substring. This is necessary when one status has several + * meanings, and only the message shows the difference (our backend's 409). + * Gson persists it inside [Descriptor]; see consumer-rules.pro. + */ + data class AcceptRule( + val status: Int, + val bodyIncludes: String? = null, + ) + + // Whether an HTTP response counts as a successful completion. A 2xx always + // counts, and a matching per-request accept rule counts. Every other response, + // 4xx and 5xx included, is a terminal http error, not a completion. + fun isAccepted(code: Int, body: String?, accept: List): Boolean = + code in 200..299 || accept.any { rule -> + rule.status == code && + (rule.bodyIncludes == null || body?.contains(rule.bodyIncludes) == true) + } +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/UploadProgress.kt b/android/src/main/java/ai/openspace/backgroundupload/UploadProgress.kt new file mode 100644 index 00000000..2363dbc1 --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/UploadProgress.kt @@ -0,0 +1,58 @@ +package ai.openspace.backgroundupload + +import android.os.Handler +import android.os.Looper + +// Stores and aggregates total progress from all workers +object UploadProgress { + private data class Progress( + var bytesUploaded: Long, + val size: Long, + var complete: Boolean = false + ) { + fun complete() { + bytesUploaded = size + complete = true + } + } + + private val map = mutableMapOf() + + @Synchronized + fun add(id: String, size: Long) { + map[id] = Progress(bytesUploaded = 0L, size = size) + } + + @Synchronized + fun set(uploadId: String, bytesUploaded: Long) { + map[uploadId]?.bytesUploaded = bytesUploaded + } + + @Synchronized + fun complete(uploadId: String) { + map[uploadId]?.complete() + + // Attempt to clear in 2 seconds. This is the simplest way to let the + // last worker reset the overall progress. + // Clearing progress ensures the notification starts at 0% next time. + Handler(Looper.getMainLooper()).postDelayed({ clearIfCompleted() }, 2000) + } + + @Synchronized + fun remove(uploadId: String) { + map.remove(uploadId) + } + + @Synchronized + fun total(): Double { + val totalBytesUploaded = map.values.sumOf { it.bytesUploaded } + val totalFileSize = map.values.sumOf { it.size } + if (totalFileSize == 0L) return 0.0 + return (totalBytesUploaded.toDouble() * 100 / totalFileSize) + } + + @Synchronized + private fun clearIfCompleted() { + if (map.values.all { it.complete }) map.clear() + } +} \ No newline at end of file diff --git a/android/src/main/java/ai/openspace/backgroundupload/UploadTransport.kt b/android/src/main/java/ai/openspace/backgroundupload/UploadTransport.kt new file mode 100644 index 00000000..cc8093ea --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/UploadTransport.kt @@ -0,0 +1,29 @@ +package ai.openspace.backgroundupload + +import kotlinx.coroutines.sync.Semaphore +import okhttp3.OkHttpClient +import java.util.concurrent.TimeUnit + +// The HTTP client and the library-wide transmission gate. The simple and +// chunked workers share them. Thus "requests transmitting at one time" means +// one thing across all uploads. + +// Max total time for a single request to complete +// This is 24hrs so plenty of time for large uploads +// Worst case is the time maxes out and the upload gets restarted. +// Not using unlimited time to prevent unexpected behaviors. +private const val REQUEST_TIMEOUT = 24L +private val REQUEST_TIMEOUT_UNIT = TimeUnit.HOURS + +// The number of requests transmitting at one time across ALL uploads, chunked +// parts included. This is the design's library-wide cap of 4. Every request, +// a simple upload or a chunked part, must pass this semaphore. Thus on +// Android the cap is hard. A semaphore controls this, not OkHttp's connection +// limits, because those limits add a delay between requests. +internal const val MAX_TRANSFER_CONCURRENCY = 4 +internal val transferSemaphore = Semaphore(MAX_TRANSFER_CONCURRENCY) + +// Use Okhttp as it provides the most standard behaviors even though it's not coroutine friendly +internal val uploadHttpClient = OkHttpClient.Builder() + .callTimeout(REQUEST_TIMEOUT, REQUEST_TIMEOUT_UNIT) + .build() diff --git a/android/src/main/java/ai/openspace/backgroundupload/UploadUtils.kt b/android/src/main/java/ai/openspace/backgroundupload/UploadUtils.kt new file mode 100644 index 00000000..154cac08 --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/UploadUtils.kt @@ -0,0 +1,164 @@ +package ai.openspace.backgroundupload + +import kotlinx.coroutines.suspendCancellableCoroutine +import okhttp3.Call +import okhttp3.Callback +import okhttp3.Headers.Companion.toHeaders +import okhttp3.MediaType +import okhttp3.OkHttpClient +import okhttp3.Request +import okhttp3.RequestBody +import okhttp3.RequestBody.Companion.asRequestBody +import okhttp3.RequestBody.Companion.toRequestBody +import okhttp3.Response +import okio.Buffer +import okio.BufferedSink +import okio.ForwardingSink +import okio.buffer +import java.io.File +import java.io.IOException +import java.io.RandomAccessFile +import kotlin.coroutines.resumeWithException + +// Throttling interval of the raw progress callback. ProgressThrottle limits +// the JS events above this. +private const val PROGRESS_INTERVAL = 500 // milliseconds + +private const val RANGE_COPY_BUFFER = 64 * 1024 + +/** [truncated] when the body passed [BodyCap.SETTLED_MAX_BYTES] and the rest was not read. */ +data class UploadResponse( + val code: Int, + val body: String, + val headers: Map, + val truncated: Boolean = false, +) + +/** One request as the worker sends it. [body] is null only for GET and DELETE with no body. */ +data class TransferRequest( + val url: String, + val method: String, + val headers: Map, + val body: RequestBody?, +) + +/** Sends one request and reports bytes written. The headers are sent as they are. */ +suspend fun okhttpSend( + client: OkHttpClient, + request: TransferRequest, + onProgress: (Long) -> Unit, +): UploadResponse { + val body = request.body?.let { withProgressListener(it, throttled(onProgress)) } + val built = Request.Builder() + .url(request.url) + .headers(request.headers.toHeaders()) + .method(request.method, body) + .build() + return awaitResponse(client, built) +} + +// Every body has a null content type, so OkHttp does not invent a +// Content-Type. The header on the request (the caller's, or the one staging +// set) is sent unchanged. + +/** A whole staged file. */ +fun fileBody(file: File): RequestBody = file.asRequestBody(null as MediaType?) + +/** A zero-length body for a POST, PUT, or PATCH with no body. OkHttp requires one. */ +fun emptyBody(): RequestBody = ByteArray(0).toRequestBody(null) + +/** + * The file bytes [start, end) as a request body: a chunked part. It streams + * from disk with no temporary chunk file. A RandomAccessFile is opened fresh + * on every writeTo, because OkHttp can replay a body (a connection-level + * retry), and a one-shot stream would then send truncated data. + */ +fun rangeRequestBody(file: File, start: Long, end: Long): RequestBody = object : RequestBody() { + override fun contentType(): MediaType? = null + + override fun contentLength() = end - start + + override fun writeTo(sink: BufferedSink) { + RandomAccessFile(file, "r").use { raf -> + raf.seek(start) + val buffer = ByteArray(RANGE_COPY_BUFFER) + var remaining = end - start + while (remaining > 0L) { + val read = raf.read(buffer, 0, minOf(remaining, buffer.size.toLong()).toInt()) + if (read < 0) throw IOException( + "source file ended before part range [$start, $end): ${file.path}", + ) + sink.write(buffer, 0, read) + remaining -= read + } + } + } +} + +private suspend fun awaitResponse(client: OkHttpClient, request: Request): UploadResponse = + suspendCancellableCoroutine { continuation -> + val call = client.newCall(request) + continuation.invokeOnCancellation { call.cancel() } + call.enqueue(object : Callback { + override fun onFailure(call: Call, e: IOException) = + continuation.resumeWithException(e) + + override fun onResponse(call: Call, response: Response) { + val result = try { + response.use { res -> // close the response asap + // The body unchanged: an empty body stays empty. A substituted + // reason phrase would make accept `bodyIncludes` rules match text + // the server never sent. The cap applies while it streams in. + val body = res.body?.let { + BodyCap.read(it.source(), BodyCap.SETTLED_MAX_BYTES, it.contentType()?.charset() ?: Charsets.UTF_8) + } + UploadResponse( + res.code, + body?.text.orEmpty(), + res.headers.toMultimap().mapValues { it.value.joinToString(", ") }, + body?.truncated ?: false, + ) + } + } catch (e: IOException) { + continuation.resumeWithException(e) + return + } + continuation.resumeWith(Result.success(result)) + } + }) + } + +private fun throttled(onProgress: (Long) -> Unit): (Long) -> Unit { + var lastProgressReport = 0L + return { progress -> + val now = System.currentTimeMillis() + if (now - lastProgressReport >= PROGRESS_INTERVAL) { + lastProgressReport = now + onProgress(progress) + } + } +} + +// OkHttp has no built-in progress report, so the body counts bytes as it writes. +private fun withProgressListener( + body: RequestBody, + onProgress: (Long) -> Unit +) = object : RequestBody() { + override fun contentType() = body.contentType() + override fun contentLength() = body.contentLength() + override fun writeTo(sink: BufferedSink) { + val countingSink = object : ForwardingSink(sink) { + var bytesWritten = 0L + + override fun write(source: Buffer, byteCount: Long) { + super.write(source, byteCount) + bytesWritten += byteCount + onProgress(bytesWritten) + } + } + + val bufferedSink = countingSink.buffer() + body.writeTo(bufferedSink) + bufferedSink.flush() + } +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/UploadWorker.kt b/android/src/main/java/ai/openspace/backgroundupload/UploadWorker.kt new file mode 100644 index 00000000..a49f7548 --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/UploadWorker.kt @@ -0,0 +1,64 @@ +package ai.openspace.backgroundupload + +import android.content.Context +import androidx.work.WorkerParameters +import okhttp3.RequestBody +import java.io.File + +/** The WorkManager class for a single-body entry. The run is [EntryWorker]'s. */ +class UploadWorker(context: Context, params: WorkerParameters) : EntryWorker(context, params) + +/** + * One request with one body: none, JSON, multipart, or a copied file. One + * attempt at a time ([EntryRun.attempt]), until a verdict ends it: + * accepted → Completed; auth → re-issue (newer headers) or park; + * transient → back off (short: here; long: release); terminal → Failed. + */ +internal class SimpleTransfer(private val run: EntryRun) { + + suspend fun run(start: QueueEntry): Settlement { + val d0 = start.descriptor!! + val file = run.store.bodyFile(start) + // The payload probe: a staged body that is gone can never be sent. + if (file != null && !file.exists()) { + return Settlement.Failed("file", "the staged request body is missing", null, null, d0.reportUrl, d0.method) + } + val total = start.body?.totalBytes ?: 0L + run.progressStarted(total, 0L) + var streak = start.backoffStreak + + while (true) { + val latest = run.ops.latest(run.entryId, run.generation) + if (RetryClassifier.isExpired(run.host.now(), latest.expiresAt)) throw EntryRun.ExpiredException() + run.waitForNetwork() + + val a = run.attempt( + partIndex = null, + body = { d, _ -> requestBody(file, d.method) }, + onProgress = { sent -> run.reportProgress(sent, total) }, + fileExists = { file == null || file.exists() }, + ) + when (val r = a.result) { + is EntryRun.AttemptResult.Accepted -> return Settlement.Completed(r.response, a.url, a.method) + is EntryRun.AttemptResult.Auth -> { + if (!r.reissue) throw EntryRun.ParkException(r.headerGeneration) + streak = 0 // updateHeaders() landed while this attempt was in flight: re-issue now. + } + EntryRun.AttemptResult.Transient -> { + run.reportProgress(0L, total) + streak++ + run.backoffOrRelease(a.policy, streak, a.entry.expiresAt) + } + is EntryRun.AttemptResult.Terminal -> + return Settlement.Failed(r.errorKind, r.message, r.response, null, a.url, a.method, bytesSent = run.liveBytes) + } + } + } + + // OkHttp needs a body for POST, PUT, and PATCH, and forbids one for GET. + private fun requestBody(file: File?, method: String): RequestBody? = when { + file != null -> fileBody(file) + method == "GET" || method == "DELETE" -> null + else -> emptyBody() + } +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/UploaderModule.kt b/android/src/main/java/ai/openspace/backgroundupload/UploaderModule.kt new file mode 100644 index 00000000..1b6e61a5 --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/UploaderModule.kt @@ -0,0 +1,233 @@ +package ai.openspace.backgroundupload + +import com.facebook.react.bridge.Arguments +import com.facebook.react.bridge.Promise +import com.facebook.react.bridge.ReactApplicationContext +import com.facebook.react.bridge.ReadableArray +import com.facebook.react.bridge.ReadableMap +import com.facebook.react.bridge.WritableArray +import com.facebook.react.bridge.WritableMap +import com.facebook.react.common.LifecycleState +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.Future +import java.util.concurrent.TimeUnit + +/** + * TurboModule (New Architecture). [NativeRNFileUploaderSpec] is generated by + * codegen from `src/NativeRNFileUploader.ts` into this same package (see + * `codegenConfig.android.javaPackageName` in package.json). + * + * A thin shell over [QueueController]. Every promise method runs on one + * single-thread executor ([queueExecutor]), because enqueue copies files and + * every method touches the disk. The promise resolves from that executor. + * getRequests() is synchronous and reads the in-memory index only. + */ +class UploaderModule(context: ReactApplicationContext) : + NativeRNFileUploaderSpec(context) { + + companion object { + const val NAME = "RNFileUploader" + const val TAG = "RNFileUploader.UploaderModule" + + /** The longest getRequests() waits for the first-launch import. */ + private const val IMPORT_WAIT_MS = 2_000L + + // The live module, so EventReporter can reach the codegen emitters: they + // are protected on the generated spec. Null whenever JS is absent + // (headless worker, mid-reload); outcomes are journaled before they are + // emitted, so a dropped live event is never lost. + @Volatile + var instance: UploaderModule? = null + private set + + /** One thread for every module-side disk operation. It outlives a JS reload. */ + val queueExecutor: ExecutorService = Executors.newSingleThreadExecutor { r -> + Thread(r, "RNFileUploader.queue") + } + } + + private val store = QueueStore.get(context) + private val journal = EventJournal.get(context) + private val settings = QueueSettingsStore.get(context) + private val scheduler = WorkManagerScheduler(context) + private val controller = QueueController(store, journal, settings, EventReporter, scheduler) + + // The v9 import, then the v9 work cancel, then the boot sweep. + // getRequests() waits for the import only (file reads and row saves), so + // the first call after an upgrade already shows the legacy rows. The + // cancel opens the WorkManager database, so it runs after that wait. + private val importDone: Future + + init { + instance = this + importDone = queueExecutor.submit { + runCatching { LegacyImport.runOnce(context, store) } + .onFailure { Diag.error("v9 import failed", it) } + .getOrDefault(true) // a failed import still cancels the v9 work + } + queueExecutor.execute { + if (runCatching { importDone.get() }.getOrDefault(false)) { + runCatching { scheduler.cancelV9Work() }.onFailure { Diag.error("v9 work cancel failed", it) } + } + runCatching { controller.sweep() }.onFailure { Diag.error("boot sweep failed", it) } + } + } + + // Set by invalidate(). A drain still queued on the executor then does not + // make this dead module the journal's listener. + @Volatile + private var invalidated = false + + override fun invalidate() { + // A reload constructs the replacement before tearing this one down, so + // only clear the pointer (and the listener) when it still refers to us. + // The flag goes first: the drain reads it under the journal lock. + invalidated = true + journal.stopListening(this) + if (instance === this) instance = null + super.invalidate() + } + + override fun getName(): String = NAME + + /** Picks the progress throttle interval: 1 s in the foreground, 10 min otherwise. */ + fun isForeground(): Boolean = reactApplicationContext.lifecycleState == LifecycleState.RESUMED + + // MARK: - Event emission (called by EventReporter) + + fun emitState(params: WritableMap) = safeEmit { emitOnState(params) } + + fun emitProgress(params: WritableMap) = safeEmit { emitOnProgress(params) } + + fun emitAttempt(params: WritableMap) = safeEmit { emitOnAttempt(params) } + + fun emitSettled(params: WritableMap) = safeEmit { emitOnSettled(params) } + + fun emitNotification(params: WritableMap) = safeEmit { emitOnNotification(params) } + + private inline fun safeEmit(emit: () -> Unit) { + try { + emit() + } catch (exc: NullPointerException) { + // The emitter callback exists only while the C++ TurboModule does, so a + // null callback is expected before setup and after teardown. It is also + // null for the whole process on the old architecture, so warn. + Diag.warn("live event dropped (no event emitter; New Architecture required)") + } catch (exc: Throwable) { + Diag.error("failed to emit live event", exc) + } + } + + // MARK: - Promise methods + + /** Runs [block] on the queue executor and settles [promise] with its value or a coded rejection. */ + private fun onQueue(promise: Promise, block: () -> Any?) { + queueExecutor.execute { + try { + promise.resolve(block()) + } catch (e: QueueException) { + promise.reject(e.code, e.message, e) + } catch (e: EntryParsing.InvalidEntryException) { + promise.reject(QueueException.E_INVALID, e.message, e) + } catch (e: Throwable) { + Diag.error("queue operation failed", e) + promise.reject(QueueException.E_STORAGE, e.message ?: e.javaClass.simpleName, e) + } + } + } + + /** + * Saves the notification configuration (read by headless workers) and the + * retry defaults. Each call replaces the full configuration. lifetimeMs is + * not stored: JS already applied it to expiresAt. + */ + override fun configure(options: ReadableMap) { + NotificationConfig.save(reactApplicationContext, NotificationConfig.fromReadableMap(options)) + @Suppress("UNCHECKED_CAST") + val retry = JsonBridge.valueOf(options, "retry") as? Map + val defaults = QueueSettingsStore.retryDefaults(retry) + queueExecutor.execute { + runCatching { controller.configureRetry(defaults) } + .onFailure { Diag.error("could not save the retry defaults", it) } + } + } + + override fun enqueue(entry: ReadableMap, promise: Promise) { + // Parse on the calling thread: the ReadableMap belongs to the bridge call. + val parsed = try { + EntryParsing.parse(entry) + } catch (e: EntryParsing.InvalidEntryException) { + promise.reject(QueueException.E_INVALID, e.message, e) + return + } + onQueue(promise) { controller.enqueue(parsed) } + } + + // scope is { keys?: string[] }; {} is the whole queue. Null (a stale JS + // bundle that sends no argument) is also the whole queue, not a crash. + // Parse on the calling thread, as enqueue does. + override fun pause(scope: ReadableMap?, promise: Promise) { + val keys = try { + EntryParsing.scopeKeys(scope) + } catch (e: EntryParsing.InvalidEntryException) { + promise.reject(QueueException.E_INVALID, e.message, e) + return + } + onQueue(promise) { controller.pause(keys); null } + } + + override fun resume(scope: ReadableMap?, promise: Promise) { + val keys = try { + EntryParsing.scopeKeys(scope) + } catch (e: EntryParsing.InvalidEntryException) { + promise.reject(QueueException.E_INVALID, e.message, e) + return + } + onQueue(promise) { controller.resume(keys); null } + } + + override fun cancel(id: String, promise: Promise) = onQueue(promise) { controller.cancel(id); null } + + override fun setWifiOnly(enabled: Boolean, promise: Promise) = + onQueue(promise) { controller.setWifiOnly(enabled); null } + + override fun updateHeaders(patch: ReadableMap, promise: Promise) { + val headers = try { + EntryParsing.headerPatch(patch) + } catch (e: EntryParsing.InvalidEntryException) { + promise.reject(QueueException.E_INVALID, e.message, e) + return + } + onQueue(promise) { controller.updateHeaders(headers); null } + } + + /** Synchronous. From the in-memory index, never from disk. */ + override fun getRequests(): WritableArray { + runCatching { importDone.get(IMPORT_WAIT_MS, TimeUnit.MILLISECONDS) } + val out = Arguments.createArray() + RequestIndex.shared.snapshot().forEach { out.pushMap(JsonBridge.toWritableMap(it.toMap())) } + return out + } + + /** + * Every unacknowledged outcome, each counted as one more delivery. JS + * subscribes to onSettled, then calls this at once, so this call makes + * the module the journal's listener. The flip and the scan share the + * journal lock: an outcome settled before it is in this drain (journaled + * at 0, returned at 1); one settled after is emitted live at 1. + */ + override fun getUnacknowledgedEvents(promise: Promise) { + onQueue(promise) { + val out = Arguments.createArray() + controller.unacknowledged(this) { !invalidated }.forEach { out.pushMap(it.toWritableMap()) } + out + } + } + + /** Resolves void. Idempotent; unknown ids are ignored. */ + override fun ackEvents(ids: ReadableArray, promise: Promise) { + val eventIds = (0 until ids.size()).mapNotNull { runCatching { ids.getString(it) }.getOrNull() } + onQueue(promise) { controller.ack(eventIds); null } + } +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/UploaderReactPackage.kt b/android/src/main/java/ai/openspace/backgroundupload/UploaderReactPackage.kt new file mode 100644 index 00000000..96193a35 --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/UploaderReactPackage.kt @@ -0,0 +1,29 @@ +package ai.openspace.backgroundupload + +import com.facebook.react.BaseReactPackage +import com.facebook.react.bridge.NativeModule +import com.facebook.react.bridge.ReactApplicationContext +import com.facebook.react.module.model.ReactModuleInfo +import com.facebook.react.module.model.ReactModuleInfoProvider + +class UploaderReactPackage : BaseReactPackage() { + + override fun getModule(name: String, reactContext: ReactApplicationContext): NativeModule? = + if (name == UploaderModule.NAME) UploaderModule(reactContext) else null + + override fun getReactModuleInfoProvider() = ReactModuleInfoProvider { + mapOf( + UploaderModule.NAME to ReactModuleInfo( + name = UploaderModule.NAME, + className = UploaderModule.NAME, + canOverrideExistingModule = false, + // Created on first JS access. Uploads outlive the module (WorkManager + // runs them, the journal records their outcomes), so nothing is lost by + // not constructing it at startup. + needsEagerInit = false, + isCxxModule = false, + isTurboModule = true, + ), + ) + } +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/WorkerGate.kt b/android/src/main/java/ai/openspace/backgroundupload/WorkerGate.kt new file mode 100644 index 00000000..d1313aeb --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/WorkerGate.kt @@ -0,0 +1,32 @@ +package ai.openspace.backgroundupload + +import java.util.concurrent.ConcurrentHashMap + +/** + * At most one worker EXECUTES per entry id, process-wide (renamed from the + * v9 ChunkedWorkerGate; both workers use it now). + * + * The unique-work chain almost guarantees this, but not across a cancel: + * cancelUniqueWork marks the row CANCELLED at once while the old worker's + * coroutine still winds down, and the wake-up work runs under a second + * unique name. Two concurrent requests for one chunked part are unsafe on + * the server. A starting worker acquires its id here and a second one waits. + * + * Same-process only. A worker in a dead process holds nothing. + */ +object WorkerGate { + private val holders = ConcurrentHashMap() + + /** True when [token] now holds the id, or already held it. False while another token holds it. */ + fun tryAcquire(id: String, token: Any): Boolean { + val current = holders.putIfAbsent(id, token) + return current == null || current === token + } + + /** Releases only when [token] is the holder, so a late release can not evict a successor. */ + fun release(id: String, token: Any) { + holders.remove(id, token) + } + + fun isRunning(id: String): Boolean = holders.containsKey(id) +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/WorkerOps.kt b/android/src/main/java/ai/openspace/backgroundupload/WorkerOps.kt new file mode 100644 index 00000000..0420fada --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/WorkerOps.kt @@ -0,0 +1,305 @@ +package ai.openspace.backgroundupload + +import java.io.IOException +import java.util.UUID + +/** How a run ended. */ +sealed class Settlement { + abstract val url: String + abstract val method: String + + /** [response] is null for a chunked completion. */ + data class Completed( + val response: UploadResponse?, + override val url: String, + override val method: String, + ) : Settlement() + + /** [bytesSent] is the live bytes of a simple entry's last attempt; null keeps the stored value. */ + data class Failed( + val errorKind: String, + val message: String, + val response: UploadResponse?, + val partIndex: Int?, + override val url: String, + override val method: String, + val bytesSent: Long? = null, + ) : Settlement() +} + +/** + * The entry was paused, cancelled, replaced, or forgotten under a running + * worker. The worker stops without a transition; the module owns what + * happened. Not a CancellationException: it must fail a chunked part's + * scope so the sibling parts stop too. + */ +class NotOwnedException(id: String) : Exception("entry '$id' is no longer owned by this run") + +/** + * Every transition the network causes: running, one attempt, part accepted, + * awaiting-auth, queued-with-backoff, a system stop, and the settle. Each is + * a `compute` guarded by the run's [generation], so a cancel, pause, or + * replace that landed first always wins. + */ +class WorkerOps( + private val store: QueueStore, + private val journal: EventJournal, + private val settings: QueueSettingsStore, + private val events: QueueEvents, + private val scheduler: WorkScheduler, + private val clock: () -> Long = System::currentTimeMillis, +) { + enum class ParkResult { PARKED, REISSUE, NOT_OWNED } + + /** + * Takes a queued entry. Null when there is nothing to run. + * + * A journal record of the entry's own generation means a settle was + * journaled and its store write was lost (a process death between the + * two, or a failed save). WorkManager can run the entry again before any + * boot sweep. Then begin applies the record, as the sweep does, and does + * not send the request again. + */ + fun begin(id: String): QueueEntry? { + val now = clock() + var row: QueueEntry? = null + var taken: QueueEntry? = null + store.locked { + val e = store.load(id) ?: return@locked + if (e.legacy || (e.state != EntryState.QUEUED && e.state != EntryState.RUNNING)) return@locked + val journaled = EntryTransitions.journaledSettle(e, journal.forEntry(id), now) + if (journaled != null) { + store.save(journaled.entry) + journal.ack(journaled.extraEventIds) + row = journaled.entry + return@locked + } + val next = EntryTransitions.toRunning(e, now) + store.save(next) + row = next + taken = next + } + row?.let { events.state(it.toRow()) } + return taken + } + + /** The stored entry, while this run still owns it. */ + fun latest(id: String, generation: Int): QueueEntry { + val e = store.load(id) + if (!EntryTransitions.isOwnedRun(e, generation)) throw NotOwnedException(id) + return e!! + } + + /** + * Write-ahead for one attempt: attempts + 1 and the X-Request-Id, persisted + * before the request is sent. Clears a short backoff's nextAttemptAt, and + * emits the row when it did. Returns the fresh entry, whose headers the + * attempt uses. + */ + fun recordAttempt(id: String, generation: Int, requestId: String): QueueEntry { + val now = clock() + var clearedBackoff = false + val next = store.compute(id) { e -> + if (EntryTransitions.isOwnedRun(e, generation)) { + clearedBackoff = e!!.nextAttemptAt != null + EntryTransitions.toAttempt(e, requestId, now) + } else e + } + if (next == null || next.lastRequestId != requestId || !EntryTransitions.isOwnedRun(next, generation)) { + throw NotOwnedException(id) + } + if (clearedBackoff) events.state(next.toRow()) + return next + } + + /** + * A short backoff the worker waits out in place: the row stays running + * and carries [nextAttemptAt]. Best effort; a lost write only hides the + * time. For a chunked entry, a sibling part's next attempt clears it. + */ + fun backingOff(id: String, generation: Int, nextAttemptAt: Long) { + val now = clock() + var applied = false + val next = store.update(id) { e -> + if (EntryTransitions.isOwnedRun(e, generation)) { + applied = true + EntryTransitions.toBackingOff(e, nextAttemptAt, now) + } else e + } + if (applied && next != null) events.state(next.toRow()) + } + + /** A chunked part the server accepted. Best effort, as v9: a lost flag only re-sends that part later. */ + fun markAccepted(id: String, generation: Int, index: Int): QueueEntry? = + store.update(id) { e -> + val parts = e.descriptor?.parts + if (e.generation != generation || parts == null || index !in parts.indices) e + else { + val next = ChunkedParts.withAccepted(parts, index) + e.copy( + descriptor = e.descriptor.copy(parts = next), + bytesSent = ChunkedParts.acceptedBytes(next), + backoffStreak = 0, + ) + } + } + + /** + * Journal, then transition, then emit, all under the store lock, so a + * cancel, pause, or replace lands either before (this run's outcome is + * dropped) or after. Returns whether this run's outcome stands. + * + * A failed journal write holds the record in memory ([EventJournal.appendOrHold]): + * the request already ran, and a retry would send it twice. A failed + * store write leaves the record for the ack, the next [begin], or the + * boot sweep to apply. + * + * A record of this generation already in the journal (a cancel whose + * entry save failed) wins: it is applied, as [begin] does, and this run's + * outcome is dropped. One life has one outcome. + */ + fun settle(id: String, generation: Int, s: Settlement): Boolean { + val now = clock() + val completed = s is Settlement.Completed + val failed = s as? Settlement.Failed + var delivered: EventJournal.SettledRecord? = null + var settled: QueueEntry? = null + store.locked { + val e = store.load(id) + if (!EntryTransitions.canSettle(e, generation, completed)) return@locked + e!! + val journaled = EntryTransitions.journaledSettle(e, journal.forEntry(id), now) + if (journaled != null) { + try { + store.save(journaled.entry) + journal.ack(journaled.extraEventIds) + settled = journaled.entry + } catch (error: IOException) { + Diag.error("settle could not apply the journaled outcome of '$id'; its ack or the boot sweep applies it", error) + } + return@locked + } + val state = if (completed) EntryState.COMPLETED else EntryState.ERROR + // A failed simple entry keeps the live bytes of its last attempt; a + // chunked one keeps its accepted bytes (the stored value). + val bytesSent = if (completed) e.totalBytes else failed?.bytesSent ?: e.bytesSent + val record = EventJournal.SettledRecord( + eventId = UUID.randomUUID().toString(), + id = e.id, + key = e.key, + varsJson = e.varsJson, + at = now, + attempts = e.attempts, + requestId = e.lastRequestId, + deliveries = 0, // the journal sets it + state = state.wire, + bytesSent = bytesSent, + totalBytes = e.totalBytes, + url = s.url, + method = s.method, + partIndex = failed?.partIndex, + kind = if (completed) EventJournal.KIND_COMPLETED else EventJournal.KIND_ERROR, + response = when (s) { + is Settlement.Completed -> s.response?.let { EventJournal.Response.of(it) } ?: EventJournal.Response.NONE + is Settlement.Failed -> s.response?.let { EventJournal.Response.of(it) } + }, + errorKind = failed?.errorKind, + message = failed?.message, + cancelReason = null, + generation = generation, + ) + // 1. The durable outcome. It never throws. + delivered = journal.appendOrHold(record) { store.referencedEventIds() + record.eventId } + // 2. The transition. + val next = EntryTransitions.toSettled(e, state, record.eventId, bytesSent, now) + try { + store.save(next) + settled = next + } catch (error: IOException) { + Diag.error("settle could not save '$id'; its ack, the next run, or the boot sweep applies the record", error) + } + } + val record = delivered + if (record == null) { + settled?.let { events.state(it.toRow()) } + return false + } + // 3 and 4. Best effort. + if (record.deliveries > 0) journal.listener()?.let { events.settled(record, it) } + settled?.let { events.state(it.toRow()) } + return true + } + + /** + * Whether the stored entry holds newer headers than the ones an attempt + * sent. [headerGeneration] is the entry's own value from [recordAttempt], + * so it always belongs to the headers that went out. The settings value + * is not used: updateHeaders() bumps it before it patches the entries. + */ + fun hasNewerHeaders(id: String, generation: Int, headerGeneration: Int): Boolean = + latest(id, generation).headerGeneration > headerGeneration + + /** + * A 401/403. [headerGeneration] is the entry's value from [recordAttempt]. + * When the entry got newer headers since, the attempt re-issues at once + * instead of parking. The check is inside the store lock, and + * updateHeaders() patches entries inside it too, so it either patched + * this entry first (REISSUE) or finds it parked and requeues it. + */ + fun park(id: String, generation: Int, headerGeneration: Int): ParkResult { + val now = clock() + var result = ParkResult.NOT_OWNED + val next = store.compute(id) { e -> + when { + !EntryTransitions.isOwnedRun(e, generation) -> e + e!!.headerGeneration > headerGeneration -> { + result = ParkResult.REISSUE + e + } + else -> { + result = ParkResult.PARKED + EntryTransitions.toParked(e, headerGeneration, now) + } + } + } + if (result == ParkResult.PARKED && next != null) { + events.state(next.toRow()) + // A parked entry still expires on time. + scheduler.scheduleWake(next, next.expiresAt, replace = true) + } + return result + } + + /** A backoff longer than a worker waits: back to queued, woken at [nextAttemptAt]. */ + fun release(id: String, generation: Int, nextAttemptAt: Long, streak: Int): Boolean { + val now = clock() + var applied = false + val next = store.compute(id) { e -> + if (EntryTransitions.isOwnedRun(e, generation)) { + applied = true + EntryTransitions.toReleased(e!!, nextAttemptAt, streak, now) + } else e + } + if (!applied || next == null) return false + events.state(next.toRow()) + scheduler.scheduleWake(next, nextAttemptAt, replace = true) + return true + } + + /** A system stop. A paused or cancelled entry is the module's, so only a running one moves. Never journals. */ + fun stopped(id: String, generation: Int) { + val now = clock() + var applied = false + val next = runCatching { + store.compute(id) { e -> + if (EntryTransitions.isOwnedRun(e, generation)) { + applied = true + EntryTransitions.toStopped(e!!, now) + } else e + } + }.getOrNull() + if (applied && next != null) events.state(next.toRow()) + } + + fun settings(): QueueSettings = settings.load() +} diff --git a/android/src/main/java/com/vydia/RNUploader/GlobalRequestObserverDelegate.kt b/android/src/main/java/com/vydia/RNUploader/GlobalRequestObserverDelegate.kt deleted file mode 100644 index cc16de19..00000000 --- a/android/src/main/java/com/vydia/RNUploader/GlobalRequestObserverDelegate.kt +++ /dev/null @@ -1,73 +0,0 @@ -package com.vydia.RNUploader - -import android.content.Context -import android.util.Log -import com.facebook.react.bridge.Arguments -import com.facebook.react.bridge.ReactApplicationContext -import com.facebook.react.bridge.WritableMap -import com.facebook.react.modules.core.DeviceEventManagerModule.RCTDeviceEventEmitter -import net.gotev.uploadservice.data.UploadInfo -import net.gotev.uploadservice.exceptions.UploadError -import net.gotev.uploadservice.exceptions.UserCancelledUploadException -import net.gotev.uploadservice.network.ServerResponse -import net.gotev.uploadservice.observer.request.RequestObserverDelegate - -class GlobalRequestObserverDelegate(reactContext: ReactApplicationContext) : RequestObserverDelegate { - private val TAG = "UploadReceiver" - - private var reactContext: ReactApplicationContext = reactContext - - override fun onCompleted(context: Context, uploadInfo: UploadInfo) { - } - - override fun onCompletedWhileNotObserving() { - } - - override fun onError(context: Context, uploadInfo: UploadInfo, exception: Throwable) { - var errorMessage = exception.message - val params = Arguments.createMap() - params.putString("id", uploadInfo.uploadId) - - when (exception) { - is UserCancelledUploadException -> { - errorMessage = "User cancelled upload" - } - is UploadError -> { - errorMessage = "responseCode=${exception.serverResponse.code} responseBody=${exception.serverResponse.bodyString}" - Log.e(TAG, "Error, upload error: ${exception.serverResponse.code} ${exception.serverResponse.bodyString}") - } - } - - params.putString("error", errorMessage) - sendEvent("error", params, context) - } - - override fun onProgress(context: Context, uploadInfo: UploadInfo) { - val params = Arguments.createMap() - params.putString("id", uploadInfo.uploadId) - params.putInt("progress", uploadInfo.progressPercent) //0-100 - - sendEvent("progress", params, context) - } - - override fun onSuccess(context: Context, uploadInfo: UploadInfo, serverResponse: ServerResponse) { - val headers = Arguments.createMap() - for ((key, value) in serverResponse.headers) { - headers.putString(key, value) - } - val params = Arguments.createMap() - params.putString("id", uploadInfo.uploadId) - params.putInt("responseCode", serverResponse.code) - params.putString("responseBody", serverResponse.bodyString) - params.putMap("responseHeaders", headers) - sendEvent("completed", params, context) - } - - /** - * Sends an event to the JS module. - */ - private fun sendEvent(eventName: String, params: WritableMap?, context: Context) { - reactContext?.getJSModule(RCTDeviceEventEmitter::class.java)?.emit("RNFileUploader-$eventName", params) - ?: Log.e(TAG, "sendEvent() failed due reactContext == null!") - } -} diff --git a/android/src/main/java/com/vydia/RNUploader/NotificationActions.kt b/android/src/main/java/com/vydia/RNUploader/NotificationActions.kt deleted file mode 100644 index 0aed437d..00000000 --- a/android/src/main/java/com/vydia/RNUploader/NotificationActions.kt +++ /dev/null @@ -1,24 +0,0 @@ -package com.vydia.RNUploader - -import android.app.PendingIntent -import android.content.Context -import android.content.Intent - -class NotificationActions { - var INTENT_ACTION = "com.vydia.RNUploader.notification.action" - - val PARAM_ACTION = "action" - val PARAM_UPLOAD_ID = "uploadId" - - val ACTION_CANCEL_UPLOAD = "cancelUpload" - - - fun getCancelUploadAction(context: Context?, - requestCode: Int, - uploadID: String?): PendingIntent? { - val intent = Intent(INTENT_ACTION) - intent.putExtra(PARAM_ACTION, ACTION_CANCEL_UPLOAD) - intent.putExtra(PARAM_UPLOAD_ID, uploadID) - return PendingIntent.getBroadcast(context, requestCode, intent, PendingIntent.FLAG_UPDATE_CURRENT) - } -} diff --git a/android/src/main/java/com/vydia/RNUploader/NotificationActionsReceiver.kt b/android/src/main/java/com/vydia/RNUploader/NotificationActionsReceiver.kt deleted file mode 100644 index 8d7752ce..00000000 --- a/android/src/main/java/com/vydia/RNUploader/NotificationActionsReceiver.kt +++ /dev/null @@ -1,44 +0,0 @@ -package com.vydia.RNUploader - -import android.content.BroadcastReceiver -import android.content.Context -import android.content.Intent -import android.util.Log -import com.facebook.react.bridge.Arguments -import com.facebook.react.bridge.ReactApplicationContext -import com.facebook.react.bridge.WritableMap -import com.facebook.react.modules.core.DeviceEventManagerModule.RCTDeviceEventEmitter -import net.gotev.uploadservice.UploadService - -class NotificationActionsReceiver : BroadcastReceiver() { - - private val TAG = "NotificationActReceiver" - - private val reactContext: ReactApplicationContext? = null - override fun onReceive(context: Context?, intent: Intent?) { - if (intent == null || NotificationActions().INTENT_ACTION == intent.action) { - return - } - - if (NotificationActions().ACTION_CANCEL_UPLOAD == intent.getStringExtra(NotificationActions().PARAM_ACTION)) { - onUserRequestedUploadCancellation(context!!, intent.getStringExtra(NotificationActions().PARAM_UPLOAD_ID)!!) - } - } - - private fun onUserRequestedUploadCancellation(context: Context, uploadId: String) { - Log.e("CANCEL_UPLOAD", "User requested cancellation of upload with ID: $uploadId") - UploadService.stopUpload(uploadId) - val params = Arguments.createMap() - params.putString("id", uploadId) - sendEvent("cancelled", params, context) - } - - /** - * Sends an event to the JS module. - */ - private fun sendEvent(eventName: String, params: WritableMap?, context: Context) { - reactContext?.getJSModule(RCTDeviceEventEmitter::class.java)?.emit("RNFileUploader-$eventName", params) - ?: Log.e(TAG, "sendEvent() failed due reactContext == null!") - } - -} diff --git a/android/src/main/java/com/vydia/RNUploader/UploaderModule.kt b/android/src/main/java/com/vydia/RNUploader/UploaderModule.kt deleted file mode 100644 index b9e32887..00000000 --- a/android/src/main/java/com/vydia/RNUploader/UploaderModule.kt +++ /dev/null @@ -1,356 +0,0 @@ -package com.vydia.RNUploader - -import android.app.Application -import android.app.NotificationChannel -import android.app.NotificationManager -import android.content.Context -import android.os.Build -import android.util.Log -import android.webkit.MimeTypeMap -import com.facebook.react.BuildConfig -import com.facebook.react.bridge.* -import net.gotev.uploadservice.UploadService -import net.gotev.uploadservice.UploadServiceConfig.httpStack -import net.gotev.uploadservice.UploadServiceConfig.initialize -import net.gotev.uploadservice.data.UploadNotificationConfig -import net.gotev.uploadservice.data.UploadNotificationStatusConfig -import net.gotev.uploadservice.observer.request.GlobalRequestObserver -import net.gotev.uploadservice.okhttp.OkHttpStack -import net.gotev.uploadservice.placeholders.Placeholder -import net.gotev.uploadservice.protocols.binary.BinaryUploadRequest -import net.gotev.uploadservice.protocols.multipart.MultipartUploadRequest -import okhttp3.OkHttpClient -import java.io.File -import java.util.concurrent.TimeUnit - -class UploaderModule(val reactContext: ReactApplicationContext) : ReactContextBaseJavaModule(reactContext), LifecycleEventListener { - private val TAG = "UploaderBridge" - private var notificationChannelID = "BackgroundUploadChannel" - private var isGlobalRequestObserver = false - - override fun getName(): String { - return "RNFileUploader" - } - - /* - Gets file information for the path specified. Example valid path is: /storage/extSdCard/DCIM/Camera/20161116_074726.mp4 - Returns an object such as: {extension: "mp4", size: "3804316", exists: true, mimeType: "video/mp4", name: "20161116_074726.mp4"} - */ - @ReactMethod - fun getFileInfo(path: String, promise: Promise) { - try { - val params = Arguments.createMap() - val fileInfo = File(path) - params.putString("name", fileInfo.name) - if (!fileInfo.exists() || !fileInfo.isFile) { - params.putBoolean("exists", false) - } else { - params.putBoolean("exists", true) - params.putString("size", fileInfo.length().toString()) //use string form of long because there is no putLong and converting to int results in a max size of 17.2 gb, which could happen. Javascript will need to convert it to a number - val extension = MimeTypeMap.getFileExtensionFromUrl(path) - params.putString("extension", extension) - val mimeType = MimeTypeMap.getSingleton().getMimeTypeFromExtension(extension.lowercase()) - params.putString("mimeType", mimeType) - } - promise.resolve(params) - } catch (exc: Exception) { - exc.printStackTrace() - Log.e(TAG, exc.message, exc) - promise.reject(exc) - } - } - - private fun configureUploadServiceHTTPStack(options: ReadableMap, promise: Promise) { - var followRedirects = true - var followSslRedirects = true - var retryOnConnectionFailure = true - var connectTimeout = 15 - var writeTimeout = 30 - var readTimeout = 30 - //TODO: make 'cache' customizable - if (options.hasKey("followRedirects")) { - if (options.getType("followRedirects") != ReadableType.Boolean) { - promise.reject(IllegalArgumentException("followRedirects must be a boolean.")) - return - } - followRedirects = options.getBoolean("followRedirects") - } - if (options.hasKey("followSslRedirects")) { - if (options.getType("followSslRedirects") != ReadableType.Boolean) { - promise.reject(IllegalArgumentException("followSslRedirects must be a boolean.")) - return - } - followSslRedirects = options.getBoolean("followSslRedirects") - } - if (options.hasKey("retryOnConnectionFailure")) { - if (options.getType("retryOnConnectionFailure") != ReadableType.Boolean) { - promise.reject(IllegalArgumentException("retryOnConnectionFailure must be a boolean.")) - return - } - retryOnConnectionFailure = options.getBoolean("retryOnConnectionFailure") - } - if (options.hasKey("connectTimeout")) { - if (options.getType("connectTimeout") != ReadableType.Number) { - promise.reject(IllegalArgumentException("connectTimeout must be a number.")) - return - } - connectTimeout = options.getInt("connectTimeout") - } - if (options.hasKey("writeTimeout")) { - if (options.getType("writeTimeout") != ReadableType.Number) { - promise.reject(IllegalArgumentException("writeTimeout must be a number.")) - return - } - writeTimeout = options.getInt("writeTimeout") - } - if (options.hasKey("readTimeout")) { - if (options.getType("readTimeout") != ReadableType.Number) { - promise.reject(IllegalArgumentException("readTimeout must be a number.")) - return - } - readTimeout = options.getInt("readTimeout") - } - httpStack = OkHttpStack(OkHttpClient().newBuilder() - .followRedirects(followRedirects) - .followSslRedirects(followSslRedirects) - .retryOnConnectionFailure(retryOnConnectionFailure) - .connectTimeout(connectTimeout.toLong(), TimeUnit.SECONDS) - .writeTimeout(writeTimeout.toLong(), TimeUnit.SECONDS) - .readTimeout(readTimeout.toLong(), TimeUnit.SECONDS) - .cache(null) - .build()) - } - - /* - * Starts a file upload. - * Returns a promise with the string ID of the upload. - */ - @ReactMethod - fun startUpload(options: ReadableMap, promise: Promise) { - for (key in arrayOf("url", "path")) { - if (!options.hasKey(key)) { - promise.reject(java.lang.IllegalArgumentException("Missing '$key' field.")) - return - } - if (options.getType(key) != ReadableType.String) { - promise.reject(java.lang.IllegalArgumentException("$key must be a string.")) - return - } - } - if (options.hasKey("headers") && options.getType("headers") != ReadableType.Map) { - promise.reject(java.lang.IllegalArgumentException("headers must be a hash.")) - return - } - if (options.hasKey("notification") && options.getType("notification") != ReadableType.Map) { - promise.reject(java.lang.IllegalArgumentException("notification must be a hash.")) - return - } - configureUploadServiceHTTPStack(options, promise) - var requestType: String? = "raw" - if (options.hasKey("type")) { - requestType = options.getString("type") - if (requestType == null) { - promise.reject(java.lang.IllegalArgumentException("type must be string.")) - return - } - if (requestType != "raw" && requestType != "multipart") { - promise.reject(java.lang.IllegalArgumentException("type should be string: raw or multipart.")) - return - } - } - val notification: WritableMap = WritableNativeMap() - notification.putBoolean("enabled", true) - if (options.hasKey("notification")) { - notification.merge(options.getMap("notification")!!) - } - - val application = reactContext.applicationContext as Application - - reactContext.addLifecycleEventListener(this) - - if (notification.hasKey("notificationChannel")) { - notificationChannelID = notification.getString("notificationChannel")!! - } - - createNotificationChannel() - - initialize(application, notificationChannelID, BuildConfig.DEBUG) - - if(!isGlobalRequestObserver) { - isGlobalRequestObserver = true - GlobalRequestObserver(application, GlobalRequestObserverDelegate(reactContext)) - } - - val url = options.getString("url") - val filePath = options.getString("path") - val method = if (options.hasKey("method") && options.getType("method") == ReadableType.String) options.getString("method") else "POST" - val maxRetries = if (options.hasKey("maxRetries") && options.getType("maxRetries") == ReadableType.Number) options.getInt("maxRetries") else 2 - val customUploadId = if (options.hasKey("customUploadId") && options.getType("method") == ReadableType.String) options.getString("customUploadId") else null - try { - val request = if (requestType == "raw") { - BinaryUploadRequest(this.reactApplicationContext, url!!) - .setFileToUpload(filePath!!) - } else { - if (!options.hasKey("field")) { - promise.reject(java.lang.IllegalArgumentException("field is required field for multipart type.")) - return - } - if (options.getType("field") != ReadableType.String) { - promise.reject(java.lang.IllegalArgumentException("field must be string.")) - return - } - MultipartUploadRequest(this.reactApplicationContext, url!!) - .addFileToUpload(filePath!!, options.getString("field")!!) - } - request.setMethod(method!!) - .setMaxRetries(maxRetries) - if (notification.getBoolean("enabled")) { - // [SMM-1836] Check if hideError flag is set - val hideError = notification.hasKey("hideError") && notification.getBoolean("hideError") - - // for placeholder guide, reference this example: - // https://github.com/gotev/android-upload-service/blob/f2254d47b33a3891e33c600f39fef7abfa9ffdf9/uploadservice/src/main/java/net/gotev/uploadservice/UploadServiceConfig.kt#L131-L157 - val notificationConfig = UploadNotificationConfig( - notificationChannelId = notificationChannelID, - isRingToneEnabled = notification.hasKey("enableRingTone") && notification.getBoolean("enableRingTone"), - progress = UploadNotificationStatusConfig( - title = if (notification.hasKey("onProgressTitle")) notification.getString("onProgressTitle")!! else "Upload in progress", - message = if (notification.hasKey("onProgressMessage")) notification.getString("onProgressMessage")!! else "Uploading at ${Placeholder.UploadRate} (${Placeholder.Progress})" - ), - success = UploadNotificationStatusConfig( - title = if (notification.hasKey("onCompleteTitle")) notification.getString("onCompleteTitle")!! else "Upload completed", - message = if (notification.hasKey("onCompleteMessage")) notification.getString("onCompleteMessage")!! else "Upload completed successfully in ${Placeholder.ElapsedTime}", - autoClear = notification.hasKey("autoClear") && notification.getBoolean("autoClear") - ), - // [SMM-1836] Conditionally hide error notifications based on hideError flag - error = if (hideError) { - // Hide error notifications using empty strings - UploadNotificationStatusConfig( - title = "", - message = "", - autoClear = true - ) - } else { - // Show normal error notifications - UploadNotificationStatusConfig( - title = if (notification.hasKey("onErrorTitle")) notification.getString("onErrorTitle")!! else "Upload failed", - message = if (notification.hasKey("onErrorMessage")) notification.getString("onErrorMessage")!! else "Error during upload" - ) - }, - cancelled = UploadNotificationStatusConfig( - title = if (notification.hasKey("onCancelledTitle")) notification.getString("onCancelledTitle")!! else "Upload cancelled", - message = if (notification.hasKey("onCancelledMessage")) notification.getString("onCancelledMessage")!! else "" - ) - ) - request.setNotificationConfig { _, _ -> - notificationConfig - } - } - if (options.hasKey("parameters")) { - if (requestType == "raw") { - promise.reject(java.lang.IllegalArgumentException("Parameters supported only in multipart type")) - return - } - val parameters = options.getMap("parameters") - val keys = parameters!!.keySetIterator() - while (keys.hasNextKey()) { - val key = keys.nextKey() - if (parameters.getType(key) != ReadableType.String) { - promise.reject(java.lang.IllegalArgumentException("Parameters must be string key/values. Value was invalid for '$key'")) - return - } - request.addParameter(key, parameters.getString(key)!!) - } - } - if (options.hasKey("headers")) { - val headers = options.getMap("headers") - val keys = headers!!.keySetIterator() - while (keys.hasNextKey()) { - val key = keys.nextKey() - if (headers.getType(key) != ReadableType.String) { - promise.reject(java.lang.IllegalArgumentException("Headers must be string key/values. Value was invalid for '$key'")) - return - } - request.addHeader(key, headers.getString(key)!!) - } - } - if (customUploadId != null) - request.setUploadID(customUploadId) - - val uploadId = request.startUpload() - promise.resolve(uploadId) - } catch (exc: java.lang.Exception) { - exc.printStackTrace() - Log.e(TAG, exc.message, exc) - promise.reject(exc) - } - } - - /* - * Cancels file upload - * Accepts upload ID as a first argument, this upload will be cancelled - * Event "cancelled" will be fired when upload is cancelled. - */ - @ReactMethod - fun cancelUpload(cancelUploadId: String?, promise: Promise) { - if (cancelUploadId !is String) { - promise.reject(java.lang.IllegalArgumentException("Upload ID must be a string")) - return - } - try { - UploadService.stopUpload(cancelUploadId) - promise.resolve(true) - } catch (exc: java.lang.Exception) { - exc.printStackTrace() - Log.e(TAG, exc.message, exc) - promise.reject(exc) - } - } - - /* - * Cancels all file uploads - */ - @ReactMethod - fun stopAllUploads(promise: Promise) { - try { - UploadService.stopAllUploads() - promise.resolve(true) - } catch (exc: java.lang.Exception) { - exc.printStackTrace() - Log.e(TAG, exc.message, exc) - promise.reject(exc) - } - } - - // Customize the notification channel as you wish. This is only for a bare minimum example - private fun createNotificationChannel() { - if (Build.VERSION.SDK_INT >= 26) { - val channel = NotificationChannel( - notificationChannelID, - "Background Upload Channel", - NotificationManager.IMPORTANCE_LOW - ) - val manager = reactApplicationContext.getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager - manager.createNotificationChannel(channel) - } - } - - @ReactMethod - fun addListener(type: String?) { - // Keep: Required for RN built in Event Emitter Calls. - } - - @ReactMethod - fun removeListeners(type: Int?) { - // Keep: Required for RN built in Event Emitter Calls. - } - - override fun onHostResume() { - } - - override fun onHostPause() { - } - - override fun onHostDestroy() { - } -} diff --git a/android/src/main/java/com/vydia/RNUploader/UploaderReactPackage.kt b/android/src/main/java/com/vydia/RNUploader/UploaderReactPackage.kt deleted file mode 100644 index d07faae4..00000000 --- a/android/src/main/java/com/vydia/RNUploader/UploaderReactPackage.kt +++ /dev/null @@ -1,18 +0,0 @@ -package com.vydia.RNUploader; - -import android.view.View -import com.facebook.react.ReactPackage -import com.facebook.react.bridge.NativeModule -import com.facebook.react.bridge.ReactApplicationContext -import com.facebook.react.uimanager.ReactShadowNode -import com.facebook.react.uimanager.ViewManager - -class UploaderReactPackage : ReactPackage { - override fun createViewManagers( - reactContext: ReactApplicationContext - ): MutableList>> = mutableListOf() - - override fun createNativeModules( - reactContext: ReactApplicationContext - ): MutableList = listOf(UploaderModule(reactContext)).toMutableList() -} \ No newline at end of file diff --git a/android/src/main/res/layout/notification.xml b/android/src/main/res/layout/notification.xml new file mode 100644 index 00000000..b5a95f19 --- /dev/null +++ b/android/src/main/res/layout/notification.xml @@ -0,0 +1,34 @@ + + + + + + + + + + + + + diff --git a/android/src/test/java/ai/openspace/backgroundupload/BodyCapTest.kt b/android/src/test/java/ai/openspace/backgroundupload/BodyCapTest.kt new file mode 100644 index 00000000..53a75385 --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/BodyCapTest.kt @@ -0,0 +1,70 @@ +package ai.openspace.backgroundupload + +import okio.Buffer +import okio.BufferedSource +import okio.buffer +import okio.ForwardingSource +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Test + +class BodyCapTest { + /** A source that counts the bytes read from it. */ + private class Counting(bytes: ByteArray) : ForwardingSource(Buffer().write(bytes)) { + var read = 0L + override fun read(sink: Buffer, byteCount: Long): Long = + super.read(sink, byteCount).also { if (it > 0) read += it } + } + + private fun source(bytes: ByteArray): Pair { + val c = Counting(bytes) + return c to c.buffer() + } + + @Test + fun `a body under the cap is read whole`() { + val (_, s) = source("hello".toByteArray()) + assertEquals(BodyCap.Capped("hello", false), BodyCap.read(s, 10)) + } + + @Test + fun `a body at exactly the cap is not truncated`() { + val (_, s) = source("0123456789".toByteArray()) + assertEquals(BodyCap.Capped("0123456789", false), BodyCap.read(s, 10)) + } + + @Test + fun `a huge body stops streaming just past the cap`() { + val (counting, s) = source(ByteArray(5_000_000) { 'x'.code.toByte() }) + val capped = BodyCap.read(s, 1_000) + assertTrue(capped.truncated) + assertEquals(1_000, capped.text.length) + // okio reads in 8 KB segments; nowhere near the 5 MB body. + assertTrue("read ${counting.read}", counting.read < 64 * 1024) + } + + @Test + fun `a cut inside a UTF-8 character backs off to the last whole one`() { + val euros = "€".repeat(4).toByteArray(Charsets.UTF_8) // 12 bytes + val (_, s) = source(euros) + val capped = BodyCap.read(s, 10) + assertEquals("€".repeat(3), capped.text) + assertTrue(capped.truncated) + } + + @Test + fun `cap measures UTF-8 bytes, not characters`() { + assertEquals("ab" to false, BodyCap.cap("ab", 2)) + assertEquals("é" to true, BodyCap.cap("éé", 3)) + assertEquals(null to false, BodyCap.cap(null, 3)) + // A 4-byte character (an emoji) is never split. + assertEquals("a" to true, BodyCap.cap("a😀", 4)) + } + + @Test + fun `bytes that are not UTF-8 are cut at the cap`() { + val bad = ByteArray(8) { 0x80.toByte() } // continuation bytes only + assertEquals(5, BodyCap.utf8Boundary(bad, 5)) + assertEquals(3, BodyCap.utf8Boundary("abc".toByteArray(), 5)) + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/BodyStagingTest.kt b/android/src/test/java/ai/openspace/backgroundupload/BodyStagingTest.kt new file mode 100644 index 00000000..216c69a6 --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/BodyStagingTest.kt @@ -0,0 +1,207 @@ +package ai.openspace.backgroundupload + +import org.junit.Assert.assertArrayEquals +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertThrows +import org.junit.Assert.assertTrue +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import java.io.File + +class BodyStagingTest { + @get:Rule + val tmp = TemporaryFolder() + + private fun source(name: String, bytes: ByteArray) = File(tmp.newFolder(), name).apply { writeBytes(bytes) } + + @Test + fun `json bytes are the data text, with a default content type`() { + val dir = tmp.newFolder() + val staged = BodyStaging.stage(desc(dataJson = """{"n":1,"t":"é"}""", headers = mapOf()), dir, 1) + assertEquals(StagedBody.JSON, staged.body.kind) + assertEquals("body-1.json", staged.body.fileName) + assertEquals("""{"n":1,"t":"é"}""", File(dir, "body-1.json").readText()) + assertEquals(File(dir, "body-1.json").length(), staged.body.totalBytes) + assertEquals(mapOf("Content-Type" to "application/json"), staged.headers) + } + + @Test + fun `a caller content type wins for json, in any case`() { + val staged = BodyStaging.stage(desc(dataJson = "{}", headers = mapOf("content-type" to "application/vnd+json")), tmp.newFolder(), 1) + assertEquals(mapOf("content-type" to "application/vnd+json"), staged.headers) + } + + @Test + fun `multipart bytes follow RFC 7578`() { + val photo = source("photo.jpg", byteArrayOf(1, 2, 3)) + val form = listOf( + FormPart("meta", "application/json", "{\"a\":\"b\"}", null, null), + FormPart("pho\"to", "image/jpeg", null, photo.path, null), + FormPart("named", "image/jpeg", null, photo.path, "new\nname.jpg"), + ) + val target = File(tmp.newFolder(), "body.multipart") + BodyStaging.writeMultipart(form, "BOUND", target) + val expected = ("--BOUND\r\n" + + "Content-Disposition: form-data; name=\"meta\"\r\n" + + "Content-Type: application/json\r\n\r\n" + + "{\"a\":\"b\"}\r\n" + + "--BOUND\r\n" + + "Content-Disposition: form-data; name=\"pho%22to\"; filename=\"photo.jpg\"\r\n" + + "Content-Type: image/jpeg\r\n\r\n").toByteArray() + byteArrayOf(1, 2, 3) + ("\r\n" + + "--BOUND\r\n" + + "Content-Disposition: form-data; name=\"named\"; filename=\"new%0Aname.jpg\"\r\n" + + "Content-Type: image/jpeg\r\n\r\n").toByteArray() + byteArrayOf(1, 2, 3) + ("\r\n" + + "--BOUND--\r\n").toByteArray() + assertArrayEquals(expected, target.readBytes()) + } + + @Test + fun `form staging always sets the library content type with its boundary`() { + val dir = tmp.newFolder() + val staged = BodyStaging.stage( + desc(form = listOf(FormPart("a", "text/plain", "x", null, null)), headers = mapOf("CONTENT-TYPE" to "text/plain")), + dir, 2, + ) + val boundary = staged.body.boundary!! + assertTrue(boundary.startsWith("----RNBGU")) + assertEquals(mapOf("Content-Type" to "multipart/form-data; boundary=$boundary"), staged.headers) + assertTrue(File(dir, "body-2.multipart").readText().startsWith("--$boundary\r\n")) + } + + @Test + fun `a file body is copied and the source stays`() { + val src = source("a.bin", ByteArray(1000) { it.toByte() }) + val dir = tmp.newFolder() + val staged = BodyStaging.stage(desc(file = src.path), dir, 3) + assertTrue(src.exists()) + assertArrayEquals(src.readBytes(), File(dir, "file-3").readBytes()) + assertEquals(1000, staged.body.totalBytes) + assertEquals(mapOf("Authorization" to "Bearer old"), staged.headers) // no content type added + src.delete() + assertTrue(File(dir, "file-3").exists()) + } + + @Test + fun `a chunked file is moved and must tile the parts`() { + val src = source("video.bin", ByteArray(20)) + val dir = tmp.newFolder() + val staged = BodyStaging.stage(desc(url = null, file = src.path, parts = listOf(part(0, 10), part(10, 20))), dir, 1) + assertFalse(src.exists()) + assertEquals(20, File(dir, "blob").length()) + assertEquals(StagedBody.CHUNKED, staged.body.kind) + assertEquals(20, staged.body.totalBytes) + } + + @Test + fun `an orphan blob is adopted when the source is gone`() { + val dir = tmp.newFolder() + File(dir, "blob").writeBytes(ByteArray(20)) + val staged = BodyStaging.stage(desc(url = null, file = "/gone.bin", parts = listOf(part(0, 20))), dir, 1) + assertEquals(StagedBody.CHUNKED, staged.body.kind) + } + + @Test + fun `a tiling mismatch rejects E_INVALID before the move, so the source stays`() { + val src = source("video.bin", ByteArray(20)) + val dir = tmp.newFolder() + val e = assertThrows(QueueException::class.java) { + BodyStaging.stage(desc(url = null, file = src.path, parts = listOf(part(0, 15))), dir, 1) + } + assertEquals(QueueException.E_INVALID, e.code) + assertTrue(src.exists()) + assertFalse(File(dir, "blob").exists()) + } + + @Test + fun `keepOwned runs over the owned blob and ignores the path`() { + val dir = tmp.newFolder() + val owned = File(dir, "blob").apply { writeBytes(ByteArray(20)) } + val other = source("other.bin", ByteArray(5)) + val staged = BodyStaging.stage( + desc(url = null, file = other.path, parts = listOf(part(0, 20))), dir, 2, owned, keepOwned = true, + ) + assertTrue(other.exists()) + assertEquals("blob", staged.body.fileName) + assertEquals(20, owned.length()) + } + + @Test + fun `a present source wins over the owned blob and moves to this generation's name`() { + val dir = tmp.newFolder() + val owned = File(dir, "blob").apply { writeBytes(ByteArray(20)) } + val bytes = ByteArray(20) { (it + 1).toByte() } + val src = source("new.bin", bytes) + val staged = BodyStaging.stage(desc(url = null, file = src.path, parts = listOf(part(0, 5), part(5, 20))), dir, 3, owned) + assertEquals("blob-3", staged.body.fileName) + assertArrayEquals(bytes, File(dir, "blob-3").readBytes()) + assertFalse(src.exists()) + // The old entry's blob is untouched until the new entry is saved and prunes it. + assertArrayEquals(ByteArray(20), owned.readBytes()) + } + + @Test + fun `a present source of another size is checked against its own length`() { + val dir = tmp.newFolder() + val owned = File(dir, "blob").apply { writeBytes(ByteArray(20)) } + val src = source("new.bin", ByteArray(30)) + val staged = BodyStaging.stage(desc(url = null, file = src.path, parts = listOf(part(0, 30))), dir, 2, owned) + assertEquals("blob-2", staged.body.fileName) + assertEquals(30, staged.body.totalBytes) + assertEquals(20, owned.length()) + } + + @Test + fun `with the source gone, the owned blob is the fallback`() { + val dir = tmp.newFolder() + File(dir, "blob").writeBytes(ByteArray(20)) + val staged = BodyStaging.stage( + desc(url = null, file = "/gone.bin", parts = listOf(part(0, 5), part(5, 20))), dir, 2, File(dir, "blob"), + ) + assertEquals("blob", staged.body.fileName) + } + + @Test + fun `with the source gone, this generation's crash leftover wins over the owned blob`() { + val dir = tmp.newFolder() + File(dir, "blob").writeBytes(ByteArray(20)) + File(dir, "blob-2").writeBytes(ByteArray(30)) + val staged = BodyStaging.stage(desc(url = null, file = "/gone.bin", parts = listOf(part(0, 30))), dir, 2, File(dir, "blob")) + assertEquals("blob-2", staged.body.fileName) + } + + @Test + fun `a chunked body with no source and no blob rejects E_FILE_MISSING`() { + val e = assertThrows(QueueException::class.java) { + BodyStaging.stage(desc(url = null, file = "/gone.bin", parts = listOf(part(0, 20))), tmp.newFolder(), 2, null) + } + assertEquals(QueueException.E_FILE_MISSING, e.code) + } + + @Test + fun `a missing source rejects E_FILE_MISSING and writes nothing`() { + val dir = tmp.newFolder() + val form = listOf( + FormPart("a", "text/plain", "x", null, null), + FormPart("b", "image/jpeg", null, "/missing.jpg", null), + ) + val e = assertThrows(QueueException::class.java) { BodyStaging.stage(desc(form = form), dir, 1) } + assertEquals(QueueException.E_FILE_MISSING, e.code) + assertEquals(0, dir.list()!!.size) + val f = assertThrows(QueueException::class.java) { BodyStaging.stage(desc(file = "/missing.bin"), dir, 1) } + assertEquals(QueueException.E_FILE_MISSING, f.code) + val c = assertThrows(QueueException::class.java) { + BodyStaging.stage(desc(url = null, file = "/missing.bin", parts = listOf(part(0, 1))), dir, 1) + } + assertEquals(QueueException.E_FILE_MISSING, c.code) + } + + @Test + fun `no body stages nothing`() { + val dir = tmp.newFolder() + val staged = BodyStaging.stage(desc(method = "DELETE"), dir, 1) + assertEquals(StagedBody(StagedBody.NONE, null, null, 0), staged.body) + assertEquals(0, dir.list()!!.size) + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/ChunkedEngineTest.kt b/android/src/test/java/ai/openspace/backgroundupload/ChunkedEngineTest.kt new file mode 100644 index 00000000..c93f1dff --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/ChunkedEngineTest.kt @@ -0,0 +1,87 @@ +package ai.openspace.backgroundupload + +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.yield +import org.junit.Assert.assertEquals +import org.junit.Assert.assertThrows +import org.junit.Assert.assertTrue +import org.junit.Test +import java.util.concurrent.ConcurrentHashMap + +class ChunkedEngineTest { + + // runBlocking is single-threaded, so the overlap is deterministic. Every + // executor suspends at yield(), so all launchable siblings start before + // any executor finishes. + private class Tracker { + var inFlight = 0 + var maxInFlight = 0 + val executions = ConcurrentHashMap() + + suspend fun execute(index: Int) { + executions.merge(index, 1, Int::plus) + inFlight++ + maxInFlight = maxOf(maxInFlight, inFlight) + yield() + yield() + inFlight-- + } + } + + @Test + fun `runs every part exactly once`() = runBlocking { + val tracker = Tracker() + ChunkedEngine.run((0 until 10).toList()) { tracker.execute(it) } + assertEquals((0 until 10).associateWith { 1 }, tracker.executions.toMap()) + } + + @Test + fun `never more than WINDOW parts in flight`() = runBlocking { + val tracker = Tracker() + ChunkedEngine.run((0 until 10).toList()) { tracker.execute(it) } + assertEquals(3, ChunkedEngine.WINDOW) + assertEquals(ChunkedEngine.WINDOW, tracker.maxInFlight) + } + + @Test + fun `respects a smaller window`() = runBlocking { + val tracker = Tracker() + ChunkedEngine.run((0 until 5).toList(), window = 1) { tracker.execute(it) } + assertEquals(1, tracker.maxInFlight) + } + + @Test + fun `an empty part list completes immediately`() = runBlocking { + ChunkedEngine.run(emptyList()) { throw AssertionError("must not execute") } + } + + @Test + fun `a part failure propagates and cancels the remaining parts`() { + val tracker = Tracker() + val thrown = assertThrows(IllegalStateException::class.java) { + runBlocking { + ChunkedEngine.run((0 until 10).toList()) { index -> + if (index == 0) throw IllegalStateException("part rejected") + tracker.execute(index) + } + } + } + assertEquals("part rejected", thrown.message) + assertTrue(tracker.executions.size < 10) + } + + @Test + fun `a park from one part stops the siblings with the park itself`() { + // The worker needs the ParkException back, not a CancellationException. + val thrown = assertThrows(EntryRun.ParkException::class.java) { + runBlocking { + ChunkedEngine.run((0 until 6).toList()) { index -> + yield() + if (index == 1) throw EntryRun.ParkException(4) + yield() + } + } + } + assertEquals(4, thrown.headerGeneration) + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/ChunkedPartsTest.kt b/android/src/test/java/ai/openspace/backgroundupload/ChunkedPartsTest.kt new file mode 100644 index 00000000..e7ba858e --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/ChunkedPartsTest.kt @@ -0,0 +1,56 @@ +package ai.openspace.backgroundupload + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test + +class ChunkedPartsTest { + + @Test + fun `tilesExactly covers the edge shapes`() { + assertTrue(ChunkedParts.tilesExactly(listOf(part(0, 100), part(100, 250)), 250)) + assertTrue(ChunkedParts.tilesExactly(listOf(part(100, 250), part(0, 100)), 250)) // any order + assertFalse(ChunkedParts.tilesExactly(emptyList(), 0)) + assertFalse(ChunkedParts.tilesExactly(listOf(part(0, 0)), 0)) // empty range + assertFalse(ChunkedParts.tilesExactly(listOf(part(0, 100), part(150, 250)), 250)) // gap + assertFalse(ChunkedParts.tilesExactly(listOf(part(0, 150), part(100, 250)), 250)) // overlap + assertFalse(ChunkedParts.tilesExactly(listOf(part(50, 250)), 250)) // not from 0 + assertFalse(ChunkedParts.tilesExactly(listOf(part(0, 200)), 250)) // short + assertFalse(ChunkedParts.tilesExactly(listOf(part(0, 251)), 250)) // past the end + } + + @Test + fun `the same parts in another order are the same upload`() { + val a = listOf(part(0, 100), part(100, 250)) + assertTrue(ChunkedParts.sameParts(a, a.reversed())) + // Headers are not compared: a resume sends fresh ones. + assertTrue(ChunkedParts.sameParts(a, a.map { it.copy(headers = mapOf("X" to "new")) })) + } + + @Test + fun `new urls or a new split are different parts`() { + val a = listOf(part(0, 100), part(100, 250)) + assertFalse(ChunkedParts.sameParts(a, listOf(part(0, 100, url = "https://v2/1"), part(100, 250)))) + assertFalse(ChunkedParts.sameParts(a, listOf(part(0, 120), part(120, 250)))) + assertFalse(ChunkedParts.sameParts(a, listOf(part(0, 250)))) + } + + @Test + fun `accepted flags follow the range, not the index`() { + val stored = listOf(part(0, 100, accepted = true), part(100, 250)) + val incoming = listOf(part(100, 250), part(0, 100)) + val carried = ChunkedParts.carryAccepted(stored, incoming) + assertTrue(carried.first { it.start == 0L }.accepted) + assertFalse(carried.first { it.start == 100L }.accepted) + } + + @Test + fun `byte math and pending indexes`() { + val parts = listOf(part(0, 100, accepted = true), part(100, 250)) + assertEquals(250, ChunkedParts.totalBytes(parts)) + assertEquals(100, ChunkedParts.acceptedBytes(parts)) + assertEquals(listOf(1), ChunkedParts.pendingIndexes(parts)) + assertEquals(emptyList(), ChunkedParts.pendingIndexes(ChunkedParts.withAccepted(parts, 1))) + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/EntryParsingTest.kt b/android/src/test/java/ai/openspace/backgroundupload/EntryParsingTest.kt new file mode 100644 index 00000000..900a94db --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/EntryParsingTest.kt @@ -0,0 +1,217 @@ +package ai.openspace.backgroundupload + +import com.facebook.react.bridge.JavaOnlyArray +import com.facebook.react.bridge.JavaOnlyMap +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertThrows +import org.junit.Test + +class EntryParsingTest { + + private fun entryMap(descriptor: JavaOnlyMap, varsJson: Any? = """{"n":1}""") = + JavaOnlyMap.of("id", "e1", "key", "note", "varsJson", varsJson, "descriptor", descriptor) + + private fun base(vararg extra: Any?) = + JavaOnlyMap.of("url", "https://example.com/items", "expiresAt", 9_000.0, *extra) + + @Test + fun `a JSON POST parses with defaults and keeps the JSON text as JS wrote it`() { + val text = """{"text":"hi","n":1,"status":null}""" + val p = EntryParsing.parse(entryMap(base("dataJson", text), varsJson = """{"b":2,"a":null}""")) + assertEquals("e1", p.id) + assertEquals("note", p.key) + assertEquals("""{"b":2,"a":null}""", p.varsJson) // key order and null values survive + assertEquals(9_000L, p.expiresAt) + assertEquals("POST", p.descriptor.method) + assertEquals(text, p.descriptor.dataJson) + assertEquals(StagedBody.JSON, p.descriptor.bodyKind) + } + + @Test + fun `null vars cross as the text null`() { + assertEquals("null", EntryParsing.parse(entryMap(base(), varsJson = "null")).varsJson) + } + + @Test + fun `a dataJson of null is a real JSON body`() { + val d = EntryParsing.parse(entryMap(base("dataJson", "null"))).descriptor + assertEquals("null", d.dataJson) + assertEquals(StagedBody.JSON, d.bodyKind) + } + + @Test + fun `no dataJson is no body`() { + val d = EntryParsing.parse(entryMap(base())).descriptor + assertNull(d.dataJson) + assertEquals(StagedBody.NONE, d.bodyKind) + } + + @Test + fun `a missing or malformed varsJson or dataJson is rejected`() { + val cases = listOf( + entryMap(base(), varsJson = null), + entryMap(base(), varsJson = JavaOnlyMap.of("n", 1.0)), // the old object form + entryMap(base(), varsJson = "{n:1}"), // lenient JSON + entryMap(base(), varsJson = """{"n":1} trailing"""), + entryMap(base("dataJson", "")), + entryMap(base("dataJson", "{'a':1}")), + entryMap(base("dataJson", JavaOnlyMap.of("a", 1.0))), + entryMap(base("data", JavaOnlyMap.of("a", 1.0))), // the old data form would send no body + ) + cases.forEach { m -> + assertThrows("$m", EntryParsing.InvalidEntryException::class.java) { EntryParsing.parse(m) } + } + } + + @Test + fun `chunked parts, headers, accept, retry, and android parse`() { + val d = JavaOnlyMap.of( + "method", "PUT", + "file", "file:///data/a%20b.bin", + "expiresAt", 9_000.0, + "headers", JavaOnlyMap.of("Content-Type", "video/mp4", "X-N", 5.0), + "parts", JavaOnlyArray.of( + JavaOnlyMap.of("url", "https://s3/1", "range", JavaOnlyMap.of("start", 0.0, "end", 10.0)), + JavaOnlyMap.of( + "url", "https://s3/2", "headers", JavaOnlyMap.of("Content-Range", "10-19"), + "range", JavaOnlyMap.of("start", 10.0, "end", 20.0), + ), + ), + "accept", JavaOnlyArray.of(JavaOnlyMap.of("status", 409.0, "bodyIncludes", "already completed")), + "retry", JavaOnlyMap.of( + "backoff", JavaOnlyMap.of("baseMs", 50.0), + "terminalHttp", JavaOnlyMap.of("exempt", JavaOnlyArray()), + ), + "android", JavaOnlyMap.of("noNotification", true), + ) + val parsed = EntryParsing.parse(entryMap(d)).descriptor + assertEquals("/data/a b.bin", parsed.file) + assertEquals(StagedBody.CHUNKED, parsed.bodyKind) + assertEquals(listOf(Part("https://s3/1", mapOf(), 0, 10), Part("https://s3/2", mapOf("Content-Range" to "10-19"), 10, 20)), parsed.parts) + assertEquals(mapOf("Content-Type" to "video/mp4", "X-N" to "5"), parsed.headers) + assertEquals(listOf(UploadOutcome.AcceptRule(409, "already completed")), parsed.accept) + assertEquals(RetryOverride(50, null, null, emptyList()), parsed.retry) + assertEquals(true, parsed.noNotification) + assertEquals("https://s3/2", parsed.reportUrl) + } + + @Test + fun `form parts parse with exactly one of string or path`() { + val d = base( + "form", JavaOnlyArray.of( + JavaOnlyMap.of("name", "meta", "contentType", "application/json", "string", "{}"), + JavaOnlyMap.of("name", "photo", "contentType", "image/jpeg", "path", "/p.jpg", "fileName", "p.jpg"), + ), + ) + assertEquals( + listOf( + FormPart("meta", "application/json", "{}", null, null), + FormPart("photo", "image/jpeg", null, "/p.jpg", "p.jpg"), + ), + EntryParsing.parse(entryMap(d)).descriptor.form, + ) + val both = base("form", JavaOnlyArray.of(JavaOnlyMap.of("name", "x", "contentType", "t", "string", "s", "path", "/p"))) + assertThrows(EntryParsing.InvalidEntryException::class.java) { EntryParsing.parse(entryMap(both)) } + } + + @Test + fun `wifiOnly parses true, false, or absent`() { + assertEquals(true, EntryParsing.parse(entryMap(base("wifiOnly", true))).descriptor.wifiOnly) + assertEquals(false, EntryParsing.parse(entryMap(base("wifiOnly", false))).descriptor.wifiOnly) + assertNull(EntryParsing.parse(entryMap(base())).descriptor.wifiOnly) + assertThrows(EntryParsing.InvalidEntryException::class.java) { + EntryParsing.parse(entryMap(base("wifiOnly", "yes"))) + } + } + + @Test + fun `a pause scope with no keys is the whole queue, and keys must be non-empty strings`() { + assertNull(EntryParsing.scopeKeys(JavaOnlyMap())) + // Codegen passes null when a stale bundle calls pause() with no argument. + assertNull(EntryParsing.scopeKeys(null)) + assertEquals(listOf("capture", "video"), EntryParsing.scopeKeys(JavaOnlyMap.of("keys", JavaOnlyArray.of("capture", "video")))) + assertEquals(emptyList(), EntryParsing.scopeKeys(JavaOnlyMap.of("keys", JavaOnlyArray()))) + // A keys field that native drops would widen the scope to the whole queue. + listOf( + JavaOnlyMap.of("keys", null), + JavaOnlyMap.of("keys", "capture"), + JavaOnlyMap.of("keys", JavaOnlyArray.of("capture", 1.0)), + // JS and the README refuse empty keys; native agrees. + JavaOnlyMap.of("keys", JavaOnlyArray.of("capture", "")), + ).forEach { m -> + assertThrows("$m", EntryParsing.InvalidEntryException::class.java) { EntryParsing.scopeKeys(m) } + } + } + + @Test + fun `what native can not run is rejected`() { + val cases = listOf( + JavaOnlyMap.of("url", "https://example.com"), // no expiresAt + JavaOnlyMap.of("expiresAt", 1.0), // no url and no parts + base("dataJson", "1", "file", "/a"), // two body kinds + base("method", "GET", "dataJson", "1"), // GET with a body + base("method", "GET", "dataJson", "null"), // GET with the JSON body null + base("method", "GET", "file", "/a"), + base("method", "TRACE"), + JavaOnlyMap.of("url", "not a url", "expiresAt", 1.0), + base("headers", JavaOnlyMap.of("Bad\nName", "v")), + base("parts", JavaOnlyArray.of(JavaOnlyMap.of("url", "https://s3/1", "range", JavaOnlyMap.of("start", 0.0, "end", 1.0)))), // parts without file + ) + cases.forEach { d -> + assertThrows("$d", EntryParsing.InvalidEntryException::class.java) { EntryParsing.parse(entryMap(d)) } + } + assertThrows(EntryParsing.InvalidEntryException::class.java) { + EntryParsing.parse(JavaOnlyMap.of("key", "k", "descriptor", base())) + } + } + + @Test + fun `a GET with no body parses`() { + assertEquals("GET", EntryParsing.parse(entryMap(base("method", "GET"))).descriptor.method) + } + + @Test + fun `a bad header value is rejected with its name and offset, never its value`() { + val secret = "Bearer s3cr3t-token" + val e = assertThrows(EntryParsing.InvalidEntryException::class.java) { + EntryParsing.parse(entryMap(base("headers", JavaOnlyMap.of("Authorization", "$secret\n")))) + } + assertEquals("headers: the value of header 'Authorization' has an invalid character at offset ${secret.length}", e.message) + assertFalse(e.message!!.contains("s3cr3t")) + } + + @Test + fun `a bad header name is rejected with the valid part before the offset only`() { + val e = assertThrows(EntryParsing.InvalidEntryException::class.java) { + EntryParsing.requireValidHeaders(mapOf("Authorization: Bearer s3cr3t" to "v"), "headers") + } + assertEquals("headers: the header name that starts 'Authorization:' has an invalid character at offset 14", e.message) + assertFalse(e.message!!.contains("s3cr3t")) + assertThrows(EntryParsing.InvalidEntryException::class.java) { + EntryParsing.requireValidHeaders(mapOf("" to "v"), "headers") + } + } + + @Test + fun `the header check accepts what OkHttp sends`() { + EntryParsing.requireValidHeaders(mapOf("X-Tab" to "a\tb", "X-Tilde" to "~!#", "Content-Range" to "bytes 0-9/10"), "headers") + okhttp3.Headers.Builder().add("X-Tab", "a\tb").add("X-Tilde", "~!#") + } + + @Test + fun `an updateHeaders patch is checked like descriptor headers`() { + assertEquals(mapOf("Authorization" to "Bearer new"), EntryParsing.headerPatch(JavaOnlyMap.of("Authorization", "Bearer new"))) + val e = assertThrows(EntryParsing.InvalidEntryException::class.java) { + EntryParsing.headerPatch(JavaOnlyMap.of("Authorization", "Bearer\nnew")) + } + assertFalse(e.message!!.contains("Bearer")) + } + + @Test + fun `file scheme stripping`() { + assertEquals("/a/b c.jpg", EntryParsing.stripFileScheme("file:///a/b%20c.jpg")) + assertEquals("/a/b.jpg", EntryParsing.stripFileScheme("/a/b.jpg")) + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/EntryRunTest.kt b/android/src/test/java/ai/openspace/backgroundupload/EntryRunTest.kt new file mode 100644 index 00000000..4471e889 --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/EntryRunTest.kt @@ -0,0 +1,511 @@ +package ai.openspace.backgroundupload + +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.runBlocking +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertThrows +import org.junit.Assert.assertTrue +import org.junit.Before +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import java.io.File +import java.io.IOException + +/** A scripted [TransferHost]: a manual clock, a send handler, and a log of what the run did. */ +internal class FakeHost(var clock: Long = 10_000L) : TransferHost { + val requests = mutableListOf() + val sleeps = mutableListOf() + val attempts = mutableListOf() + val progress = mutableListOf() + var handler: suspend (TransferRequest, (Long) -> Unit) -> UploadResponse = { _, _ -> UploadResponse(200, "ok", mapOf()) } + var onSleep: () -> Unit = {} + var network = ArrayDeque() + var timeout = false + var foregroundError: Throwable? = null + /** The wifiOnly value of every connectivity check, in order. */ + val wifiChecks = mutableListOf() + + override fun now() = clock + + override suspend fun sleep(ms: Long) { + sleeps += ms + clock += ms + onSleep() + } + + override suspend fun send(request: TransferRequest, onProgress: (Long) -> Unit): UploadResponse { + requests += request + return handler(request, onProgress) + } + + override fun connectivity(wifiOnly: Boolean): Connectivity { + wifiChecks += wifiOnly + return network.removeFirstOrNull() ?: Connectivity.Ok + } + + override suspend fun foreground(entry: QueueEntry) { + foregroundError?.let { throw it } + } + + override fun progressStarted(id: String, total: Long, sent: Long) { + progress += "start:$total:$sent" + } + + override fun progress(id: String, sent: Long, total: Long) { + progress += "$sent" + } + + override fun progressEnded(id: String, completed: Boolean) { + progress += "end:$completed" + } + + override fun attempt(event: AttemptEvent) { + attempts += event + } + + override fun stoppedByTimeout() = timeout +} + +class EntryRunTest { + @get:Rule + val tmp = TemporaryFolder() + + private lateinit var store: QueueStore + private lateinit var journal: EventJournal + private lateinit var settings: QueueSettingsStore + private val events = RecordingEvents() + private val scheduler = FakeScheduler() + private val host = FakeHost() + private lateinit var ops: WorkerOps + private lateinit var controller: QueueController + + @Before + fun setUp() { + val root = tmp.newFolder("queue") + store = QueueStore(root, RequestIndex()) + journal = EventJournal(tmp.newFolder("journal"), retryLater = { _, _ -> }) + settings = QueueSettingsStore(File(root, "settings.json")) + ops = WorkerOps(store, journal, settings, events, scheduler) { host.clock } + controller = QueueController(store, journal, settings, events, scheduler, { false }, { host.clock }) + controller.configureRetry(RetryDefaults(baseMs = 1_000, jitter = 0.0)) + journal.drain(Any()) + } + + private fun run(id: String = "e1") = runBlocking { EntryRun(id, store, ops, host).run() } + + private fun respond(vararg codes: Int) { + val queue = ArrayDeque(codes.toList()) + host.handler = { _, onProgress -> + onProgress(7) + UploadResponse(queue.removeFirst(), "body", mapOf()) + } + } + + private fun outcome() = journal.unacknowledged().single() + + // MARK: - simple + + @Test + fun `an accepted response settles completed after one write-ahead attempt`() { + controller.enqueue(parsed()) + respond(200) + run() + val e = store.load("e1")!! + assertEquals(EntryState.COMPLETED, e.state) + assertEquals(1, e.attempts) + val request = host.requests.single() + assertEquals(e.lastRequestId, request.headers["X-Request-Id"]) + assertEquals("Bearer old", request.headers["Authorization"]) + assertEquals("application/json", request.headers["Content-Type"]) + assertEquals(7L, request.body!!.contentLength()) + assertEquals(listOf("completed"), host.attempts.map { it.outcome }) + assertEquals(EventJournal.KIND_COMPLETED, outcome().kind) + assertEquals(200, outcome().response!!.status) + assertEquals(listOf("start:7:0", "7", "end:true"), host.progress) + } + + @Test + fun `a transient response waits a short backoff in place, with nextAttemptAt on the running row`() { + controller.enqueue(parsed()) + respond(503, 200) + run() + assertEquals(listOf(1_000L), host.sleeps) + assertEquals(listOf("error", "completed"), host.attempts.map { it.outcome }) + assertEquals("http", host.attempts[0].errorKind) + assertEquals(503, host.attempts[0].httpCode) + val waiting = events.rows.first { it.toMap().containsKey("nextAttemptAt") } + assertEquals("running", waiting.state) + assertEquals(11_000.0, waiting.toMap()["nextAttemptAt"]) + assertEquals(2, store.load("e1")!!.attempts) + assertEquals(listOf("start:7:0", "7", "0", "7", "end:true"), host.progress) // bytes reset between attempts + } + + @Test + fun `a backoff longer than 30 s releases the run back to queued with a wake`() { + controller.configureRetry(RetryDefaults(baseMs = 60_000, jitter = 0.0)) + controller.enqueue(parsed()) + respond(503) + run() + val e = store.load("e1")!! + assertEquals(EntryState.QUEUED, e.state) + assertEquals(host.clock + 60_000, e.nextAttemptAt) + assertEquals(1, e.backoffStreak) + assertEquals(listOf("e1" to host.clock + 60_000), scheduler.wakes) + assertEquals(emptyList(), host.sleeps) + assertEquals("end:false", host.progress.last()) + assertEquals(emptyList(), journal.unacknowledged()) + } + + @Test + fun `a transport failure is a network attempt and retries`() { + controller.enqueue(parsed()) + var calls = 0 + host.handler = { _, _ -> if (calls++ == 0) throw IOException("reset") else UploadResponse(200, "", mapOf()) } + run() + assertEquals(listOf("network", null), host.attempts.map { it.errorKind }) + assertEquals("reset", host.attempts[0].errorMessage) + assertEquals(EntryState.COMPLETED, store.load("e1")!!.state) + } + + @Test + fun `a terminal response settles error http with the response and the live bytes`() { + controller.enqueue(parsed()) + respond(400) + run() + val r = outcome() + assertEquals(EventJournal.KIND_ERROR, r.kind) + assertEquals("http", r.errorKind) + assertEquals(400, r.response!!.status) + assertEquals(7, r.bytesSent) + assertEquals(7, store.load("e1")!!.bytesSent) + assertEquals("end:false", host.progress.last()) + } + + @Test + fun `a staged body gone before the run settles error file with no request`() { + controller.enqueue(parsed()) + store.bodyFile(store.load("e1")!!)!!.delete() + run() + assertEquals("file", outcome().errorKind) + assertEquals(emptyList(), host.requests) + } + + @Test + fun `a body that vanishes mid-send settles error file, and the attempt says file`() { + controller.enqueue(parsed()) + host.handler = { _, _ -> + store.bodyFile(store.load("e1")!!)!!.delete() + throw IOException("ENOENT") + } + run() + assertEquals("file", outcome().errorKind) + assertEquals(listOf("file"), host.attempts.map { it.errorKind }) + } + + @Test + fun `a 401 parks the entry and wakes it at expiry`() { + controller.enqueue(parsed(expiresAt = 90_000)) + respond(401) + run() + assertEquals(EntryState.AWAITING_AUTH, store.load("e1")!!.state) + assertEquals(listOf("e1" to 90_000L), scheduler.wakes) + assertEquals(listOf("error"), host.attempts.map { it.outcome }) + assertEquals(emptyList(), journal.unacknowledged()) + } + + @Test + fun `a 401 with newer headers from updateHeaders mid-flight re-issues at once with them`() { + controller.enqueue(parsed()) + var calls = 0 + host.handler = { _, _ -> + if (calls++ == 0) { + controller.updateHeaders(mapOf("Authorization" to "Bearer new")) + UploadResponse(401, "", mapOf()) + } else UploadResponse(200, "", mapOf()) + } + run() + assertEquals(listOf("Bearer old", "Bearer new"), host.requests.map { it.headers["Authorization"] }) + assertEquals(emptyList(), host.sleeps) + assertEquals(EntryState.COMPLETED, store.load("e1")!!.state) + } + + @Test + fun `the expiry wake of a parked entry settles expired and keeps the stored bytes`() { + controller.enqueue(parsed(expiresAt = 20_000)) + store.save(store.load("e1")!!.copy(state = EntryState.AWAITING_AUTH, bytesSent = 3)) + run() + assertEquals(EntryState.AWAITING_AUTH, store.load("e1")!!.state) // not yet expired + host.clock = 20_000 + run() + assertEquals("expired", outcome().errorKind) + assertEquals(3, outcome().bytesSent) + assertEquals(emptyList(), host.requests) + } + + @Test + fun `an entry past expiresAt settles error expired`() { + controller.enqueue(parsed(expiresAt = 5_000)) + run() + assertEquals("expired", outcome().errorKind) + assertEquals(emptyList(), host.requests) + } + + @Test + fun `a paused entry past expiresAt stays paused, and settles expired at resume`() { + controller.enqueue(parsed(expiresAt = 20_000)) + controller.pause() + host.clock = 30_000 + run() // a wake that fires during the pause + assertEquals(EntryState.PAUSED, store.load("e1")!!.state) + assertEquals(emptyList(), journal.unacknowledged()) + controller.resume() + assertEquals(EntryState.QUEUED, store.load("e1")!!.state) + run() + assertEquals("expired", outcome().errorKind) + assertEquals(EntryState.ERROR, store.load("e1")!!.state) + } + + @Test + fun `the connectivity check uses the entry's wifiOnly, else the queue setting at each poll`() { + controller.setWifiOnly(true) + controller.enqueue(parsed(id = "cell", descriptor = desc(dataJson = """{"a":1}""", wifiOnly = false))) + respond(200) + run("cell") + assertEquals(listOf(false), host.wifiChecks) // its own false beats the queue's true + + host.wifiChecks.clear() + controller.enqueue(parsed(id = "follow")) + host.network = ArrayDeque(listOf(Connectivity.NoWifi)) + host.onSleep = { controller.setWifiOnly(false) } // a toggle while it waits + respond(200) + run("follow") + assertEquals(listOf(true, false), host.wifiChecks) + assertEquals(EntryState.COMPLETED, store.load("follow")!!.state) + } + + @Test + fun `a run under a key pause does not start, and other keys run`() { + controller.enqueue(parsed(id = "c", key = "capture")) + controller.enqueue(parsed(id = "n", key = "note")) + controller.pause(listOf("capture")) + respond(200) + run("c") + run("n") + assertEquals(EntryState.PAUSED, store.load("c")!!.state) + assertEquals(EntryState.COMPLETED, store.load("n")!!.state) + assertEquals(1, host.requests.size) + } + + @Test + fun `a key pause while waiting for the network stops the run with no outcome`() { + controller.enqueue(parsed(id = "c", key = "capture")) + host.network = ArrayDeque(listOf(Connectivity.NoInternet)) + host.onSleep = { controller.pause(listOf("capture")) } + run("c") + assertEquals(EntryState.PAUSED, store.load("c")!!.state) + assertEquals(emptyList(), host.requests) + assertEquals(emptyList(), journal.unacknowledged()) + } + + @Test + fun `a failure that lands after pause is not an outcome`() { + controller.enqueue(parsed()) + host.handler = { _, _ -> + controller.pause() + UploadResponse(400, "", mapOf()) + } + run() + assertEquals(EntryState.PAUSED, store.load("e1")!!.state) + assertEquals(emptyList(), journal.unacknowledged()) + } + + @Test + fun `an accepted response that lands after pause settles completed`() { + controller.enqueue(parsed()) + host.handler = { _, _ -> + controller.pause() + UploadResponse(200, "", mapOf()) + } + run() + assertEquals(EntryState.COMPLETED, store.load("e1")!!.state) + } + + @Test + fun `a transient response after cancel stops the run with only the cancel outcome`() { + controller.enqueue(parsed()) + host.handler = { _, _ -> + controller.cancel("e1") + UploadResponse(503, "", mapOf()) + } + run() + assertEquals(EventJournal.KIND_CANCELLED, outcome().kind) + assertEquals("end:false", host.progress.last()) + } + + @Test + fun `a system stop moves the entry back to queued with no outcome and no attempt event`() { + controller.enqueue(parsed()) + host.handler = { _, _ -> throw CancellationException("stopped") } + assertThrows(CancellationException::class.java) { run() } + val e = store.load("e1")!! + assertEquals(EntryState.QUEUED, e.state) + assertNull(e.nextAttemptAt) + assertEquals(emptyList(), host.attempts) + assertEquals(emptyList(), journal.unacknowledged()) + assertEquals("end:false", host.progress.last()) + } + + @Test + fun `a timeout stop takes one more backoff step instead of restarting at once`() { + controller.enqueue(parsed()) + store.save(store.load("e1")!!.copy(backoffStreak = 2)) + host.timeout = true + host.handler = { _, _ -> throw CancellationException("timeout") } + assertThrows(CancellationException::class.java) { run() } + val e = store.load("e1")!! + assertEquals(EntryState.QUEUED, e.state) + assertEquals(3, e.backoffStreak) + assertEquals(host.clock + 4_000, e.nextAttemptAt) // base 1 s * 2^(3-1) + assertEquals(listOf("e1" to host.clock + 4_000), scheduler.wakes) + } + + @Test + fun `a store write that fails mid-run throws with no outcome`() { + controller.enqueue(parsed()) + respond(503, 200) + val dir = store.entryDir("e1") + host.onSleep = { dir.setWritable(false) } // the next attempt's write-ahead fails + try { + assertThrows(IOException::class.java) { run() } + } finally { + dir.setWritable(true) + } + assertEquals(emptyList(), journal.unacknowledged()) + assertEquals(1, host.requests.size) + } + + @Test + fun `an unexpected error settles error unknown`() { + controller.enqueue(parsed()) + host.foregroundError = IllegalStateException("boom") + run() + assertEquals("unknown", outcome().errorKind) + assertEquals("boom", outcome().message) + assertEquals(listOf("end:false"), host.progress) + } + + @Test + fun `a re-run after a lost settle write applies the record and sends nothing`() { + controller.enqueue(parsed()) + store.save(store.load("e1")!!.copy(state = EntryState.RUNNING)) + journal.append(record("00000000-0000-0000-0000-000000000041")) + run() + assertEquals(EntryState.COMPLETED, store.load("e1")!!.state) + assertEquals(emptyList(), host.requests) + } + + @Test + fun `a short remaining backoff is slept out before the run, a long one is left to the wake`() { + controller.enqueue(parsed()) + store.save(store.load("e1")!!.copy(nextAttemptAt = host.clock + 5_000)) + run() + assertEquals(listOf(5_000L), host.sleeps) + assertEquals(EntryState.COMPLETED, store.load("e1")!!.state) + + controller.enqueue(parsed(id = "later")) + store.save(store.load("later")!!.copy(nextAttemptAt = host.clock + 60_000)) + run("later") + assertEquals(EntryState.QUEUED, store.load("later")!!.state) + assertEquals(1, host.requests.size) + } + + @Test + fun `no usable network polls until it returns`() { + controller.enqueue(parsed()) + host.network = ArrayDeque(listOf(Connectivity.NoWifi, Connectivity.NoInternet)) + run() + assertEquals(listOf(EntryRun.CONNECTIVITY_POLL_MS, EntryRun.CONNECTIVITY_POLL_MS), host.sleeps) + assertEquals(EntryState.COMPLETED, store.load("e1")!!.state) + } + + // MARK: - chunked + + private fun chunked(size: Int = 30) { + val src = File(tmp.newFolder(), "video.bin").apply { writeBytes(ByteArray(size) { it.toByte() }) } + controller.enqueue(parsed(descriptor = desc(url = null, method = "PUT", file = src.path, + parts = listOf(part(0, 10), part(10, 20), part(20, 30))))) + } + + @Test + fun `every part accepted settles completed with no status, one attempt per part`() { + chunked() + run() + val e = store.load("e1")!! + assertEquals(EntryState.COMPLETED, e.state) + assertEquals(3, e.attempts) + assertTrue(e.descriptor!!.parts!!.all { it.accepted }) + assertNull(outcome().response!!.status) + val byUrl = host.requests.associateBy { it.url } + assertEquals("20-29", byUrl["https://example.com/part?start=20"]!!.headers["Content-Range"]) + assertEquals(10L, byUrl["https://example.com/part?start=20"]!!.body!!.contentLength()) + assertEquals(listOf(0, 1, 2), host.attempts.map { it.partIndex }.sortedBy { it }) + } + + @Test + fun `a part that fails terminally settles error with its index, and the other parts stop`() { + chunked() + host.handler = { r, _ -> + UploadResponse(if (r.url.endsWith("start=10")) 400 else 200, "", mapOf()) + } + run() + val r = outcome() + assertEquals("http", r.errorKind) + assertEquals(1, r.partIndex) + assertEquals("HTTP 400 on part 1", r.message) + assertEquals("https://example.com/part?start=10", r.url) + val accepted = store.load("e1")!!.descriptor!!.parts!!.map { it.accepted } + assertFalse(accepted[1]) + } + + @Test + fun `a part 503 backs off in that part while the others go on`() { + chunked() + var failed = false + host.handler = { r, _ -> + if (r.url.endsWith("start=0") && !failed) { + failed = true + UploadResponse(503, "", mapOf()) + } else UploadResponse(200, "", mapOf()) + } + run() + assertEquals(EntryState.COMPLETED, store.load("e1")!!.state) + assertEquals(4, host.requests.size) + assertEquals(listOf(1_000L), host.sleeps) + } + + @Test + fun `a part 401 parks the whole entry and keeps the accepted parts`() { + chunked() + host.handler = { r, _ -> + UploadResponse(if (r.url.endsWith("start=20")) 401 else 200, "", mapOf()) + } + run() + val e = store.load("e1")!! + assertEquals(EntryState.AWAITING_AUTH, e.state) + assertEquals(listOf(true, true, false), e.descriptor!!.parts!!.map { it.accepted }) + } + + @Test + fun `a failed chunked settle keeps the accepted bytes, not the in-flight ones`() { + chunked() + host.handler = { r, onProgress -> + onProgress(5) + UploadResponse(if (r.url.endsWith("start=20")) 400 else 200, "", mapOf()) + } + run() + assertEquals(20, outcome().bytesSent) + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/EntryTransitionsTest.kt b/android/src/test/java/ai/openspace/backgroundupload/EntryTransitionsTest.kt new file mode 100644 index 00000000..e6121262 --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/EntryTransitionsTest.kt @@ -0,0 +1,220 @@ +package ai.openspace.backgroundupload + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test + +class EntryTransitionsTest { + + @Test + fun `a settle on a cancelled entry or an older generation is not allowed`() { + for (accepted in listOf(true, false)) { + assertTrue(EntryTransitions.canSettle(entry(state = EntryState.RUNNING), 1, accepted)) + assertTrue(EntryTransitions.canSettle(entry(state = EntryState.AWAITING_AUTH), 1, accepted)) // the expiry wake + assertFalse(EntryTransitions.canSettle(entry(state = EntryState.CANCELLED), 1, accepted)) + assertFalse(EntryTransitions.canSettle(entry(state = EntryState.COMPLETED), 1, accepted)) + assertFalse(EntryTransitions.canSettle(entry(state = EntryState.RUNNING, generation = 2), 1, accepted)) + assertFalse(EntryTransitions.canSettle(null, 1, accepted)) + } + } + + @Test + fun `under pause only an accepted response settles`() { + assertTrue(EntryTransitions.canSettle(entry(state = EntryState.PAUSED), 1, accepted = true)) + assertFalse(EntryTransitions.canSettle(entry(state = EntryState.PAUSED), 1, accepted = false)) + } + + @Test + fun `a live entry with a record of its own generation settles from the newest one`() { + val live = entry(state = EntryState.RUNNING, generation = 2) + val older = record("00000000-0000-0000-0000-0000000000a1", generation = 2, at = 1) + val newest = record("00000000-0000-0000-0000-0000000000a2", kind = EventJournal.KIND_ERROR, generation = 2, at = 2) + val otherLife = record("00000000-0000-0000-0000-0000000000a3", generation = 1, at = 3) + val otherId = record("00000000-0000-0000-0000-0000000000a4", id = "x", generation = 2, at = 4) + val j = EntryTransitions.journaledSettle(live, listOf(older, newest, otherLife, otherId), now = 9)!! + assertEquals(EntryState.ERROR, j.entry.state) + assertEquals(newest.eventId, j.entry.settledEventId) + assertEquals(listOf(older.eventId), j.extraEventIds) + assertNull(EntryTransitions.journaledSettle(live, listOf(otherLife, otherId), 9)) + assertNull(EntryTransitions.journaledSettle(entry(state = EntryState.ERROR, generation = 2), listOf(newest), 9)) + } + + @Test + fun `resume returns to awaiting-auth only when the parked generation is current`() { + val paused = entry(state = EntryState.PAUSED, parkedGeneration = 3) + assertEquals(EntryState.AWAITING_AUTH, EntryTransitions.toResumed(paused, headerGeneration = 3, now = 9).state) + val stale = EntryTransitions.toResumed(paused, headerGeneration = 4, now = 9) + assertEquals(EntryState.QUEUED, stale.state) + assertNull(stale.parkedGeneration) + assertEquals(EntryState.QUEUED, EntryTransitions.toResumed(entry(state = EntryState.PAUSED), 0, 9).state) + } + + @Test + fun `pause keeps the parked generation and clears the wake time`() { + val p = EntryTransitions.toPaused(entry(state = EntryState.AWAITING_AUTH, parkedGeneration = 2, nextAttemptAt = 50), 9) + assertEquals(EntryState.PAUSED, p.state) + assertEquals(2, p.parkedGeneration) + assertNull(p.nextAttemptAt) + } + + @Test + fun `park, release, run, stop, settle`() { + val running = EntryTransitions.toRunning(entry(nextAttemptAt = 5), 9) + assertEquals(EntryState.RUNNING, running.state) + assertNull(running.nextAttemptAt) + + val parked = EntryTransitions.toParked(running.copy(backoffStreak = 4), 7, 10) + assertEquals(EntryState.AWAITING_AUTH, parked.state) + assertEquals(7, parked.parkedGeneration) + assertEquals(0, parked.backoffStreak) + + val released = EntryTransitions.toReleased(running, 99, 6, 11) + assertEquals(EntryState.QUEUED, released.state) + assertEquals(99L, released.nextAttemptAt) + assertEquals(6, released.backoffStreak) + + assertEquals(EntryState.QUEUED, EntryTransitions.toStopped(running, 12).state) + + val settled = EntryTransitions.toSettled(parked, EntryState.ERROR, "ev", 5, 13) + assertEquals("ev", settled.settledEventId) + assertNull(settled.parkedGeneration) + assertEquals(13, settled.updatedAt) + } + + @Test + fun `a short backoff keeps the row running and shows the time, and the next attempt clears it`() { + val waiting = EntryTransitions.toBackingOff(entry(state = EntryState.RUNNING), 9_000, 5) + assertEquals(EntryState.RUNNING, waiting.state) + assertEquals(9_000L, waiting.nextAttemptAt) + assertEquals(9_000.0, waiting.toRow().toMap()["nextAttemptAt"]) + val attempt = EntryTransitions.toAttempt(waiting, "req-2", 6) + assertNull(attempt.nextAttemptAt) + assertEquals(1, attempt.attempts) + assertEquals("req-2", attempt.lastRequestId) + assertFalse(attempt.toRow().toMap().containsKey("nextAttemptAt")) + } +} + +class EnqueueRulesTest { + private val body = desc(dataJson = """{"a":1}""") + private val other = desc(dataJson = """{"a":2}""") + + private fun decide(existing: QueueEntry?, incoming: Descriptor = body, hasRecord: Boolean = true, v9: LegacyManifest? = null) = + EnqueueRules.decide(existing, v9, incoming) { hasRecord } + + @Test + fun `the same-id table`() { + assertEquals(EnqueueRules.Action.Create, decide(null)) + val v9 = LegacyManifest("e1", listOf(part(0, 10)), emptyList()) + assertEquals(EnqueueRules.Action.AdoptV9(v9, 1), decide(null, v9 = v9)) + val legacy = entry(legacy = true, descriptor = null, body = null) + assertEquals(EnqueueRules.Action.Replace, decide(legacy)) + // A legacy row over a v9 manifest: adopt it, one generation up. + assertEquals(EnqueueRules.Action.AdoptV9(v9, 2), decide(legacy, v9 = v9)) + assertEquals(EnqueueRules.Action.ReEmit("ev"), decide(entry(state = EntryState.COMPLETED, settledEventId = "ev"))) + assertEquals(EnqueueRules.Action.Replace, decide(entry(state = EntryState.COMPLETED, settledEventId = "ev"), hasRecord = false)) + EntryState.values().filter { it != EntryState.COMPLETED }.forEach { state -> + assertEquals("$state", EnqueueRules.Action.Resume, decide(entry(state = state))) + } + assertEquals(EnqueueRules.Action.RejectRunning, decide(entry(state = EntryState.RUNNING), other)) + EntryState.values().filter { it != EntryState.RUNNING }.forEach { state -> + assertEquals("$state", EnqueueRules.Action.Replace, decide(entry(state = state), other)) + } + } + + @Test + fun `resume replaces the metadata, keeps the body and accepted parts`() { + val parts = listOf(part(0, 10, accepted = true), part(10, 20)) + val stored = entry( + state = EntryState.AWAITING_AUTH, + descriptor = desc(url = null, method = "PUT", file = "/f", parts = parts), + body = StagedBody(StagedBody.CHUNKED, "blob", null, 20), + attempts = 4, + parkedGeneration = 1, + ) + val incoming = parsed( + descriptor = desc(url = null, method = "PUT", file = "/f", headers = mapOf("Authorization" to "Bearer new"), + parts = parts.map { it.copy(accepted = false) }), + varsJson = """{"n":2}""", + expiresAt = 77, + ) + val next = EnqueueRules.resumed(stored, incoming, paused = false, headerGeneration = 5, now = 9) + assertEquals(EntryState.QUEUED, next.state) + assertEquals(mapOf("Authorization" to "Bearer new"), next.descriptor!!.headers) + assertTrue(next.descriptor!!.parts!![0].accepted) + assertEquals(10, next.bytesSent) + assertEquals(4, next.attempts) + assertEquals(77, next.expiresAt) + assertEquals("""{"n":2}""", next.varsJson) + assertEquals(5, next.headerGeneration) + assertNull(next.parkedGeneration) + assertEquals(1, next.generation) // a live entry keeps its life + } + + @Test + fun `resume takes the incoming wifiOnly, including its removal`() { + val stored = entry(descriptor = desc(dataJson = """{"a":1}""", wifiOnly = true)) + val off = parsed(descriptor = desc(dataJson = """{"a":1}""", wifiOnly = false)) + assertEquals(false, EnqueueRules.resumed(stored, off, false, 0, 9).descriptor!!.wifiOnly) + assertNull(EnqueueRules.resumed(stored, parsed(), false, 0, 9).descriptor!!.wifiOnly) + } + + @Test + fun `resume of a settled entry reopens it with a fresh generation and attempts 0`() { + val settled = entry(state = EntryState.ERROR, settledEventId = "ev", generation = 2, attempts = 3) + val next = EnqueueRules.resumed(settled, parsed(), false, 0, 9) + assertEquals(EntryState.QUEUED, next.state) + assertEquals(3, next.generation) + assertEquals(0, next.attempts) + assertNull(next.settledEventId) + } + + @Test + fun `resume clears a pending backoff so the entry runs now`() { + val waiting = entry(state = EntryState.QUEUED, nextAttemptAt = 99_000).copy(backoffStreak = 6) + val next = EnqueueRules.resumed(waiting, parsed(), false, 0, 9) + assertNull(next.nextAttemptAt) + assertEquals(0, next.backoffStreak) + } + + @Test + fun `resume of a running entry stays running, and under pause becomes paused`() { + assertEquals(EntryState.RUNNING, EnqueueRules.resumed(entry(state = EntryState.RUNNING), parsed(), true, 0, 9).state) + assertEquals(EntryState.PAUSED, EnqueueRules.resumed(entry(state = EntryState.QUEUED), parsed(), true, 0, 9).state) + } + + @Test + fun `resume re-applies the json content type`() { + val next = EnqueueRules.resumed(entry(), parsed(descriptor = desc(dataJson = """{"a":1}""", headers = mapOf())), false, 0, 9) + assertEquals(mapOf("Content-Type" to "application/json"), next.descriptor!!.headers) + } + + @Test + fun `adopting v9 parts carries the flags only for the same parts`() { + val v9 = LegacyManifest("e1", listOf(part(0, 10, accepted = true), part(10, 20)), emptyList()) + assertTrue(EnqueueRules.adoptedParts(v9, listOf(part(0, 10), part(10, 20)))[0].accepted) + assertFalse(EnqueueRules.adoptedParts(v9, listOf(part(0, 20)))[0].accepted) + } + + @Test + fun `a copied body is staged outside the lock only when no worker can change the decision`() { + val json = desc(dataJson = """{"a":2}""") + val create = EnqueueRules.Action.Create + val replace = EnqueueRules.Action.Replace + assertEquals(1, EnqueueRules.preStageGeneration(null, create, json)) + assertEquals(1, EnqueueRules.preStageGeneration(null, create, desc(file = "/f"))) + assertEquals(3, EnqueueRules.preStageGeneration(entry(state = EntryState.ERROR, generation = 2), replace, json)) + assertEquals(2, EnqueueRules.preStageGeneration(entry(state = EntryState.PAUSED), replace, json)) + // A worker can take a queued entry meanwhile, and a running one is the worker's. + assertNull(EnqueueRules.preStageGeneration(entry(state = EntryState.QUEUED), replace, json)) + assertNull(EnqueueRules.preStageGeneration(entry(state = EntryState.RUNNING), replace, json)) + // A chunked move and a bodiless request stage under the lock. + assertNull(EnqueueRules.preStageGeneration(null, create, desc(url = null, file = "/f", parts = listOf(part(0, 10))))) + assertNull(EnqueueRules.preStageGeneration(null, create, desc())) + // Nothing to stage. + assertNull(EnqueueRules.preStageGeneration(entry(), EnqueueRules.Action.Resume, json)) + assertNull(EnqueueRules.preStageGeneration(entry(), EnqueueRules.Action.RejectRunning, json)) + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/EventJournalTest.kt b/android/src/test/java/ai/openspace/backgroundupload/EventJournalTest.kt new file mode 100644 index 00000000..474c570c --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/EventJournalTest.kt @@ -0,0 +1,297 @@ +package ai.openspace.backgroundupload + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertThrows +import org.junit.Assert.assertTrue +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import java.io.File +import java.io.IOException + +class EventJournalTest { + @get:Rule + val tmp = TemporaryFolder() + + private val id1 = "00000000-0000-0000-0000-000000000001" + private val id2 = "00000000-0000-0000-0000-000000000002" + + private val listener = Any() + + @Test + fun `append then read returns the record`() { + val journal = EventJournal(tmp.newFolder()) + journal.drain(listener) + assertEquals(record(id1), journal.append(record(id1))) + assertEquals(listOf(record(id1)), journal.unacknowledged()) + } + + @Test + fun `append starts at 1 delivery with a listener and at 0 without one`() { + val journal = EventJournal(tmp.newFolder()) + assertFalse(journal.isListening()) + assertEquals(0, journal.append(record(id1)).deliveries) + assertEquals(listOf(1), journal.drain(listener).map { it.deliveries }) // the drain delivers it + assertTrue(journal.isListening()) + assertEquals(1, journal.append(record(id2)).deliveries) + } + + @Test + fun `a drain for a torn-down module sets no listener and counts nothing`() { + val journal = EventJournal(tmp.newFolder()) + journal.append(record(id1)) + assertEquals(emptyList(), journal.drain(listener) { false }) + assertFalse(journal.isListening()) + assertEquals(0, journal.find(id1)!!.deliveries) + } + + @Test + fun `listener is the owner of the last drain`() { + val journal = EventJournal(tmp.newFolder()) + assertNull(journal.listener()) + journal.drain(listener) + assertTrue(journal.listener() === listener) + val next = Any() + journal.drain(next) + assertTrue(journal.listener() === next) + } + + @Test + fun `stopListening clears only its own listener`() { + val journal = EventJournal(tmp.newFolder()) + val next = Any() + journal.drain(listener) + journal.drain(next) // a reload: the next module drains before the old one is torn down + journal.stopListening(listener) + assertTrue(journal.isListening()) + journal.stopListening(next) + assertFalse(journal.isListening()) + } + + @Test + fun `redeliver counts a delivery only with a listener`() { + val journal = EventJournal(tmp.newFolder()) + journal.append(record(id1)) + assertNull(journal.redeliver(id1)) + assertEquals(0, journal.find(id1)!!.deliveries) + journal.drain(listener) // 1 + assertEquals(2, journal.redeliver(id1)!!.deliveries) + } + + @Test + fun `ack removes only the acked record and is idempotent`() { + val journal = EventJournal(tmp.newFolder()) + journal.append(record(id1)) + journal.append(record(id2)) + journal.ack(listOf(id1, id1, "unknown")) + assertEquals(listOf(id2), journal.unacknowledged().map { it.eventId }) + } + + @Test + fun `ack ignores ids that are not event ids`() { + val dir = tmp.newFolder() + val outside = File(dir.parentFile, "precious.json").apply { writeText("x") } + EventJournal(dir).ack(listOf("../precious")) + assertTrue(outside.exists()) + } + + @Test + fun `records survive a new journal instance`() { + val dir = tmp.newFolder() + EventJournal(dir).append(record(id1)) + assertEquals(1, EventJournal(dir).unacknowledged().size) + } + + @Test + fun `a body over 1 MB of UTF-8 is cut on a character and flagged`() { + val journal = EventJournal(tmp.newFolder()) + // 2-byte characters, so a char cap would keep 2 MB. + val big = "\u00e9".repeat(BodyCap.SETTLED_MAX_BYTES) + journal.append(record(id1).copy(response = EventJournal.Response(200, null, big, false))) + val read = journal.unacknowledged()[0].response!! + assertTrue(read.bodyTruncated) + assertEquals(BodyCap.SETTLED_MAX_BYTES, read.body!!.toByteArray(Charsets.UTF_8).size) + assertEquals(BodyCap.SETTLED_MAX_BYTES / 2, read.body!!.length) + } + + @Test + fun `a response the stream cap cut stays flagged`() { + val r = EventJournal.Response.of(UploadResponse(500, "partial", mapOf(), truncated = true)) + assertEquals("partial", r.body) + assertTrue(r.bodyTruncated) + assertFalse(EventJournal.Response.of(UploadResponse(500, "whole", mapOf())).bodyTruncated) + } + + @Test + fun `a corrupt file is skipped`() { + val dir = tmp.newFolder() + val journal = EventJournal(dir) + journal.append(record(id1)) + File(dir, "garbage.json").writeText("{not json") + File(dir, "partial.json").writeText("""{"eventId":"x"}""") + assertEquals(listOf(id1), journal.unacknowledged().map { it.eventId }) + } + + @Test + fun `records are ordered by time`() { + val journal = EventJournal(tmp.newFolder()) + journal.append(record(id1, at = 2_000)) + journal.append(record(id2, at = 1_000)) + assertEquals(listOf(id2, id1), journal.unacknowledged().map { it.eventId }) + } + + @Test + fun `append throws when it could not write and keeps nothing`() { + val journal = EventJournal(tmp.newFile()) // a file where the directory should be + assertThrows(IOException::class.java) { journal.append(record(id1)) } + assertEquals(emptyList(), journal.unacknowledged()) + } + + @Test + fun `appendOrHold holds a record it could not write, and every read and ack sees it`() { + val tasks = mutableListOf Unit>>() + val dir = tmp.newFolder() + val journal = EventJournal(dir, retryLater = { delay, task -> tasks += delay to task }) + dir.setWritable(false) + try { + val held = journal.appendOrHold(record(id1)) + assertTrue(journal.isHeld(id1)) + assertEquals(listOf(held), journal.unacknowledged()) + assertEquals(held, journal.find(id1)) + assertEquals(listOf(id1), journal.forEntry("e1").map { it.eventId }) + assertEquals(1, journal.drain(listener).single().deliveries) + // The retry fails while the disk is full, and waits twice as long. + tasks.removeAt(0).also { (delay, task) -> assertEquals(EventJournal.RETRY_MS, delay); task() } + assertEquals(EventJournal.RETRY_MS * 2, tasks.single().first) + } finally { + dir.setWritable(true) + } + tasks.removeAt(0).second() + assertFalse(journal.isHeld(id1)) + assertTrue(File(dir, "$id1.json").exists()) + assertEquals(1, EventJournal(dir).find(id1)!!.deliveries) + assertEquals(emptyList Unit>>(), tasks) + } + + @Test + fun `an ack removes a held record`() { + val dir = tmp.newFolder() + val journal = EventJournal(dir, retryLater = { _, _ -> }) + dir.setWritable(false) + try { + journal.appendOrHold(record(id1)) + } finally { + dir.setWritable(true) + } + journal.ack(listOf(id1)) + assertFalse(journal.isHeld(id1)) + assertEquals(emptyList(), journal.unacknowledged()) + } + + @Test + fun `ackEntry removes every record of one entry`() { + val journal = EventJournal(tmp.newFolder()) + journal.append(record(id1, id = "a")) + journal.append(record(id2, id = "a", generation = 2)) + journal.append(record("00000000-0000-0000-0000-000000000003", id = "b")) + journal.ackEntry("a") + assertEquals(listOf("b"), journal.unacknowledged().map { it.id }) + } + + @Test + fun `prunes the oldest records beyond the cap`() { + val dir = tmp.newFolder() + val journal = EventJournal(dir, maxEntries = 3) + val ids = (1..4).map { "00000000-0000-0000-0000-00000000000$it" } + ids.take(3).forEachIndexed { i, id -> + journal.append(record(id)) + File(dir, "$id.json").setLastModified(1_000L * (i + 1)) + } + journal.append(record(ids[3])) + val left = journal.unacknowledged().map { it.eventId } + assertEquals(3, left.size) + assertFalse(left.contains(ids[0])) + } + + @Test + fun `the prune never deletes a record a row names`() { + val dir = tmp.newFolder() + val journal = EventJournal(dir, maxEntries = 3) + val ids = (1..4).map { "00000000-0000-0000-0000-00000000000$it" } + ids.take(3).forEachIndexed { i, id -> + journal.append(record(id)) + File(dir, "$id.json").setLastModified(1_000L * (i + 1)) + } + // The oldest is named by a row; the next oldest goes instead. + journal.append(record(ids[3])) { setOf(ids[0]) } + assertEquals(setOf(ids[0], ids[2], ids[3]), journal.unacknowledged().map { it.eventId }.toSet()) + } + + @Test + fun `incrementDeliveries persists`() { + val dir = tmp.newFolder() + val journal = EventJournal(dir) + journal.drain(listener) + journal.append(record(id1)) + assertEquals(2, journal.incrementDeliveries(id1)!!.deliveries) + assertEquals(3, journal.incrementDeliveries(id1)!!.deliveries) + assertEquals(3, EventJournal(dir).find(id1)!!.deliveries) + assertNull(journal.incrementDeliveries(id2)) + } + + @Test + fun `forEntry filters by entry id`() { + val journal = EventJournal(tmp.newFolder()) + journal.append(record(id1, id = "a")) + journal.append(record(id2, id = "b")) + assertEquals(listOf(id2), journal.forEntry("b").map { it.eventId }) + } + + @Test + fun `the completed shape is SettledEvent`() { + val map = record(id1).toMap() + assertEquals( + listOf( + "eventId", "id", "key", "vars", "at", "attempts", "requestId", "deliveries", "state", + "bytesSent", "totalBytes", "url", "method", "kind", "response", + ), + map.keys.toList(), + ) + assertEquals(mapOf("n" to 1.0), map["vars"]) + assertEquals(mapOf("status" to 200.0, "headers" to mapOf(), "body" to "ok", "bodyTruncated" to false), map["response"]) + } + + @Test + fun `a chunked completion has a response with no status`() { + val map = record(id1).copy(response = null).toMap() + assertEquals(mapOf("bodyTruncated" to false), map["response"]) + } + + @Test + fun `the error shape nests errorKind, message, response, and partIndex`() { + val map = record(id1, kind = EventJournal.KIND_ERROR).copy( + partIndex = 2, + response = EventJournal.Response(404, null, "gone", false), + ).toMap() + assertEquals(2.0, map["partIndex"]) + assertEquals( + mapOf( + "errorKind" to "http", "message" to "HTTP 400", + "response" to mapOf("status" to 404.0, "body" to "gone", "bodyTruncated" to false), + "partIndex" to 2.0, + ), + map["error"], + ) + assertFalse(map.containsKey("response")) + } + + @Test + fun `the cancelled shape carries the reason`() { + val map = record(id1, kind = EventJournal.KIND_CANCELLED).toMap() + assertEquals("user", map["cancelReason"]) + assertFalse(map.containsKey("error")) + assertFalse(map.containsKey("response")) + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/JsonBridgeTest.kt b/android/src/test/java/ai/openspace/backgroundupload/JsonBridgeTest.kt new file mode 100644 index 00000000..928540e7 --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/JsonBridgeTest.kt @@ -0,0 +1,84 @@ +package ai.openspace.backgroundupload + +import com.facebook.react.bridge.JavaOnlyArray +import com.facebook.react.bridge.JavaOnlyMap +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Assert.assertThrows +import org.junit.Test + +class JsonBridgeTest { + + @Test + fun `integral doubles print as integers, as JSON stringify does`() { + assertEquals("1", JsonBridge.numberText(1.0)) + assertEquals("-3", JsonBridge.numberText(-3.0)) + assertEquals("0", JsonBridge.numberText(-0.0)) + assertEquals("12345678901", JsonBridge.numberText(12_345_678_901.0)) + } + + @Test + fun `fractions and very large magnitudes keep a decimal form`() { + assertEquals("1.5", JsonBridge.numberText(1.5)) + assertEquals("0.1", JsonBridge.numberText(0.1)) + // Above 2^53 a double can not hold every integer, so it stays a double. + assertEquals(1e20, (JsonBridge.parse(JsonBridge.numberText(1e20)) as Double), 0.0) + } + + @Test + fun `JSON text parses to plain values`() { + val value = mapOf("a" to listOf(1.0, "x", true, null, mapOf("b" to 2.5)), "c" to mapOf()) + assertEquals(value, JsonBridge.parse("""{"a":[1,"x",true,null,{"b":2.5}],"c":{}}""")) + assertNull(JsonBridge.parse("null")) + } + + @Test + fun `malformed text throws`() { + assertThrows(Exception::class.java) { JsonBridge.parse("{\"a\":") } + assertThrows(Exception::class.java) { JsonBridge.parse("[1,") } + } + + @Test + fun `bridge maps read into plain values`() { + val map = JavaOnlyMap.of( + "n", 2.0, + "s", "x", + "b", false, + "z", null, + "m", JavaOnlyMap.of("k", 1.0), + "a", JavaOnlyArray.of(1.0, "y"), + ) + assertEquals( + mapOf("n" to 2.0, "s" to "x", "b" to false, "z" to null, "m" to mapOf("k" to 1.0), "a" to listOf(1.0, "y")), + JsonBridge.fromReadable(map), + ) + assertNull(JsonBridge.valueOf(map, "absent")) + } + + @Test + fun `plain values write to the bridge`() { + val out = JsonBridge.toWritableMap( + mapOf("n" to 1.0, "list" to listOf("a", 2.0), "nested" to mapOf("k" to true), "none" to null), + ::JavaOnlyMap, + ::JavaOnlyArray, + ) as JavaOnlyMap + assertEquals(1.0, out.getDouble("n"), 0.0) + assertEquals("a", out.getArray("list")!!.getString(0)) + assertEquals(true, out.getMap("nested")!!.getBoolean("k")) + assertEquals(true, out.isNull("none")) + } + + @Test + fun `isJson accepts one strict JSON value of any kind`() { + listOf("null", "1", "-0.5e3", "\"s\"", "true", "[]", "{}", """{"a":[1,null,{"b":"c"}]}""", " {\"a\":1} ").forEach { + assertEquals(it, true, JsonBridge.isJson(it)) + } + } + + @Test + fun `isJson rejects lenient and malformed text`() { + listOf("", " ", "{a:1}", "{'a':1}", "[1,]", "{\"a\":1} x", "undefined", "NaN", "{\"a\":1}{}").forEach { + assertEquals(it, false, JsonBridge.isJson(it)) + } + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/LegacyImportTest.kt b/android/src/test/java/ai/openspace/backgroundupload/LegacyImportTest.kt new file mode 100644 index 00000000..1386c2cb --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/LegacyImportTest.kt @@ -0,0 +1,114 @@ +package ai.openspace.backgroundupload + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import java.io.File + +class LegacyImportTest { + @get:Rule + val tmp = TemporaryFolder() + + private fun v9(dir: File, eventId: String, uploadId: String, type: String, timestamp: Long) = + File(dir, "$eventId.json").writeText( + """{"eventId":"$eventId","uploadId":"$uploadId","type":"$type","timestamp":$timestamp,"responseCode":200}""", + ) + + @Test + fun `each v9 id becomes one legacy row with its newest outcome`() { + val v9Dir = tmp.newFolder() + v9(v9Dir, "a", "up-1", "completed", 100) + v9(v9Dir, "b", "up-2", "error", 200) + v9(v9Dir, "c", "up-3", "cancelled", 300) + v9(v9Dir, "d", "up-2", "completed", 250) // newer for up-2 + File(v9Dir, "bad.json").writeText("{not json") + val store = QueueStore(tmp.newFolder(), RequestIndex()) + + assertTrue(LegacyImport.import(v9Dir, store)) + + val rows = store.all().associateBy { it.id } + assertEquals(setOf("up-1", "up-2", "up-3"), rows.keys) + assertEquals(EntryState.COMPLETED, rows["up-1"]!!.state) + assertEquals(EntryState.COMPLETED, rows["up-2"]!!.state) + assertEquals(EntryState.CANCELLED, rows["up-3"]!!.state) + val row = rows["up-2"]!! + assertEquals("legacy", row.key) + assertEquals("null", row.varsJson) + assertTrue(row.legacy) + assertEquals(0, row.attempts) + assertEquals(250, row.updatedAt) + assertNull(row.descriptor) + assertEquals(0, v9Dir.list()!!.size) // every v9 file is gone + } + + @Test + fun `a second run imports nothing`() { + val v9Dir = tmp.newFolder() + val store = QueueStore(tmp.newFolder(), RequestIndex()) + v9(v9Dir, "a", "up-1", "completed", 100) + LegacyImport.import(v9Dir, store) + store.remove("up-1") + assertTrue(LegacyImport.import(v9Dir, store)) + assertEquals(emptyList(), store.all()) + } + + @Test + fun `an id that a v10 entry owns is left alone`() { + val v9Dir = tmp.newFolder() + val store = QueueStore(tmp.newFolder(), RequestIndex()) + store.save(entry(id = "up-1")) + v9(v9Dir, "a", "up-1", "error", 100) + LegacyImport.import(v9Dir, store) + assertEquals("note", store.load("up-1")!!.key) + } + + @Test + fun `a failed save keeps the v9 file and reports incomplete`() { + val v9Dir = tmp.newFolder() + v9(v9Dir, "a", "up-1", "error", 100) + val broken = QueueStore(tmp.newFile(), RequestIndex()) // a file where the directory should be + assertEquals(false, LegacyImport.import(v9Dir, broken)) + assertTrue(File(v9Dir, "a.json").exists()) + } + + @Test + fun `runOnce imports, then writes the marker, and a second launch does nothing`() { + val v9Dir = tmp.newFolder() + val marker = File(tmp.newFolder(), LegacyImport.MARKER) + val store = QueueStore(tmp.newFolder(), RequestIndex()) + v9(v9Dir, "a", "up-1", "completed", 100) + assertTrue(LegacyImport.runOnce(marker, v9Dir, store)) + assertTrue(marker.exists()) + assertEquals(listOf("up-1"), store.all().map { it.id }) + // A v9 file that shows up later is not imported: the marker says done. + v9(v9Dir, "b", "up-2", "error", 200) + store.remove("up-1") + assertFalse(LegacyImport.runOnce(marker, v9Dir, store)) + assertEquals(emptyList(), store.all()) + assertTrue(File(v9Dir, "b.json").exists()) + } + + @Test + fun `runOnce with a failed row save writes no marker, so the next launch tries again`() { + val v9Dir = tmp.newFolder() + val marker = File(tmp.newFolder(), LegacyImport.MARKER) + v9(v9Dir, "a", "up-1", "error", 100) + val broken = QueueStore(tmp.newFile(), RequestIndex()) // a file where the directory should be + assertTrue(LegacyImport.runOnce(marker, v9Dir, broken)) // it ran, so v9 work is cancelled + assertFalse(marker.exists()) + val store = QueueStore(tmp.newFolder(), RequestIndex()) + assertTrue(LegacyImport.runOnce(marker, v9Dir, store)) + assertTrue(marker.exists()) + assertEquals(listOf("up-1"), store.all().map { it.id }) + } + + @Test + fun `an unknown type makes no row`() { + assertNull(LegacyImport.legacyRow(LegacyImport.V9Entry("a", "up", "progress", 1))) + assertNull(LegacyImport.legacyRow(LegacyImport.V9Entry("a", null, "completed", 1))) + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/NotificationConfigTest.kt b/android/src/test/java/ai/openspace/backgroundupload/NotificationConfigTest.kt new file mode 100644 index 00000000..d3545c78 --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/NotificationConfigTest.kt @@ -0,0 +1,53 @@ +package ai.openspace.backgroundupload + +import com.google.gson.Gson +import org.junit.Assert.assertEquals +import org.junit.Test + +class NotificationConfigTest { + private val gson = Gson() + + private val configured = NotificationConfig( + notificationId = "my-id", + notificationTitle = "Backing up…", + notificationTitleNoInternet = "Offline", + notificationTitleNoWifi = "No wifi", + notificationChannel = "my-channel", + ) + + @Test + fun `no stored config yields the defaults`() { + assertEquals(NotificationConfig.DEFAULTS, NotificationConfig.fromJson(null)) + } + + @Test + fun `a configured blob survives a persistence round trip`() { + assertEquals(configured, NotificationConfig.fromJson(gson.toJson(configured))) + } + + @Test + fun `a corrupt blob degrades to the defaults`() { + assertEquals(NotificationConfig.DEFAULTS, NotificationConfig.fromJson("][")) + } + + // A blob from a build with fewer fields must not make the other fields null. + @Test + fun `fields missing from a stored blob fall back individually`() { + val config = NotificationConfig.fromJson("""{"notificationTitle":"Custom"}""") + assertEquals("Custom", config.notificationTitle) + assertEquals(NotificationConfig.DEFAULTS.notificationChannel, config.notificationChannel) + assertEquals(NotificationConfig.DEFAULTS.notificationTitleNoWifi, config.notificationTitleNoWifi) + assertEquals( + NotificationConfig.DEFAULTS.notificationTitleNoInternet, + config.notificationTitleNoInternet, + ) + } + + // This is the same derivation that v8 applied to the per-upload option. Thus + // an app that gives its old notificationId to configure() keeps the same + // system notification. + @Test + fun `the system notification id derives from the configured string`() { + assertEquals("my-id".hashCode(), configured.systemNotificationId) + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/QueueControllerTest.kt b/android/src/test/java/ai/openspace/backgroundupload/QueueControllerTest.kt new file mode 100644 index 00000000..c428fa09 --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/QueueControllerTest.kt @@ -0,0 +1,884 @@ +package ai.openspace.backgroundupload + +import org.junit.Assert.assertArrayEquals +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNotNull +import org.junit.Assert.assertNull +import org.junit.Assert.assertThrows +import org.junit.Assert.assertTrue +import org.junit.Before +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import java.io.File + +class QueueControllerTest { + @get:Rule + val tmp = TemporaryFolder() + + private lateinit var root: File + private lateinit var store: QueueStore + private lateinit var journal: EventJournal + private lateinit var settings: QueueSettingsStore + private val events = RecordingEvents() + private val scheduler = FakeScheduler() + private val running = mutableSetOf() + private var now = 10_000L + private lateinit var controller: QueueController + private val listener = Any() + + @Before + fun setUp() { + root = tmp.newFolder("queue") + store = QueueStore(root, RequestIndex()) + journal = EventJournal(tmp.newFolder("journal"), retryLater = { _, _ -> }) + settings = QueueSettingsStore(File(root, "settings.json")) + controller = QueueController(store, journal, settings, events, scheduler, { it in running }, { now }) + journal.drain(listener) // JS is subscribed + } + + private fun source(name: String, size: Int) = File(tmp.newFolder(), name).apply { writeBytes(ByteArray(size) { it.toByte() }) } + + private fun dirFiles(id: String = "e1") = store.entryDir(id).list()!!.toSet() + + // MARK: - enqueue + + @Test + fun `create stages the json body, persists, schedules, then emits`() { + assertEquals("e1", controller.enqueue(parsed())) + val e = store.load("e1")!! + assertEquals(EntryState.QUEUED, e.state) + assertEquals(1, e.generation) + assertEquals("""{"a":1}""", File(store.entryDir("e1"), e.body!!.fileName!!).readText()) + assertEquals("application/json", e.descriptor!!.headers["Content-Type"]) + assertEquals(setOf("entry.json", "body-1.json"), dirFiles()) + assertEquals(listOf("e1"), scheduler.scheduled) + assertEquals(listOf("state:e1:queued"), events.log) + } + + @Test + fun `create while paused is paused and not scheduled`() { + controller.pause() + controller.enqueue(parsed()) + assertEquals(EntryState.PAUSED, store.load("e1")!!.state) + assertEquals(emptyList(), scheduler.scheduled) + } + + @Test + fun `a file body is copied, so the caller may delete its source`() { + val src = source("photo.jpg", 100) + controller.enqueue(parsed(descriptor = desc(file = src.path))) + src.delete() + val e = store.load("e1")!! + assertEquals(100, File(store.entryDir("e1"), e.body!!.fileName!!).length()) + assertEquals(100, e.totalBytes) + } + + @Test + fun `a missing file rejects E_FILE_MISSING and persists nothing`() { + val e = assertThrows(QueueException::class.java) { controller.enqueue(parsed(descriptor = desc(file = "/nope.bin"))) } + assertEquals(QueueException.E_FILE_MISSING, e.code) + assertNull(store.load("e1")) + assertEquals(emptyList(), events.log) + } + + @Test + fun `same body on a queued entry resumes with the new headers, vars, and expiry`() { + controller.enqueue(parsed()) + val bodyFile = store.load("e1")!!.body!!.fileName + controller.enqueue(parsed(descriptor = desc(dataJson = """{"a":1}""", headers = mapOf("Authorization" to "Bearer new")), varsJson = """{"n":2}""", expiresAt = 5)) + val e = store.load("e1")!! + assertEquals("Bearer new", e.descriptor!!.headers["Authorization"]) + assertEquals("""{"n":2}""", e.varsJson) + assertEquals(5, e.expiresAt) + assertEquals(1, e.generation) + assertEquals(bodyFile, e.body!!.fileName) + } + + @Test + fun `same body on a running entry stays running and is not scheduled again`() { + store.save(entry(state = EntryState.RUNNING)) + controller.enqueue(parsed(descriptor = desc(dataJson = """{"a":1}""", headers = mapOf("Authorization" to "Bearer new")))) + val e = store.load("e1")!! + assertEquals(EntryState.RUNNING, e.state) + assertEquals("Bearer new", e.descriptor!!.headers["Authorization"]) + assertEquals(emptyList(), scheduler.scheduled) + } + + @Test + fun `a different body on a running entry rejects E_RUNNING and changes nothing`() { + store.save(entry(state = EntryState.RUNNING)) + val e = assertThrows(QueueException::class.java) { controller.enqueue(parsed(descriptor = desc(dataJson = """{"a":2}"""))) } + assertEquals(QueueException.E_RUNNING, e.code) + assertEquals(entry(state = EntryState.RUNNING), store.load("e1")) + } + + @Test + fun `a different body on a queued entry with a worker sleeping out a short backoff is accepted`() { + // The gate is held, but the entry is queued. The contract: queued is not running. + controller.enqueue(parsed()) + running += "e1" + controller.enqueue(parsed(descriptor = desc(dataJson = """{"a":2}"""))) + assertEquals("""{"a":2}""", store.load("e1")!!.descriptor!!.dataJson) + } + + @Test + fun `a different body on an error entry replaces it and reopens it`() { + controller.enqueue(parsed()) + store.save(store.load("e1")!!.copy(state = EntryState.ERROR, attempts = 3, settledEventId = "ev")) + controller.enqueue(parsed(descriptor = desc(dataJson = """{"a":2}"""))) + val e = store.load("e1")!! + assertEquals(EntryState.QUEUED, e.state) + assertEquals(2, e.generation) + assertEquals(0, e.attempts) + assertNull(e.settledEventId) + assertEquals("""{"a":2}""", File(store.entryDir("e1"), "body-2.json").readText()) + assertEquals(setOf("entry.json", "body-2.json"), dirFiles()) // the old body is pruned + } + + @Test + fun `same body on a completed unacked entry re-emits with one more delivery and does not re-run`() { + controller.enqueue(parsed()) + val eventId = "00000000-0000-0000-0000-00000000000a" + journal.append(record(eventId)) + store.save(store.load("e1")!!.copy(state = EntryState.COMPLETED, settledEventId = eventId)) + scheduler.scheduled.clear() + events.log.clear() + controller.enqueue(parsed()) + assertEquals(listOf("settled:e1:completed"), events.log) + assertEquals(2, events.records.single().deliveries) + assertEquals(2, journal.find(eventId)!!.deliveries) + assertEquals(EntryState.COMPLETED, store.load("e1")!!.state) + assertEquals(emptyList(), scheduler.scheduled) + } + + @Test + fun `same body on a cancelled unacked entry gets a fresh generation, and the old ack forgets nothing`() { + controller.enqueue(parsed()) + controller.cancel("e1") + val cancelled = journal.unacknowledged().single() + controller.enqueue(parsed()) + val e = store.load("e1")!! + assertEquals(EntryState.QUEUED, e.state) + assertEquals(2, e.generation) + controller.ack(listOf(cancelled.eventId)) + assertNotNull(store.load("e1")) + } + + @Test + fun `same body on an error entry reopens it with attempts 0`() { + controller.enqueue(parsed()) + store.save(store.load("e1")!!.copy(state = EntryState.ERROR, attempts = 3, settledEventId = "ev")) + controller.enqueue(parsed(expiresAt = FAR_FUTURE + 1)) + val e = store.load("e1")!! + assertEquals(EntryState.QUEUED, e.state) + assertEquals(2, e.generation) + assertEquals(0, e.attempts) // attempts count the current generation + } + + @Test + fun `same body on a live entry keeps its attempts`() { + controller.enqueue(parsed()) + store.save(store.load("e1")!!.copy(attempts = 3)) + controller.enqueue(parsed(expiresAt = FAR_FUTURE + 1)) + assertEquals(3, store.load("e1")!!.attempts) + } + + @Test + fun `a same-id resume of a queued entry waiting out a backoff clears it and runs now`() { + controller.enqueue(parsed()) + store.save(store.load("e1")!!.copy(nextAttemptAt = now + 3_600_000, backoffStreak = 9)) + scheduler.scheduled.clear() + controller.enqueue(parsed()) + val e = store.load("e1")!! + assertNull(e.nextAttemptAt) + assertEquals(0, e.backoffStreak) + assertEquals(listOf("e1"), scheduler.scheduled) + assertEquals(emptyList>(), scheduler.wakes) + assertFalse(events.rows.last().toMap().containsKey("nextAttemptAt")) + } + + @Test + fun `a completed unacked re-emit with no listener yet waits for the drain`() { + controller.enqueue(parsed()) + val eventId = "00000000-0000-0000-0000-00000000001a" + journal.append(record(eventId)) + store.save(store.load("e1")!!.copy(state = EntryState.COMPLETED, settledEventId = eventId)) + journal.stopListening(listener) + events.log.clear() + controller.enqueue(parsed()) + assertEquals(emptyList(), events.log) + assertEquals(1, journal.find(eventId)!!.deliveries) + assertEquals(2, controller.unacknowledged(listener).single().deliveries) + } + + @Test + fun `enqueue over a legacy row replaces it`() { + store.save(LegacyImport.legacyRow(LegacyImport.V9Entry("x", "e1", "completed", 5))!!) + controller.enqueue(parsed()) + val e = store.load("e1")!! + assertFalse(e.legacy) + assertEquals(2, e.generation) + assertEquals(EntryState.QUEUED, e.state) + } + + // MARK: - legacy row over a v9 manifest + + @Test + fun `a same-id enqueue over a legacy row with a v9 manifest adopts it one generation up`() { + v9Dir("e1", v9Parts, 20) + store.save(LegacyImport.legacyRow(LegacyImport.V9Entry("x", "e1", "error", 5))!!) + val legacy = store.load("e1")!! + assertEquals(0L to 0L, legacy.bytesSent to legacy.totalBytes) // legacy rows report 0/0 bytes + controller.enqueue(parsed(descriptor = desc(url = null, method = "PUT", file = "/gone.bin", parts = listOf(part(0, 10), part(10, 20))))) + val e = store.load("e1")!! + assertFalse(e.legacy) + assertEquals(2, e.generation) + assertEquals(5, e.createdAt) + assertTrue(e.descriptor!!.parts!![0].accepted) // the v9 progress is kept + assertFalse(e.descriptor!!.parts!![1].accepted) + assertEquals(10, e.bytesSent) + assertEquals(setOf("entry.json", "blob"), dirFiles()) // the manifest is pruned + } + + @Test + fun `a same-id enqueue over a legacy row with no manifest replaces it`() { + store.save(LegacyImport.legacyRow(LegacyImport.V9Entry("x", "e1", "error", 5))!!) + controller.enqueue(parsed()) + val e = store.load("e1")!! + assertEquals(2, e.generation) + assertEquals(0, e.bytesSent) + } + + // MARK: - chunked replace (a present file wins over the old blob) + + private fun chunkedErrorEntry(): File { + val first = source("first.bin", 20) + controller.enqueue(parsed(descriptor = desc(url = null, method = "PUT", file = first.path, parts = listOf(part(0, 10), part(10, 20))))) + store.save(store.load("e1")!!.copy(state = EntryState.ERROR, settledEventId = "ev")) + return File(store.entryDir("e1"), "blob") + } + + @Test + fun `a different-parts replace with a present file uploads that file, not the old blob`() { + chunkedErrorEntry() + val bytes = ByteArray(20) { (it * 3).toByte() } + val second = File(tmp.newFolder(), "second.bin").apply { writeBytes(bytes) } + controller.enqueue(parsed(descriptor = desc(url = null, method = "PUT", file = second.path, parts = listOf(part(0, 20))))) + val e = store.load("e1")!! + assertEquals(2, e.generation) + assertEquals("blob-2", e.body!!.fileName) + assertArrayEquals(bytes, store.bodyFile(e)!!.readBytes()) + assertEquals(setOf("entry.json", "blob-2"), dirFiles()) // the old blob is pruned + } + + @Test + fun `a different-parts replace with a present file of another size is accepted`() { + chunkedErrorEntry() + val second = source("second.bin", 30) + controller.enqueue(parsed(descriptor = desc(url = null, method = "PUT", file = second.path, parts = listOf(part(0, 30))))) + assertEquals(30, store.load("e1")!!.totalBytes) + } + + @Test + fun `a different-parts replace whose file was moved away runs over the old blob`() { + chunkedErrorEntry() + controller.enqueue(parsed(descriptor = desc(url = null, method = "PUT", file = "/moved.bin", parts = listOf(part(0, 20))))) + val e = store.load("e1")!! + assertEquals("blob", e.body!!.fileName) + assertEquals(setOf("entry.json", "blob"), dirFiles()) + } + + @Test + fun `a replace whose plan does not tile the present file changes nothing`() { + val oldBlob = chunkedErrorEntry() + val second = source("second.bin", 30) + val e = assertThrows(QueueException::class.java) { + controller.enqueue(parsed(descriptor = desc(url = null, method = "PUT", file = second.path, parts = listOf(part(0, 20))))) + } + assertEquals(QueueException.E_INVALID, e.code) + assertTrue(second.exists()) + assertEquals(20, oldBlob.length()) + assertEquals(1, store.load("e1")!!.generation) + } + + // MARK: - staging outside the lock + + @Test + fun `a new file body is copied outside the store lock`() { + val src = source("big.bin", 1000) + var hookRan = false + controller.afterPreStage = { + hookRan = true + assertFalse(Thread.holdsLock(store)) + assertTrue(File(store.entryDir("e1"), "file-1").exists()) + } + controller.enqueue(parsed(descriptor = desc(file = src.path))) + assertTrue(hookRan) + assertEquals("file-1", store.load("e1")!!.body!!.fileName) + assertEquals(setOf("entry.json", "file-1"), dirFiles()) + } + + @Test + fun `a replace over a queued entry stages under the lock`() { + controller.enqueue(parsed()) + var hookRan = false + controller.afterPreStage = { hookRan = true } + controller.enqueue(parsed(descriptor = desc(dataJson = """{"a":2}"""))) + assertFalse(hookRan) + assertEquals(setOf("entry.json", "body-2.json"), dirFiles()) + } + + @Test + fun `a pre-staged body that no longer fits is deleted and staged again under the lock`() { + controller.enqueue(parsed()) + store.save(store.load("e1")!!.copy(state = EntryState.ERROR)) + // Between the staging and the lock, the entry moves on to another generation. + controller.afterPreStage = { store.save(store.load("e1")!!.copy(generation = 5)) } + controller.enqueue(parsed(descriptor = desc(dataJson = """{"a":2}"""))) + val e = store.load("e1")!! + assertEquals(6, e.generation) + assertEquals("body-6.json", e.body!!.fileName) + assertEquals(setOf("entry.json", "body-6.json"), dirFiles()) // body-2.json is gone + } + + @Test + fun `a pre-staged body is deleted when the enqueue rejects`() { + controller.enqueue(parsed()) + store.save(store.load("e1")!!.copy(state = EntryState.ERROR)) + controller.afterPreStage = { store.save(store.load("e1")!!.copy(state = EntryState.RUNNING)) } + val e = assertThrows(QueueException::class.java) { controller.enqueue(parsed(descriptor = desc(dataJson = """{"a":2}"""))) } + assertEquals(QueueException.E_RUNNING, e.code) + assertEquals(setOf("entry.json", "body-1.json"), dirFiles()) + } + + // MARK: - v9 adoption + + private fun v9Dir(id: String, parts: String, blobSize: Int): File { + val dir = store.entryDir(id).apply { mkdirs() } + File(dir, "blob").writeBytes(ByteArray(blobSize)) + File(dir, "manifest.json").writeText( + """{"id":"$id","sourcePath":"${File(dir, "blob").path}","parts":$parts,"accept":[],"expiresAt":1,"wifiOnly":false,"noNotification":false,"createdAt":1}""", + ) + return dir + } + + private val v9Parts = """[{"url":"https://example.com/part?start=0","headers":{},"start":0,"end":10,"accepted":true},""" + + """{"url":"https://example.com/part?start=10","headers":{},"start":10,"end":20,"accepted":false}]""" + + @Test + fun `a same-id enqueue with the same parts adopts the v9 blob and accepted parts`() { + v9Dir("e1", v9Parts, 20) + controller.enqueue(parsed(descriptor = desc(url = null, method = "PUT", file = "/gone.bin", parts = listOf(part(0, 10), part(10, 20))))) + val e = store.load("e1")!! + assertTrue(e.descriptor!!.parts!![0].accepted) + assertFalse(e.descriptor!!.parts!![1].accepted) + assertEquals(10, e.bytesSent) + assertEquals(setOf("entry.json", "blob"), dirFiles()) + } + + @Test + fun `a v9 adoption with the same parts keeps the v9 blob even when the caller's file is present`() { + v9Dir("e1", v9Parts, 20) + val present = source("again.bin", 20) + controller.enqueue(parsed(descriptor = desc(url = null, method = "PUT", file = present.path, parts = listOf(part(0, 10), part(10, 20))))) + assertTrue(present.exists()) + assertTrue(store.load("e1")!!.descriptor!!.parts!![0].accepted) + assertArrayEquals(ByteArray(20), File(store.entryDir("e1"), "blob").readBytes()) + } + + @Test + fun `a v9 adoption with different parts and a present file uploads that file`() { + v9Dir("e1", v9Parts, 20) + val present = source("new.bin", 30) + controller.enqueue(parsed(descriptor = desc(url = null, method = "PUT", file = present.path, parts = listOf(part(0, 30))))) + assertFalse(present.exists()) + val e = store.load("e1")!! + assertFalse(e.descriptor!!.parts!![0].accepted) + assertEquals(30, store.bodyFile(e)!!.length()) + assertEquals(setOf("entry.json", "blob"), dirFiles()) + } + + @Test + fun `a v9 adoption with parts that do not tile rejects E_INVALID and keeps the v9 files`() { + v9Dir("e1", v9Parts, 20) + val e = assertThrows(QueueException::class.java) { + controller.enqueue(parsed(descriptor = desc(url = null, method = "PUT", file = "/gone.bin", parts = listOf(part(0, 30))))) + } + assertEquals(QueueException.E_INVALID, e.code) + assertEquals(setOf("manifest.json", "blob"), dirFiles()) + } + + // MARK: - cancel + + @Test + fun `cancel of a live entry journals, settles cancelled, stops work, emits, and forgets after ack`() { + controller.enqueue(parsed()) + events.log.clear() + controller.cancel("e1") + val record = journal.unacknowledged().single() + assertEquals(EventJournal.KIND_CANCELLED, record.kind) + assertEquals("user", record.cancelReason) + assertEquals("https://example.com/items", record.url) + val e = store.load("e1")!! + assertEquals(EntryState.CANCELLED, e.state) + assertEquals(record.eventId, e.settledEventId) + assertEquals(listOf("e1"), scheduler.cancelled) + assertEquals(listOf("settled:e1:cancelled", "state:e1:cancelled"), events.log) + assertTrue(events.listeners.single() === listener) + controller.ack(listOf(record.eventId)) + assertNull(store.load("e1")) + assertFalse(store.entryDir("e1").exists()) + } + + @Test + fun `cancel of a settled entry forgets it now with its unacked outcomes`() { + controller.enqueue(parsed()) + val eventId = "00000000-0000-0000-0000-00000000000b" + val older = "00000000-0000-0000-0000-00000000001b" + journal.append(record(older, generation = 0)) + journal.append(record(eventId, kind = EventJournal.KIND_ERROR)) + journal.append(record("00000000-0000-0000-0000-00000000002b", id = "other")) + store.save(store.load("e1")!!.copy(state = EntryState.ERROR, settledEventId = eventId)) + events.log.clear() + controller.cancel("e1") + assertNull(store.load("e1")) + assertFalse(store.entryDir("e1").exists()) + assertEquals(emptyList(), events.log) + assertEquals(listOf("other"), journal.unacknowledged().map { it.id }) + } + + @Test + fun `cancel of a live entry whose journal can not write rejects E_STORAGE and changes nothing`() { + controller.enqueue(parsed()) + val before = store.load("e1")!! + events.log.clear() + scheduler.cancelled.clear() + val dir = tmp.root.resolve("journal") + dir.setWritable(false) + val e = try { + assertThrows(QueueException::class.java) { controller.cancel("e1") } + } finally { + dir.setWritable(true) + } + assertEquals(QueueException.E_STORAGE, e.code) + assertEquals(before, store.load("e1")) + assertEquals(emptyList(), journal.unacknowledged()) + assertEquals(emptyList(), events.log) + assertEquals(emptyList(), scheduler.cancelled) + } + + @Test + fun `cancel whose entry save fails still cancels and resolves, and a retry adds no second outcome`() { + controller.enqueue(parsed()) + events.log.clear() + scheduler.cancelled.clear() + val dir = store.entryDir("e1") + dir.setWritable(false) + try { + controller.cancel("e1") // resolves: the journaled record is the cancel + } finally { + dir.setWritable(true) + } + val record = journal.unacknowledged().single() + assertEquals(EventJournal.KIND_CANCELLED, record.kind) + assertEquals(EntryState.QUEUED, store.load("e1")!!.state) // the save was lost + assertEquals(listOf("e1"), scheduler.cancelled) // a running request can not settle again + assertEquals(listOf("settled:e1:cancelled"), events.log) + // JS calls cancel() again: the journaled cancel is applied, not a second one. + events.log.clear() + controller.cancel("e1") + val e = store.load("e1")!! + assertEquals(EntryState.CANCELLED, e.state) + assertEquals(record.eventId, e.settledEventId) + assertEquals(listOf(record.eventId), journal.unacknowledged().map { it.eventId }) + assertEquals(listOf("state:e1:cancelled"), events.log) + } + + @Test + fun `cancel over the journal cap keeps its own record`() { + val small = EventJournal(tmp.newFolder("small"), maxEntries = 1) + val smallController = QueueController(store, small, settings, events, scheduler, { false }, { now }) + val named = "00000000-0000-0000-0000-000000000042" + small.append(record(named, id = "done")) + store.save(entry(id = "done", state = EntryState.ERROR, settledEventId = named)) + smallController.enqueue(parsed()) + smallController.cancel("e1") + val own = store.load("e1")!!.settledEventId!! + assertNotNull(small.find(own)) // without it, the sweep forgets the row with no outcome + assertNotNull(small.find(named)) + } + + @Test + fun `cancel with no listener journals at 0 and does not emit the outcome`() { + controller.enqueue(parsed()) + journal.stopListening(listener) + events.log.clear() + controller.cancel("e1") + assertEquals(listOf("state:e1:cancelled"), events.log) + assertEquals(0, journal.unacknowledged().single().deliveries) + } + + @Test + fun `cancel of an unknown id is a no-op`() { + controller.cancel("nope") + assertEquals(emptyList(), events.log) + } + + // MARK: - pause, resume, wifi, headers + + @Test + fun `pause moves live rows to paused with no outcome, and resume brings them back`() { + store.save(entry(id = "q", state = EntryState.QUEUED)) + store.save(entry(id = "r", state = EntryState.RUNNING)) + store.save(entry(id = "a", state = EntryState.AWAITING_AUTH, parkedGeneration = 0)) + store.save(entry(id = "s", state = EntryState.AWAITING_AUTH, parkedGeneration = 0)) + store.save(entry(id = "x", state = EntryState.ERROR)) + controller.pause() + assertTrue(settings.load().paused) + assertEquals(listOf("paused", "paused", "paused", "paused", "error"), listOf("q", "r", "a", "s", "x").map { store.load(it)!!.state.wire }) + assertEquals(setOf("q", "r", "a", "s"), scheduler.cancelled.toSet()) + assertEquals(emptyList(), journal.unacknowledged()) + + // A header change while paused makes one parked entry's generation stale. + store.save(store.load("s")!!.copy(parkedGeneration = -1)) + controller.resume() + assertFalse(settings.load().paused) + assertEquals(EntryState.QUEUED, store.load("q")!!.state) + assertEquals(EntryState.QUEUED, store.load("r")!!.state) + assertEquals(EntryState.AWAITING_AUTH, store.load("a")!!.state) + assertEquals(EntryState.QUEUED, store.load("s")!!.state) + assertTrue(scheduler.scheduled.containsAll(listOf("q", "r", "s"))) + assertEquals(listOf("a" to FAR_FUTURE), scheduler.wakes) // the parked one waits for its expiry + } + + @Test + fun `a key pause moves only that key's live rows, one state event each, and no outcome`() { + store.save(entry(id = "c1", key = "capture", state = EntryState.QUEUED)) + store.save(entry(id = "c2", key = "capture", state = EntryState.RUNNING)) + store.save(entry(id = "c3", key = "capture", state = EntryState.ERROR)) + store.save(entry(id = "n1", key = "note", state = EntryState.QUEUED)) + controller.pause(listOf("capture")) + assertFalse(settings.load().paused) + assertEquals(setOf("capture"), settings.load().pausedKeys) + assertEquals(listOf("paused", "paused", "error", "queued"), listOf("c1", "c2", "c3", "n1").map { store.load(it)!!.state.wire }) + assertEquals(setOf("c1", "c2"), scheduler.cancelled.toSet()) + assertEquals(listOf("state:c1:paused", "state:c2:paused"), events.log.sorted()) + assertEquals(emptyList(), journal.unacknowledged()) + + events.log.clear() + controller.resume(listOf("capture")) + assertEquals(emptySet(), settings.load().pausedKeys) + assertEquals(listOf("queued", "queued", "error", "queued"), listOf("c1", "c2", "c3", "n1").map { store.load(it)!!.state.wire }) + assertEquals(listOf("state:c1:queued", "state:c2:queued"), events.log.sorted()) + assertTrue(scheduler.scheduled.containsAll(listOf("c1", "c2"))) + } + + @Test + fun `a key resume under the global gate stays paused, and a global resume under a key pause too`() { + store.save(entry(id = "c", key = "capture")) + store.save(entry(id = "n", key = "note")) + controller.pause(listOf("capture")) + controller.pause() + events.log.clear() + controller.resume(listOf("capture")) + assertEquals(EntryState.PAUSED, store.load("c")!!.state) // the gate still pauses it + assertEquals(emptyList(), events.log) + + controller.pause(listOf("capture")) + controller.resume() + assertEquals(EntryState.PAUSED, store.load("c")!!.state) // its key still pauses it + assertEquals(EntryState.QUEUED, store.load("n")!!.state) + assertEquals(listOf("state:n:queued"), events.log) + assertFalse("c" in scheduler.scheduled) + } + + @Test + fun `a key pause of an already paused entry emits nothing`() { + store.save(entry(id = "c", key = "capture")) + controller.pause() + events.log.clear() + controller.pause(listOf("capture")) + assertEquals(emptyList(), events.log) + assertEquals(setOf("capture"), settings.load().pausedKeys) + } + + @Test + fun `an empty key list changes nothing`() { + store.save(entry(id = "c", key = "capture")) + controller.pause(emptyList()) + assertEquals(QueueSettings(), settings.load()) + assertEquals(EntryState.QUEUED, store.load("c")!!.state) + controller.pause(listOf("capture")) + controller.resume(emptyList()) + assertEquals(setOf("capture"), settings.load().pausedKeys) + assertEquals(EntryState.PAUSED, store.load("c")!!.state) + } + + @Test + fun `enqueue under a key pause starts paused, and other keys run`() { + controller.pause(listOf("capture")) + controller.enqueue(parsed(id = "c", key = "capture")) + controller.enqueue(parsed(id = "n", key = "note")) + assertEquals(EntryState.PAUSED, store.load("c")!!.state) + assertEquals(EntryState.QUEUED, store.load("n")!!.state) + assertEquals(listOf("n"), scheduler.scheduled) + } + + @Test + fun `updateHeaders unparks an entry under a key pause to paused`() { + store.save(entry(id = "c", key = "capture", state = EntryState.AWAITING_AUTH, parkedGeneration = 0)) + settings.update { it.copy(pausedKeys = setOf("capture")) } + controller.updateHeaders(mapOf("Authorization" to "Bearer new")) + assertEquals(EntryState.PAUSED, store.load("c")!!.state) + assertFalse("c" in scheduler.scheduled) + } + + @Test + fun `the per-request wifiOnly is persisted with the entry`() { + controller.enqueue(parsed(id = "w", descriptor = desc(dataJson = """{"a":1}""", wifiOnly = true))) + controller.enqueue(parsed(id = "c", descriptor = desc(dataJson = """{"a":1}""", wifiOnly = false))) + controller.enqueue(parsed(id = "f")) + val reread = QueueStore(root, RequestIndex()) + assertEquals(true, reread.load("w")!!.descriptor!!.wifiOnly) + assertEquals(false, reread.load("c")!!.descriptor!!.wifiOnly) + assertNull(reread.load("f")!!.descriptor!!.wifiOnly) + } + + @Test + fun `setWifiOnly persists`() { + controller.setWifiOnly(true) + assertTrue(QueueSettingsStore(File(root, "settings.json")).load().wifiOnly) + } + + @Test + fun `updateHeaders patches every entry, bumps the generation, and requeues the parked`() { + store.save(entry(id = "p", state = EntryState.AWAITING_AUTH, parkedGeneration = 0)) + store.save(entry(id = "x", state = EntryState.ERROR)) + store.save(entry(id = "c", state = EntryState.QUEUED, descriptor = desc(url = null, file = "/f", + parts = listOf(Part("https://p/1", mapOf("authorization" to "stale"), 0, 10))))) + controller.updateHeaders(mapOf("Authorization" to "Bearer new")) + assertEquals(1, settings.load().headerGeneration) + val p = store.load("p")!! + assertEquals(EntryState.QUEUED, p.state) + assertNull(p.parkedGeneration) + assertEquals("Bearer new", p.descriptor!!.headers["Authorization"]) + assertEquals(1, p.headerGeneration) + assertEquals("Bearer new", store.load("x")!!.descriptor!!.headers["Authorization"]) + assertEquals(mapOf("Authorization" to "Bearer new"), store.load("c")!!.descriptor!!.parts!![0].headers) + assertEquals(listOf("p"), scheduler.scheduled) + assertEquals(listOf("state:p:queued"), events.log) + } + + // MARK: - ack and replay + + @Test + fun `ack forgets a completed entry of the current generation only`() { + val current = "00000000-0000-0000-0000-00000000000c" + val old = "00000000-0000-0000-0000-00000000000d" + store.save(entry(state = EntryState.COMPLETED, settledEventId = current, generation = 2)) + journal.append(record(old, generation = 1)) + journal.append(record(current, generation = 2)) + controller.ack(listOf(old, "unknown", "../../x")) + assertNotNull(store.load("e1")) + controller.ack(listOf(current)) + assertNull(store.load("e1")) + assertEquals(listOf("e1"), scheduler.cancelled) + controller.ack(listOf(current)) // idempotent + } + + @Test + fun `ack of an error removes the record and keeps the row`() { + val id = "00000000-0000-0000-0000-00000000000e" + store.save(entry(state = EntryState.ERROR, settledEventId = id)) + journal.append(record(id, kind = EventJournal.KIND_ERROR)) + controller.ack(listOf(id)) + assertNull(journal.find(id)) + assertNotNull(store.load("e1")) + } + + // A settle journaled and emitted its record, but the store write failed: + // the entry is still live at the record's generation. + + @Test + fun `ack of a completed record on a still-running entry settles and forgets it, so the sweep does not re-run it`() { + val id = "00000000-0000-0000-0000-000000000011" + store.save(entry(state = EntryState.RUNNING)) + journal.append(record(id)) + controller.ack(listOf(id)) + assertNull(store.load("e1")) + assertNull(journal.find(id)) + controller.sweep() + assertEquals(emptyList(), scheduler.scheduled) + } + + @Test + fun `ack of an error record on a still-running entry settles it as error and keeps the row`() { + val id = "00000000-0000-0000-0000-000000000012" + store.save(entry(state = EntryState.RUNNING)) + journal.append(record(id, kind = EventJournal.KIND_ERROR)) + controller.ack(listOf(id)) + val e = store.load("e1")!! + assertEquals(EntryState.ERROR, e.state) + assertEquals(id, e.settledEventId) + assertNull(journal.find(id)) + assertEquals(listOf("state:e1:error"), events.log) + controller.sweep() + assertEquals(emptyList(), scheduler.scheduled) + } + + @Test + fun `ack of a record of an older generation does not settle the live entry`() { + val id = "00000000-0000-0000-0000-000000000013" + store.save(entry(state = EntryState.QUEUED, generation = 2)) + journal.append(record(id, generation = 1)) + controller.ack(listOf(id)) + assertEquals(EntryState.QUEUED, store.load("e1")!!.state) + assertNull(journal.find(id)) + } + + @Test + fun `when the ack repair can not save, the record stays for the sweep`() { + val id = "00000000-0000-0000-0000-000000000014" + store.save(entry(state = EntryState.RUNNING)) + journal.append(record(id)) + val dir = store.entryDir("e1") + dir.setWritable(false) + try { + controller.ack(listOf(id)) + } finally { + dir.setWritable(true) + } + assertNotNull(journal.find(id)) + assertEquals(EntryState.RUNNING, store.load("e1")!!.state) + controller.sweep() + assertEquals(EntryState.COMPLETED, store.load("e1")!!.state) + } + + @Test + fun `each replay counts one more delivery`() { + journal.append(record("00000000-0000-0000-0000-00000000000f")) + assertEquals(2, controller.unacknowledged(listener).single().deliveries) + assertEquals(3, controller.unacknowledged(listener).single().deliveries) + } + + // MARK: - boot sweep + + @Test + fun `sweep applies a record that the store transition missed`() { + val id = "00000000-0000-0000-0000-000000000010" + store.save(entry(state = EntryState.RUNNING)) + journal.append(record(id)) + controller.sweep() + val e = store.load("e1")!! + assertEquals(EntryState.COMPLETED, e.state) + assertEquals(id, e.settledEventId) + assertEquals(listOf("state:e1:completed"), events.log) + } + + @Test + fun `sweep applies a cancel whose store save was lost`() { + val id = "00000000-0000-0000-0000-000000000011" + store.save(entry(state = EntryState.QUEUED)) + journal.append(record(id, kind = EventJournal.KIND_CANCELLED)) + controller.sweep() + assertEquals(EntryState.CANCELLED, store.load("e1")!!.state) + assertEquals(emptyList(), scheduler.scheduled) + } + + @Test + fun `sweep queues a running entry with no worker and schedules queued work`() { + store.save(entry(id = "r", state = EntryState.RUNNING)) + store.save(entry(id = "q", state = EntryState.QUEUED)) + store.save(entry(id = "w", state = EntryState.QUEUED, nextAttemptAt = now + 3_600_000)) + store.save(entry(id = "p", state = EntryState.AWAITING_AUTH, expiresAt = now + 50)) + controller.sweep() + assertEquals(EntryState.QUEUED, store.load("r")!!.state) + assertEquals(setOf("r", "q"), scheduler.scheduled.toSet()) + assertEquals(setOf("w" to now + 3_600_000, "p" to now + 50), scheduler.wakes.toSet()) + } + + @Test + fun `sweep skips an entry whose worker runs in this process`() { + store.save(entry(state = EntryState.RUNNING)) + journal.append(record("00000000-0000-0000-0000-000000000012")) + running += "e1" + controller.sweep() + assertEquals(EntryState.RUNNING, store.load("e1")!!.state) + } + + @Test + fun `sweep forgets a completed entry whose record was acked, and acks orphans`() { + store.save(entry(id = "done", state = EntryState.COMPLETED, settledEventId = "gone")) + val own = "00000000-0000-0000-0000-000000000013" + val orphan = "00000000-0000-0000-0000-000000000014" + store.save(entry(id = "c", state = EntryState.CANCELLED, settledEventId = own)) + journal.append(record(own, id = "c", kind = EventJournal.KIND_CANCELLED)) + journal.append(record(orphan, id = "c")) + controller.sweep() + assertNull(store.load("done")) + assertEquals(listOf(own), journal.unacknowledged().map { it.eventId }) + assertNotNull(store.load("c")) + } + + @Test + fun `sweep keeps a completed entry whose record is held in memory`() { + val eventId = "00000000-0000-0000-0000-000000000015" + val dir = tmp.root.resolve("journal") + dir.setWritable(false) + try { + journal.appendOrHold(record(eventId)) + } finally { + dir.setWritable(true) + } + store.save(entry(state = EntryState.COMPLETED, settledEventId = eventId)) + controller.sweep() + assertNotNull(store.load("e1")) + // Its ack still forgets it. + controller.ack(listOf(eventId)) + assertNull(store.load("e1")) + } + + @Test + fun `sweep leaves paused entries alone`() { + controller.pause() + store.save(entry(state = EntryState.PAUSED)) + controller.sweep() + assertEquals(EntryState.PAUSED, store.load("e1")!!.state) + assertEquals(emptyList(), scheduler.scheduled) + } + + @Test + fun `sweep finishes a pause or resume that a process death cut short`() { + // resume() saved the setting, then died before the rows. + store.save(entry(id = "p", state = EntryState.PAUSED)) + controller.sweep() + assertEquals(EntryState.QUEUED, store.load("p")!!.state) + assertEquals(listOf("p"), scheduler.scheduled) + + // pause() saved the setting, then died before the rows. + settings.update { it.copy(paused = true) } + store.save(entry(id = "q", state = EntryState.QUEUED)) + store.save(entry(id = "r", state = EntryState.RUNNING)) + controller.sweep() + assertEquals(EntryState.PAUSED, store.load("q")!!.state) + assertEquals(EntryState.PAUSED, store.load("r")!!.state) + assertEquals(listOf("p"), scheduler.scheduled) + } + + @Test + fun `sweep finishes a key pause or resume that a process death cut short`() { + // pause({ keys }) saved the set, then died before the rows. + settings.update { it.copy(pausedKeys = setOf("capture")) } + store.save(entry(id = "c", key = "capture", state = EntryState.QUEUED)) + store.save(entry(id = "n", key = "note", state = EntryState.QUEUED)) + // resume({ keys }) of "video" saved the set, then died before the rows. + store.save(entry(id = "v", key = "video", state = EntryState.PAUSED)) + controller.sweep() + assertEquals(EntryState.PAUSED, store.load("c")!!.state) + assertEquals(EntryState.QUEUED, store.load("n")!!.state) + assertEquals(EntryState.QUEUED, store.load("v")!!.state) + assertEquals(setOf("n", "v"), scheduler.scheduled.toSet()) + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/QueueEntryTest.kt b/android/src/test/java/ai/openspace/backgroundupload/QueueEntryTest.kt new file mode 100644 index 00000000..c0b286b3 --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/QueueEntryTest.kt @@ -0,0 +1,103 @@ +package ai.openspace.backgroundupload + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test + +class QueueEntryTest { + + private val form = listOf(FormPart("photo", "image/jpeg", null, "/p.jpg", null)) + + @Test + fun `sameBodyAs compares each body kind by content`() { + val json = entry(descriptor = desc(dataJson = "{\"a\":1}")) + assertTrue(json.sameBodyAs(desc(dataJson = "{\"a\":1}", headers = mapOf("New" to "h")))) + assertFalse(json.sameBodyAs(desc(dataJson = "{\"a\":2}"))) + + val multipart = entry(descriptor = desc(form = form)) + assertTrue(multipart.sameBodyAs(desc(form = form))) + assertFalse(multipart.sameBodyAs(desc(form = form.map { it.copy(name = "other") }))) + + val file = entry(descriptor = desc(file = "/a.bin")) + assertTrue(file.sameBodyAs(desc(file = "/a.bin"))) + assertFalse(file.sameBodyAs(desc(file = "/b.bin"))) + + val none = entry(descriptor = desc(method = "DELETE")) + assertTrue(none.sameBodyAs(desc(method = "DELETE"))) + } + + @Test + fun `chunked compares parts and ignores the path`() { + val parts = listOf(part(0, 100), part(100, 250)) + val chunked = entry(descriptor = desc(url = null, method = "PUT", file = "/moved.bin", parts = parts)) + assertTrue(chunked.sameBodyAs(desc(url = null, method = "PUT", file = "/elsewhere.bin", parts = parts.reversed()))) + assertFalse(chunked.sameBodyAs(desc(url = null, method = "PUT", file = "/moved.bin", parts = listOf(part(0, 250))))) + } + + @Test + fun `a kind change, url change, or method change is a different body`() { + val json = entry(descriptor = desc(dataJson = "{}")) + assertFalse(json.sameBodyAs(desc(form = form))) + assertFalse(json.sameBodyAs(desc(url = "https://example.com/other", dataJson = "{}"))) + assertFalse(json.sameBodyAs(desc(method = "PUT", dataJson = "{}"))) + assertFalse(entry(descriptor = null, legacy = true).sameBodyAs(desc(dataJson = "{}"))) + } + + @Test + fun `withHeadersPatched matches names in any case and keeps the patch spelling`() { + val e = entry(descriptor = desc(headers = mapOf("authorization" to "Bearer old", "X-Keep" to "1"))) + val patched = e.withHeadersPatched(mapOf("Authorization" to "Bearer new", "X-Add" to "2"), generation = 3) + assertEquals( + mapOf("X-Keep" to "1", "Authorization" to "Bearer new", "X-Add" to "2"), + patched.descriptor!!.headers, + ) + assertEquals(3, patched.headerGeneration) + } + + @Test + fun `withHeadersPatched replaces a part's own copy of a patched header only`() { + val parts = listOf( + Part("https://p/1", mapOf("AUTHORIZATION" to "Bearer stale", "Content-Range" to "0-99"), 0, 100), + Part("https://p/2", mapOf("Content-Range" to "100-249"), 100, 250), + ) + val e = entry(descriptor = desc(url = null, file = "/f", parts = parts)) + val patched = e.withHeadersPatched(mapOf("Authorization" to "Bearer new"), 1).descriptor!!.parts!! + assertEquals(mapOf("Content-Range" to "0-99", "Authorization" to "Bearer new"), patched[0].headers) + assertEquals(mapOf("Content-Range" to "100-249"), patched[1].headers) + } + + @Test + fun `toRow carries vars as an object and nextAttemptAt only when set`() { + val row = entry().toRow().toMap() + assertEquals(mapOf("n" to 1.0), row["vars"]) + assertEquals("queued", row["state"]) + assertFalse(row.containsKey("nextAttemptAt")) + assertEquals( + setOf("id", "key", "vars", "state", "bytesSent", "totalBytes", "attempts", "updatedAt"), + row.keys, + ) + val waiting = entry(nextAttemptAt = 9_000).toRow().toMap() + assertEquals(9_000.0, waiting["nextAttemptAt"]) + } + + @Test + fun `a malformed vars text reads as null in the row`() { + assertNull(entry().copy(varsJson = "{bad").toRow().vars) + } + + @Test + fun `isLive covers the four live states`() { + val live = EntryState.values().filter { it.isLive }.toSet() + assertEquals(setOf(EntryState.QUEUED, EntryState.RUNNING, EntryState.AWAITING_AUTH, EntryState.PAUSED), live) + } + + @Test + fun `header maps match names without regard to case`() { + assertTrue(HeaderMap.contains(mapOf("Content-Type" to "x"), "content-type")) + assertEquals("x", HeaderMap.get(mapOf("content-type" to "x"), "Content-Type")) + assertEquals(mapOf("B" to "2", "a" to "3"), HeaderMap.merge(mapOf("A" to "1", "B" to "2"), mapOf("a" to "3"))) + assertEquals(mapOf("B" to "2"), HeaderMap.without(mapOf("a" to "1", "B" to "2"), "A")) + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/QueueSettingsTest.kt b/android/src/test/java/ai/openspace/backgroundupload/QueueSettingsTest.kt new file mode 100644 index 00000000..d21668ee --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/QueueSettingsTest.kt @@ -0,0 +1,103 @@ +package ai.openspace.backgroundupload + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import java.io.File + +class QueueSettingsTest { + @get:Rule + val tmp = TemporaryFolder() + + @Test + fun `defaults when there is no file`() { + val s = QueueSettingsStore(File(tmp.newFolder(), "settings.json")).load() + assertEquals(QueueSettings(), s) + assertEquals(RetryDefaults(1_000, 7_200_000, 0.2, listOf(404)), s.retry) + } + + @Test + fun `update persists and reloads in a new instance`() { + val file = File(tmp.newFolder(), "settings.json") + QueueSettingsStore(file).update { it.copy(wifiOnly = true, paused = true) } + val reloaded = QueueSettingsStore(file).load() + assertTrue(reloaded.wifiOnly) + assertTrue(reloaded.paused) + } + + @Test + fun `paused keys persist and reload in a new instance`() { + val file = File(tmp.newFolder(), "settings.json") + QueueSettingsStore(file).update { it.copy(pausedKeys = setOf("capture", "video")) } + assertEquals(setOf("capture", "video"), QueueSettingsStore(file).load().pausedKeys) + } + + @Test + fun `a null paused keys field reads an empty set`() { + val file = File(tmp.newFolder(), "settings.json").apply { writeText("""{"paused":true,"pausedKeys":null}""") } + val s = QueueSettingsStore(file).load() + assertTrue(s.paused) + assertEquals(emptySet(), s.pausedKeys) + } + + @Test + fun `an entry is paused under the gate or its key`() { + assertFalse(QueueSettings().isPaused("note")) + assertTrue(QueueSettings(paused = true).isPaused("note")) + val keyed = QueueSettings(pausedKeys = setOf("capture")) + assertTrue(keyed.isPaused("capture")) + assertFalse(keyed.isPaused("note")) + } + + @Test + fun `the entry's own wifiOnly wins, and absent follows the queue setting`() { + val on = QueueSettings(wifiOnly = true) + val off = QueueSettings(wifiOnly = false) + assertTrue(on.wifiOnlyFor(entry(descriptor = desc()))) + assertFalse(off.wifiOnlyFor(entry(descriptor = desc()))) + assertFalse(on.wifiOnlyFor(entry(descriptor = desc(wifiOnly = false)))) + assertTrue(off.wifiOnlyFor(entry(descriptor = desc(wifiOnly = true)))) + } + + @Test + fun `a corrupt file reads as the defaults`() { + val file = File(tmp.newFolder(), "settings.json").apply { writeText("{not json") } + assertEquals(QueueSettings(), QueueSettingsStore(file).load()) + } + + @Test + fun `a file missing fields keeps the defaults for them`() { + val file = File(tmp.newFolder(), "settings.json").apply { writeText("""{"wifiOnly":true}""") } + val s = QueueSettingsStore(file).load() + assertTrue(s.wifiOnly) + assertFalse(s.paused) + assertEquals(emptySet(), s.pausedKeys) + assertEquals(RetryDefaults(), s.retry) + } + + @Test + fun `header generation increments`() { + val store = QueueSettingsStore(File(tmp.newFolder(), "settings.json")) + assertEquals(1, store.update { it.copy(headerGeneration = it.headerGeneration + 1) }.headerGeneration) + assertEquals(2, store.update { it.copy(headerGeneration = it.headerGeneration + 1) }.headerGeneration) + } + + @Test + fun `a failed write keeps the old value`() { + // A regular file where the directory should be: the write fails. + val notADir = tmp.newFile() + val store = QueueSettingsStore(File(notADir, "settings.json")) + runCatching { store.update { it.copy(paused = true) } } + assertFalse(store.load().paused) + } + + @Test + fun `configure retry fills absent fields with the library defaults`() { + val d = QueueSettingsStore.retryDefaults(mapOf("backoff" to mapOf("baseMs" to 500.0), "terminalHttp" to mapOf("exempt" to listOf()))) + assertEquals(RetryDefaults(baseMs = 500, exempt = emptyList()), d) + assertEquals(RetryDefaults(), QueueSettingsStore.retryDefaults(null)) + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/QueueStoreTest.kt b/android/src/test/java/ai/openspace/backgroundupload/QueueStoreTest.kt new file mode 100644 index 00000000..98d107ba --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/QueueStoreTest.kt @@ -0,0 +1,253 @@ +package ai.openspace.backgroundupload + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNotNull +import org.junit.Assert.assertNull +import org.junit.Assert.assertThrows +import org.junit.Assert.assertTrue +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import java.io.File +import java.io.IOException +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit + +class QueueStoreTest { + @get:Rule + val tmp = TemporaryFolder() + + private fun store(dir: File = tmp.newFolder(), index: RequestIndex = RequestIndex()) = QueueStore(dir, index) + + @Test + fun `save then load round-trips across store instances`() { + val dir = tmp.newFolder() + val e = entry( + descriptor = desc(url = null, method = "PUT", file = "/f", parts = listOf(part(0, 10, accepted = true), part(10, 20))), + body = StagedBody(StagedBody.CHUNKED, "blob", null, 20), + nextAttemptAt = 5, + parkedGeneration = 2, + ) + store(dir).save(e) + // A new instance over the same dir is what a process relaunch looks like. + assertEquals(e, store(dir).load("e1")) + } + + @Test + fun `the state enum is stored by its wire string`() { + val s = store() + s.save(entry(state = EntryState.AWAITING_AUTH)) + assertTrue(File(s.entryDir("e1"), QueueStore.ENTRY_FILE).readText().contains("\"awaiting-auth\"")) + } + + @Test + fun `ids with filesystem-hostile characters round-trip`() { + val s = store() + val id = "a/b:c dü..\\e" + s.save(entry(id = id)) + assertEquals(id, s.load(id)!!.id) + s.remove(id) + assertNull(s.load(id)) + } + + @Test + fun `a corrupt entry reads as absent`() { + val s = store() + s.save(entry()) + File(s.entryDir("e1"), QueueStore.ENTRY_FILE).writeText("{not json") + assertNull(s.load("e1")) + assertEquals(emptyList(), s.all()) + } + + @Test + fun `an entry missing a required field reads as absent`() { + val s = store() + s.entryDir("e1").mkdirs() + File(s.entryDir("e1"), QueueStore.ENTRY_FILE).writeText("""{"id":"e1","key":"k","state":"queued"}""") + assertNull(s.load("e1")) // not legacy, and no descriptor or body + } + + @Test + fun `an older file gets safe defaults`() { + val s = store() + s.entryDir("e1").mkdirs() + File(s.entryDir("e1"), QueueStore.ENTRY_FILE).writeText( + """{"id":"e1","key":"k","state":"queued","descriptor":{"url":"https://x"},"body":{"kind":"none","totalBytes":0}}""", + ) + val e = s.load("e1")!! + assertEquals("null", e.varsJson) + assertEquals("POST", e.descriptor!!.method) + assertEquals(emptyMap(), e.descriptor!!.headers) + assertEquals(1, e.generation) + } + + @Test + fun `all skips v9-only directories and legacyManifest reads them`() { + val s = store() + val dir = s.entryDir("v9").apply { mkdirs() } + File(dir, QueueStore.V9_MANIFEST_FILE).writeText( + """{"id":"v9","sourcePath":"/x/blob","parts":[{"url":"https://p/1","headers":{},"start":0,"end":10,"accepted":true}],""" + + """"accept":[{"status":409}],"expiresAt":99,"wifiOnly":false,"noNotification":true,"createdAt":1}""", + ) + s.save(entry(id = "v10")) + assertEquals(listOf("v10"), s.all().map { it.id }) + val m = s.legacyManifest("v9")!! + assertEquals(listOf(Part("https://p/1", mapOf(), 0, 10, accepted = true)), m.parts) + assertEquals(listOf(UploadOutcome.AcceptRule(409)), m.accept) + assertNull(s.legacyManifest("v10")) + assertNull(s.legacyManifest("nope")) + } + + @Test + fun `compute holds the lock across load, transform, and save`() { + // A module transition and a worker's update race. If the lock did not + // span all three steps, the update could land between load and save and + // be erased. The update must block while the transform runs. + val s = store() + s.save(entry(descriptor = desc(url = null, file = "/f", parts = listOf(part(0, 10), part(10, 20))))) + val inTransform = CountDownLatch(1) + val finish = CountDownLatch(1) + val computing = Thread { + s.compute("e1") { e -> + inTransform.countDown() + finish.await(5, TimeUnit.SECONDS) + e!!.copy(expiresAt = 99_000) + } + }.apply { start() } + assertTrue(inTransform.await(5, TimeUnit.SECONDS)) + val updating = Thread { + s.update("e1") { e -> e.copy(descriptor = e.descriptor!!.copy(parts = ChunkedParts.withAccepted(e.descriptor.parts!!, 0))) } + }.apply { start() } + // Without the lock the update finishes (TERMINATED) while the transform waits. + val deadline = System.currentTimeMillis() + 5_000 + while (updating.state != Thread.State.BLOCKED && updating.state != Thread.State.TERMINATED && + System.currentTimeMillis() < deadline + ) Thread.sleep(5) + assertEquals(Thread.State.BLOCKED, updating.state) + finish.countDown() + computing.join() + updating.join() + val final = s.load("e1")!! + assertEquals(99_000, final.expiresAt) + assertTrue(final.descriptor!!.parts!![0].accepted) + } + + @Test + fun `a throwing transform writes nothing`() { + val s = store() + s.save(entry()) + assertThrows(IllegalStateException::class.java) { s.compute("e1") { throw IllegalStateException("no") } } + assertEquals(entry(), s.load("e1")) + } + + @Test + fun `compute returning the same object or null writes nothing`() { + val s = store() + assertNull(s.compute("e1") { null }) + assertFalse(s.entryDir("e1").exists() && File(s.entryDir("e1"), QueueStore.ENTRY_FILE).exists()) + s.save(entry()) + val file = File(s.entryDir("e1"), QueueStore.ENTRY_FILE) + file.setLastModified(1_000) + s.compute("e1") { it } + assertEquals(1_000, file.lastModified()) + } + + @Test + fun `remove deletes the row and every staged byte`() { + val index = RequestIndex() + val s = store(index = index) + s.save(entry()) + File(s.entryDir("e1"), "body-1.json").writeText("{}") + File(s.entryDir("e1"), "blob").writeText("bytes") + s.remove("e1") + assertNull(s.load("e1")) + assertFalse(s.entryDir("e1").exists()) + assertEquals(emptyList(), index.snapshot()) + } + + @Test + fun `the index follows every save and remove, and loads on start`() { + val dir = tmp.newFolder() + val index = RequestIndex() + val s = store(dir, index) + s.save(entry(id = "a")) + s.save(entry(id = "b", state = EntryState.ERROR)) + assertEquals(listOf("a", "b"), index.snapshot().map { it.id }) + s.remove("a") + assertEquals(listOf("b"), index.snapshot().map { it.id }) + // A process relaunch: a fresh index loaded from disk. + val fresh = RequestIndex() + QueueStore(dir, fresh).loadIndex() + assertEquals(listOf("error"), fresh.snapshot().map { it.state }) + } + + @Test + fun `pruneUnreferenced keeps only the entry file and its body`() { + val s = store() + val e = entry(body = StagedBody(StagedBody.JSON, "body-2.json", null, 2)) + s.save(e) + val dir = s.entryDir("e1") + listOf("body-1.json", "body-2.json", "blob", "manifest.json", "entry.json.tmp").forEach { File(dir, it).writeText("x") } + s.pruneUnreferenced(e) + assertEquals(setOf("entry.json", "body-2.json"), dir.list()!!.toSet()) + } + + // MARK: - crash mid-write + + @Test + fun `crash mid-write (a) a partial entry tmp next to a valid entry`() { + val s = store() + s.save(entry(attempts = 1)) + // The process died while writing the next version: only the tmp is partial. + File(s.entryDir("e1"), "entry.json.tmp").writeText("""{"id":"e1","key":"no""") + assertEquals(1, s.load("e1")!!.attempts) + s.save(entry(attempts = 2)) + assertEquals(2, s.load("e1")!!.attempts) + assertFalse(File(s.entryDir("e1"), "entry.json.tmp").exists()) + } + + @Test + fun `crash mid-write (b) a staged body with no entry is not a row`() { + val s = store() + val dir = s.entryDir("e1").apply { mkdirs() } + File(dir, "body-1.json.tmp").writeText("{\"a\":") + File(dir, "body-1.json").writeText("{\"a\":1}") + assertEquals(emptyList(), s.all()) + assertNull(s.load("e1")) + // The next create saves an entry and prunes what it does not use. + val e = entry(body = StagedBody(StagedBody.JSON, "body-1.json", null, 7)) + s.save(e) + s.pruneUnreferenced(e) + assertEquals(setOf("entry.json", "body-1.json"), dir.list()!!.toSet()) + } + + @Test + fun `crash mid-write (c) a write that throws leaves the old target intact`() { + val target = File(tmp.newFolder(), "entry.json").apply { writeText("old") } + assertThrows(IOException::class.java) { + AtomicFiles.writeAtomically(target) { out -> + out.write("new, half".toByteArray()) + throw IOException("disk full") + } + } + assertEquals("old", target.readText()) + assertFalse(AtomicFiles.tmpFor(target).exists()) + } + + @Test + fun `a save into an unwritable directory throws`() { + val notADir = tmp.newFile() + val s = QueueStore(notADir, RequestIndex()) + assertThrows(IOException::class.java) { s.save(entry()) } + } + + @Test + fun `update is best effort`() { + val s = store() + assertNull(s.update("nope") { it }) + s.save(entry()) + assertNotNull(s.update("e1") { it.copy(attempts = 3) }) + assertEquals(3, s.load("e1")!!.attempts) + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/RetryClassifierTest.kt b/android/src/test/java/ai/openspace/backgroundupload/RetryClassifierTest.kt new file mode 100644 index 00000000..189bf8be --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/RetryClassifierTest.kt @@ -0,0 +1,108 @@ +package ai.openspace.backgroundupload + +import ai.openspace.backgroundupload.RetryClassifier.Verdict +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test +import java.io.IOException +import kotlin.random.Random + +class RetryClassifierTest { + private val defaultExempt = listOf(404) + + private fun classify(code: Int, body: String = "", accept: List = emptyList(), exempt: List = defaultExempt) = + RetryClassifier.classifyResponse(code, body, accept, exempt) + + @Test + fun `the retry table`() { + assertEquals(Verdict.Accepted, classify(200)) + assertEquals(Verdict.Accepted, classify(204)) + assertEquals(Verdict.Accepted, classify(409, "upload already completed", listOf(UploadOutcome.AcceptRule(409, "already completed")))) + assertEquals(Verdict.Terminal("http", "HTTP 409"), classify(409, "conflict", listOf(UploadOutcome.AcceptRule(409, "already completed")))) + assertEquals(Verdict.Auth, classify(401)) + assertEquals(Verdict.Auth, classify(403)) + assertEquals(Verdict.Transient, classify(408)) + assertEquals(Verdict.Transient, classify(429)) + assertEquals(Verdict.Transient, classify(500)) + assertEquals(Verdict.Transient, classify(599)) + assertEquals(Verdict.Transient, classify(404)) // default exempt + assertEquals(Verdict.Terminal("http", "HTTP 400"), classify(400)) + assertEquals(Verdict.Terminal("http", "HTTP 409"), classify(409)) + assertEquals(Verdict.Terminal("http", "HTTP 304"), classify(304)) + assertEquals(Verdict.Terminal("http", "HTTP 101"), classify(101)) + } + + @Test + fun `a chunked part 404 with exempt empty is terminal`() { + assertEquals(Verdict.Terminal("http", "HTTP 404"), classify(404, exempt = emptyList())) + } + + @Test + fun `an auth status stays auth even when exempt lists it`() { + assertEquals(Verdict.Auth, classify(401, exempt = listOf(401))) + } + + @Test + fun `transport failures`() { + assertEquals(Verdict.Transient, RetryClassifier.classifyFailure(IOException("reset"), fileExists = true)) + val file = RetryClassifier.classifyFailure(IOException("ENOENT"), fileExists = false) + assertTrue(file is Verdict.Terminal && file.errorKind == "file") + val other = RetryClassifier.classifyFailure(IllegalArgumentException("bad url"), fileExists = true) + assertEquals(Verdict.Terminal("unknown", "bad url"), other) + } + + @Test + fun `the attempt errorKind of a failure comes from its verdict`() { + assertEquals("network", RetryClassifier.failureKind(RetryClassifier.classifyFailure(IOException(), true))) + assertEquals("file", RetryClassifier.failureKind(RetryClassifier.classifyFailure(IOException(), false))) + assertEquals("unknown", RetryClassifier.failureKind(RetryClassifier.classifyFailure(RuntimeException("boom"), true))) + } + + private val policy = RetryClassifier.Policy(baseMs = 1_000, maxMs = 7_200_000, jitter = 0.0, exempt = defaultExempt) + + @Test + fun `backoff doubles from base and caps at max`() { + assertEquals(1_000, RetryClassifier.backoffMs(policy, 1)) + assertEquals(2_000, RetryClassifier.backoffMs(policy, 2)) + assertEquals(4_000, RetryClassifier.backoffMs(policy, 3)) + assertEquals(7_200_000, RetryClassifier.backoffMs(policy, 14)) + assertEquals(7_200_000, RetryClassifier.backoffMs(policy, 10_000)) + assertEquals(1_000, RetryClassifier.backoffMs(policy, 0)) // defensive + } + + @Test + fun `jitter stays within bounds and under max`() { + val jittered = policy.copy(jitter = 0.2) + val random = Random(42) + repeat(1_000) { + val ms = RetryClassifier.backoffMs(jittered, 3, random) + assertTrue("$ms", ms in 3_200..4_800) + } + repeat(1_000) { + assertTrue(RetryClassifier.backoffMs(jittered, 30, random) <= 7_200_000) + } + } + + @Test + fun `nextAttemptAt clamps to expiresAt`() { + assertEquals(3_000, RetryClassifier.nextAttemptAt(now = 1_000, backoffMs = 2_000, expiresAt = 10_000)) + assertEquals(10_000, RetryClassifier.nextAttemptAt(now = 1_000, backoffMs = 7_200_000, expiresAt = 10_000)) + } + + @Test + fun `expiry is inclusive of the deadline`() { + assertFalse(RetryClassifier.isExpired(4_999, 5_000)) + assertTrue(RetryClassifier.isExpired(5_000, 5_000)) + } + + @Test + fun `policy overrides field by field`() { + val defaults = RetryDefaults() + assertEquals(RetryClassifier.Policy(1_000, 7_200_000, 0.2, listOf(404)), RetryClassifier.policy(defaults, null)) + assertEquals( + RetryClassifier.Policy(50, 7_200_000, 0.2, emptyList()), + RetryClassifier.policy(defaults, RetryOverride(baseMs = 50, maxMs = null, jitter = null, exempt = emptyList())), + ) + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/SmallPartsTest.kt b/android/src/test/java/ai/openspace/backgroundupload/SmallPartsTest.kt new file mode 100644 index 00000000..a5bfda74 --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/SmallPartsTest.kt @@ -0,0 +1,176 @@ +package ai.openspace.backgroundupload + +import androidx.work.WorkInfo.State.BLOCKED +import androidx.work.WorkInfo.State.CANCELLED +import androidx.work.WorkInfo.State.ENQUEUED +import androidx.work.WorkInfo.State.FAILED +import androidx.work.WorkInfo.State.RUNNING +import androidx.work.WorkInfo.State.SUCCEEDED +import kotlinx.coroutines.runBlocking +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test + +class AttemptEventTest { + private val response = UploadResponse(401, "x".repeat(5_000), mapOf("a" to "b")) + + @Test + fun `the body is cut at 4 KB of UTF-8 and flagged`() { + val e = AttemptEvent.ofResponse(entry(attempts = 2), "r1", "https://x", null, response, accepted = false, at = 7) + assertEquals(BodyCap.ATTEMPT_MAX_BYTES, e.responseBody!!.length) + assertEquals(true, e.responseBodyTruncated) + assertEquals(2, e.attempt) + // 3-byte characters: the cut backs off to a whole character. + val wide = AttemptEvent.ofResponse(entry(), "r1", "https://x", null, response.copy(body = "\u20ac".repeat(2_000)), false, 7) + assertEquals(BodyCap.ATTEMPT_MAX_BYTES / 3, wide.responseBody!!.length) + } + + @Test + fun `a response the stream cap cut is flagged even under 4 KB`() { + val e = AttemptEvent.ofResponse(entry(), "r1", "https://x", null, response.copy(body = "short", truncated = true), false, 7) + assertEquals("short", e.responseBody) + assertEquals(true, e.responseBodyTruncated) + } + + @Test + fun `outcome is completed only when accepted`() { + val rejected = AttemptEvent.ofResponse(entry(), "r1", "https://x", null, response, accepted = false, at = 7) + assertEquals("error", rejected.outcome) + assertEquals(401, rejected.httpCode) + assertEquals("http", rejected.errorKind) + val ok = AttemptEvent.ofResponse(entry(), "r1", "https://x", 3, response.copy(code = 200, body = "ok"), accepted = true, at = 7) + assertEquals("completed", ok.outcome) + assertNull(ok.errorKind) + assertEquals(3.0, ok.toMap()["partIndex"]) + } + + @Test + fun `partIndex and response fields are optional in the map`() { + val failure = AttemptEvent.ofFailure(entry(), "r1", "https://x", null, "network", "reset", 7).toMap() + assertEquals( + setOf("id", "key", "requestId", "attempt", "url", "method", "outcome", "errorKind", "errorMessage", "at"), + failure.keys, + ) + } +} + +class ProgressThrottleTest { + private var now = 0L + private val emitted = mutableListOf() + private val throttle = ProgressThrottle({ now }) { _, sent, _ -> emitted += sent } + + @Test + fun `the first offer emits, then at most one per second in the foreground`() { + throttle.offer("a", 1, 10, foreground = true) + now = 500 + throttle.offer("a", 2, 10, foreground = true) + assertEquals(listOf(1L), emitted) + now = 1_000 + throttle.offer("a", 3, 10, foreground = true) + assertEquals(listOf(1L, 3L), emitted) + } + + @Test + fun `the background interval is 10 minutes`() { + throttle.offer("a", 1, 10, foreground = false) + now = 599_999 + throttle.offer("a", 2, 10, foreground = false) + assertEquals(listOf(1L), emitted) + now = 600_000 + throttle.offer("a", 3, 10, foreground = false) + assertEquals(listOf(1L, 3L), emitted) + } + + @Test + fun `flush sends the held value once`() { + throttle.offer("a", 1, 10, foreground = true) + throttle.offer("a", 2, 10, foreground = true) + throttle.flush("a") + throttle.flush("a") + assertEquals(listOf(1L, 2L), emitted) + } + + @Test + fun `ids are independent, and drop forgets an id`() { + throttle.offer("a", 1, 10, foreground = true) + throttle.offer("b", 5, 10, foreground = true) + assertEquals(listOf(1L, 5L), emitted) + throttle.offer("a", 2, 10, foreground = true) + throttle.drop("a") + throttle.flush("a") + assertEquals(listOf(1L, 5L), emitted) + } +} + +class RequestIndexTest { + @Test + fun `put, remove, and snapshot order`() { + val index = RequestIndex() + index.put(entry(id = "b", createdAt = 2).toRow()) + index.put(entry(id = "a", createdAt = 2).toRow()) + index.put(entry(id = "c", createdAt = 1).toRow()) + assertEquals(listOf("c", "a", "b"), index.snapshot().map { it.id }) + index.remove("a") + assertEquals(listOf("c", "b"), index.snapshot().map { it.id }) + } + + @Test + fun `setBytes on a missing id is a no-op`() { + val index = RequestIndex() + index.setBytes("nope", 5) + assertEquals(emptyList(), index.snapshot()) + } + + @Test + fun `a save of a running entry does not move bytes backwards`() { + val index = RequestIndex() + val running = entry(state = EntryState.RUNNING, body = StagedBody(StagedBody.FILE, "f", null, 100)) + index.put(running.toRow()) + index.setBytes("e1", 60) + index.put(running.copy(attempts = 2).toRow()) + assertEquals(60, index.snapshot().single().bytesSent) + index.put(running.copy(state = EntryState.QUEUED).toRow()) + assertEquals(0, index.snapshot().single().bytesSent) + } +} + +class SchedulerTest { + @Test + fun `initialDelayMs is the time left, never negative`() { + assertEquals(0, WorkManagerScheduler.initialDelayMs(null, 1_000)) + assertEquals(0, WorkManagerScheduler.initialDelayMs(500, 1_000)) + assertEquals(4_000, WorkManagerScheduler.initialDelayMs(5_000, 1_000)) + } + + @Test + fun `a queued successor suppresses another append`() { + assertTrue(WorkManagerScheduler.hasQueuedSuccessor(listOf(RUNNING, BLOCKED))) + assertTrue(WorkManagerScheduler.hasQueuedSuccessor(listOf(ENQUEUED))) + assertFalse(WorkManagerScheduler.hasQueuedSuccessor(listOf(RUNNING))) + assertFalse(WorkManagerScheduler.hasQueuedSuccessor(listOf(SUCCEEDED, FAILED, CANCELLED))) + assertFalse(WorkManagerScheduler.hasQueuedSuccessor(emptyList())) + } + + @Test + fun `the wake name differs from the main chain`() { + assertEquals("e1#wake", WorkManagerScheduler.wakeName("e1")) + } +} + +class TransferSemaphoreTest { + @Test + fun `the global cap is 4 and a fifth request waits`() = runBlocking { + assertEquals(4, MAX_TRANSFER_CONCURRENCY) + repeat(4) { transferSemaphore.acquire() } + try { + assertFalse(transferSemaphore.tryAcquire()) // no fifth permit + transferSemaphore.release() + assertTrue(transferSemaphore.tryAcquire()) // one freed, one taken + } finally { + repeat(4) { transferSemaphore.release() } + } + assertEquals(4, transferSemaphore.availablePermits) + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/TestSupport.kt b/android/src/test/java/ai/openspace/backgroundupload/TestSupport.kt new file mode 100644 index 00000000..c90665d4 --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/TestSupport.kt @@ -0,0 +1,126 @@ +package ai.openspace.backgroundupload + +// Builders and fakes shared by the JVM tests. No android.* here. + +internal const val FAR_FUTURE = 4_000_000_000_000L + +internal fun desc( + url: String? = "https://example.com/items", + method: String = "POST", + headers: Map = mapOf("Authorization" to "Bearer old"), + dataJson: String? = null, + form: List? = null, + file: String? = null, + parts: List? = null, + accept: List = emptyList(), + retry: RetryOverride? = null, + noNotification: Boolean = false, + wifiOnly: Boolean? = null, +) = Descriptor(url, method, headers, dataJson, form, file, parts, accept, retry, noNotification, wifiOnly) + +internal fun part(start: Long, end: Long, accepted: Boolean = false, url: String? = null) = Part( + url = url ?: "https://example.com/part?start=$start", + headers = mapOf("Content-Range" to "$start-${end - 1}"), + start = start, + end = end, + accepted = accepted, +) + +internal fun entry( + id: String = "e1", + state: EntryState = EntryState.QUEUED, + descriptor: Descriptor? = desc(dataJson = """{"a":1}"""), + body: StagedBody? = StagedBody(StagedBody.JSON, "body-1.json", null, 7), + generation: Int = 1, + attempts: Int = 0, + settledEventId: String? = null, + parkedGeneration: Int? = null, + nextAttemptAt: Long? = null, + expiresAt: Long = FAR_FUTURE, + legacy: Boolean = false, + key: String = "note", + createdAt: Long = 1_000, +) = QueueEntry( + id = id, + key = key, + varsJson = """{"n":1}""", + descriptor = descriptor, + body = body, + state = state, + attempts = attempts, + bytesSent = 0, + totalBytes = body?.totalBytes ?: 0, + expiresAt = expiresAt, + createdAt = createdAt, + updatedAt = createdAt, + nextAttemptAt = nextAttemptAt, + parkedGeneration = parkedGeneration, + generation = generation, + settledEventId = settledEventId, + legacy = legacy, +) + +internal fun parsed( + id: String = "e1", + descriptor: Descriptor = desc(dataJson = """{"a":1}"""), + varsJson: String = """{"n":1}""", + expiresAt: Long = FAR_FUTURE, + key: String = "note", +) = EntryParsing.Parsed(id, key, varsJson, descriptor, expiresAt) + +internal fun record( + eventId: String, + id: String = "e1", + kind: String = EventJournal.KIND_COMPLETED, + generation: Int = 1, + at: Long = 5_000, + state: String = kind, +) = EventJournal.SettledRecord( + eventId = eventId, id = id, key = "note", varsJson = """{"n":1}""", at = at, attempts = 1, + requestId = "r1", deliveries = 1, state = state, bytesSent = 0, totalBytes = 0, + url = "https://example.com/items", method = "POST", partIndex = null, kind = kind, + response = if (kind == EventJournal.KIND_COMPLETED) EventJournal.Response(200, mapOf(), "ok", false) else null, + errorKind = if (kind == EventJournal.KIND_ERROR) "http" else null, + message = if (kind == EventJournal.KIND_ERROR) "HTTP 400" else null, + cancelReason = if (kind == EventJournal.KIND_CANCELLED) "user" else null, + generation = generation, +) + +/** Records every event in order, as "state::" and "settled::". */ +internal class RecordingEvents : QueueEvents { + val log = mutableListOf() + val rows = mutableListOf() + val records = mutableListOf() + + /** The listener each settled event went to, in order. */ + val listeners = mutableListOf() + + override fun state(row: RequestRow) { + rows += row + log += "state:${row.id}:${row.state}" + } + + override fun settled(record: EventJournal.SettledRecord, listener: Any) { + records += record + listeners += listener + log += "settled:${record.id}:${record.kind}" + } +} + +internal class FakeScheduler : WorkScheduler { + val scheduled = mutableListOf() + val wakes = mutableListOf>() + val cancelled = mutableListOf() + + override fun schedule(entry: QueueEntry) { + scheduled += entry.id + } + + override fun scheduleWake(entry: QueueEntry, at: Long, replace: Boolean) { + wakes += entry.id to at + } + + override fun cancel(id: String) { + cancelled += id + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/UploadOutcomeTest.kt b/android/src/test/java/ai/openspace/backgroundupload/UploadOutcomeTest.kt new file mode 100644 index 00000000..9976f240 --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/UploadOutcomeTest.kt @@ -0,0 +1,59 @@ +package ai.openspace.backgroundupload + +import ai.openspace.backgroundupload.UploadOutcome.AcceptRule +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test + +class UploadOutcomeTest { + + @Test + fun `2xx is accepted`() { + assertTrue(UploadOutcome.isAccepted(200, "", listOf())) + assertTrue(UploadOutcome.isAccepted(204, "", listOf())) + assertTrue(UploadOutcome.isAccepted(299, "", listOf())) + } + + @Test + fun `non-2xx is not accepted by default`() { + assertFalse(UploadOutcome.isAccepted(199, "", listOf())) + assertFalse(UploadOutcome.isAccepted(300, "", listOf())) + assertFalse(UploadOutcome.isAccepted(404, "", listOf())) + assertFalse(UploadOutcome.isAccepted(500, "", listOf())) + } + + @Test + fun `a status-only rule accepts its status regardless of body`() { + val rules = listOf(AcceptRule(409)) + assertTrue(UploadOutcome.isAccepted(409, "anything", rules)) + assertTrue(UploadOutcome.isAccepted(409, null, rules)) + } + + @Test + fun `rules do not accept unlisted codes`() { + assertFalse(UploadOutcome.isAccepted(500, "", listOf(AcceptRule(409)))) + } + + // The backend's 409 carries several meanings, and only the message shows the + // difference. bodyIncludes is what separates the success meanings from the + // bug meanings. + @Test + fun `bodyIncludes narrows a rule to matching bodies`() { + val rules = listOf(AcceptRule(409, bodyIncludes = "already completed")) + assertTrue(UploadOutcome.isAccepted(409, "part already completed", rules)) + assertFalse(UploadOutcome.isAccepted(409, "part number mismatch", rules)) + assertFalse(UploadOutcome.isAccepted(409, null, rules)) + } + + @Test + fun `any matching rule accepts`() { + val rules = listOf( + AcceptRule(409, bodyIncludes = "already completed"), + AcceptRule(208), + ) + assertTrue(UploadOutcome.isAccepted(208, "", rules)) + assertTrue(UploadOutcome.isAccepted(409, "already completed", rules)) + assertFalse(UploadOutcome.isAccepted(410, "already completed", rules)) + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/WorkerGateTest.kt b/android/src/test/java/ai/openspace/backgroundupload/WorkerGateTest.kt new file mode 100644 index 00000000..dde4a6b0 --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/WorkerGateTest.kt @@ -0,0 +1,51 @@ +package ai.openspace.backgroundupload + +import org.junit.After +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test + +class WorkerGateTest { + private val a = Any() + private val b = Any() + + @After + fun tearDown() { + // A process-wide singleton. Leave nothing for other tests. + listOf("g1", "g2").forEach { id -> WorkerGate.release(id, a); WorkerGate.release(id, b) } + } + + @Test + fun `a second worker for the same id must wait`() { + assertTrue(WorkerGate.tryAcquire("g1", a)) + assertFalse(WorkerGate.tryAcquire("g1", b)) + WorkerGate.release("g1", a) + assertTrue(WorkerGate.tryAcquire("g1", b)) + } + + @Test + fun `reacquiring with the same token is idempotent`() { + assertTrue(WorkerGate.tryAcquire("g1", a)) + assertTrue(WorkerGate.tryAcquire("g1", a)) + } + + @Test + fun `a stale release can not evict a successor`() { + assertTrue(WorkerGate.tryAcquire("g1", a)) + WorkerGate.release("g1", a) + assertTrue(WorkerGate.tryAcquire("g1", b)) + WorkerGate.release("g1", a) + assertTrue(WorkerGate.isRunning("g1")) + assertFalse(WorkerGate.tryAcquire("g1", a)) + } + + @Test + fun `ids are independent`() { + assertTrue(WorkerGate.tryAcquire("g1", a)) + assertFalse(WorkerGate.isRunning("g2")) + assertTrue(WorkerGate.tryAcquire("g2", a)) + WorkerGate.release("g1", a) + assertFalse(WorkerGate.isRunning("g1")) + assertTrue(WorkerGate.isRunning("g2")) + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/WorkerOpsTest.kt b/android/src/test/java/ai/openspace/backgroundupload/WorkerOpsTest.kt new file mode 100644 index 00000000..4ffb7fac --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/WorkerOpsTest.kt @@ -0,0 +1,436 @@ +package ai.openspace.backgroundupload + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNotNull +import org.junit.Assert.assertNull +import org.junit.Assert.assertThrows +import org.junit.Assert.assertTrue +import org.junit.Before +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import java.io.File +import java.util.concurrent.CyclicBarrier + +class WorkerOpsTest { + @get:Rule + val tmp = TemporaryFolder() + + private lateinit var store: QueueStore + private lateinit var journal: EventJournal + private lateinit var settings: QueueSettingsStore + private val events = RecordingEvents() + private val scheduler = FakeScheduler() + private var now = 20_000L + private lateinit var ops: WorkerOps + private lateinit var controller: QueueController + private val listener = Any() + private lateinit var journalDir: File + + @Before + fun setUp() { + val root = tmp.newFolder("queue") + store = QueueStore(root, RequestIndex()) + journalDir = tmp.newFolder("journal") + journal = EventJournal(journalDir, retryLater = { _, _ -> }) + settings = QueueSettingsStore(File(root, "settings.json")) + ops = WorkerOps(store, journal, settings, events, scheduler) { now } + controller = QueueController(store, journal, settings, events, scheduler, { false }, { now }) + journal.drain(listener) // JS is subscribed + } + + private fun withJournalReadOnly(block: () -> T): T { + journalDir.setWritable(false) + try { + return block() + } finally { + journalDir.setWritable(true) + } + } + + private val ok = UploadResponse(200, """{"id":7}""", mapOf("x" to "y")) + + @Test + fun `begin takes a queued entry and emits running`() { + store.save(entry(nextAttemptAt = 5)) + val e = ops.begin("e1")!! + assertEquals(EntryState.RUNNING, e.state) + assertNull(e.nextAttemptAt) + assertEquals(listOf("state:e1:running"), events.log) + } + + @Test + fun `begin applies a record of the entry's own generation and does not run it again`() { + // The process died between the journal write and the store transition, + // and WorkManager runs the entry again before any boot sweep. + val own = "00000000-0000-0000-0000-000000000021" + val older = "00000000-0000-0000-0000-000000000022" + store.save(entry(state = EntryState.RUNNING, generation = 2)) + journal.append(record(older, generation = 2, at = 1)) + journal.append(record(own, generation = 2, at = 2)) + assertNull(ops.begin("e1")) + val e = store.load("e1")!! + assertEquals(EntryState.COMPLETED, e.state) + assertEquals(own, e.settledEventId) + assertEquals(listOf(own), journal.unacknowledged().map { it.eventId }) // the extra one is acked + assertEquals(listOf("state:e1:completed"), events.log) + } + + @Test + fun `begin runs an entry whose only record is of an older generation`() { + store.save(entry(state = EntryState.QUEUED, generation = 2)) + journal.append(record("00000000-0000-0000-0000-000000000023", generation = 1)) + assertEquals(EntryState.RUNNING, ops.begin("e1")!!.state) + } + + @Test + fun `begin does nothing for a paused, settled, or legacy entry`() { + store.save(entry(id = "p", state = EntryState.PAUSED)) + store.save(entry(id = "x", state = EntryState.ERROR)) + store.save(entry(id = "l", state = EntryState.QUEUED, legacy = true, descriptor = null, body = null)) + assertNull(ops.begin("p")) + assertNull(ops.begin("x")) + assertNull(ops.begin("l")) + assertNull(ops.begin("nope")) + } + + @Test + fun `recordAttempt persists attempts and the request id before the send`() { + store.save(entry(state = EntryState.RUNNING, attempts = 2)) + val e = ops.recordAttempt("e1", 1, "req-3") + assertEquals(3, e.attempts) + assertEquals("req-3", store.load("e1")!!.lastRequestId) + assertEquals(3, store.load("e1")!!.attempts) + } + + @Test + fun `recordAttempt refuses once the module took the entry`() { + store.save(entry(state = EntryState.PAUSED)) + assertThrows(NotOwnedException::class.java) { ops.recordAttempt("e1", 1, "r") } + store.save(entry(state = EntryState.RUNNING, generation = 2)) + assertThrows(NotOwnedException::class.java) { ops.recordAttempt("e1", 1, "r") } + } + + @Test + fun `settle journals, transitions, then emits settled before state`() { + store.save(entry(state = EntryState.RUNNING, attempts = 1).copy(lastRequestId = "req-1")) + assertTrue(ops.settle("e1", 1, Settlement.Completed(ok, "https://example.com/items", "POST"))) + val record = journal.unacknowledged().single() + assertEquals(1, record.deliveries) + assertEquals("req-1", record.requestId) + assertEquals(200, record.response!!.status) + assertEquals(1, record.generation) + val e = store.load("e1")!! + assertEquals(EntryState.COMPLETED, e.state) + assertEquals(record.eventId, e.settledEventId) + assertEquals(e.totalBytes, e.bytesSent) + assertEquals(listOf("settled:e1:completed", "state:e1:completed"), events.log) + } + + @Test + fun `a settle with no listener starts at 0 deliveries, so the first replay is 1`() { + journal.stopListening(listener) + store.save(entry(state = EntryState.RUNNING)) + assertTrue(ops.settle("e1", 1, Settlement.Completed(ok, "u", "POST"))) + assertEquals(0, journal.unacknowledged().single().deliveries) + assertEquals(listOf("state:e1:completed"), events.log) // nothing to emit to + assertEquals(1, controller.unacknowledged(listener).single().deliveries) + } + + @Test + fun `a settle whose journal can not write holds the record, settles, and emits`() { + store.save(entry(state = EntryState.RUNNING)) + assertTrue(withJournalReadOnly { ops.settle("e1", 1, Settlement.Completed(ok, "u", "POST")) }) + val e = store.load("e1")!! + assertEquals(EntryState.COMPLETED, e.state) + assertTrue(journal.isHeld(e.settledEventId!!)) + assertEquals(listOf("settled:e1:completed", "state:e1:completed"), events.log) + // The sweep does not forget a row whose record is held, and the ack does. + controller.sweep() + assertNotNull(store.load("e1")) + controller.ack(listOf(e.settledEventId!!)) + assertNull(store.load("e1")) + } + + @Test + fun `a settle whose store write fails leaves the record for the next run to apply`() { + store.save(entry(state = EntryState.RUNNING)) + val dir = store.entryDir("e1") + dir.setWritable(false) + val stood = try { + ops.settle("e1", 1, Settlement.Completed(ok, "u", "POST")) + } finally { + dir.setWritable(true) + } + assertTrue(stood) + assertEquals(EntryState.RUNNING, store.load("e1")!!.state) + val record = journal.unacknowledged().single() + assertEquals(listOf("settled:e1:completed"), events.log) // no state event: the row did not change + // WorkManager runs it again: begin applies the record, no second send. + assertNull(ops.begin("e1")) + assertEquals(record.eventId, store.load("e1")!!.settledEventId) + } + + @Test + fun `a settle after a cancel whose save failed applies the cancel, not a second outcome`() { + // cancel() journaled its record, then its entry save failed: the entry + // is still running, and its request comes back. + val cancelId = "00000000-0000-0000-0000-000000000041" + store.save(entry(state = EntryState.RUNNING)) + journal.append(record(cancelId, kind = EventJournal.KIND_CANCELLED)) + assertFalse(ops.settle("e1", 1, Settlement.Completed(ok, "u", "POST"))) + val e = store.load("e1")!! + assertEquals(EntryState.CANCELLED, e.state) + assertEquals(cancelId, e.settledEventId) + assertEquals(listOf(cancelId), journal.unacknowledged().map { it.eventId }) + assertEquals(listOf("state:e1:cancelled"), events.log) // no second outcome + } + + @Test + fun `a live settle goes to the listener that drained, not the newest module`() { + store.save(entry(id = "a", state = EntryState.RUNNING)) + ops.settle("a", 1, Settlement.Completed(ok, "u", "POST")) + // A reload: the next module's JS drains and takes over. + val next = Any() + controller.unacknowledged(next) + store.save(entry(id = "b", state = EntryState.RUNNING)) + ops.settle("b", 1, Settlement.Completed(ok, "u", "POST")) + assertEquals(2, events.listeners.size) + assertTrue(events.listeners[0] === listener) + assertTrue(events.listeners[1] === next) + } + + @Test + fun `a settle over the journal cap spares every record a row names`() { + val small = EventJournal(tmp.newFolder("small"), maxEntries = 2) + val smallOps = WorkerOps(store, small, settings, events, scheduler) { now } + val named = "00000000-0000-0000-0000-000000000031" + small.append(record(named, id = "done")) + store.save(entry(id = "done", state = EntryState.ERROR, settledEventId = named)) + File(tmp.root, "small/$named.json").setLastModified(1_000) + small.append(record("00000000-0000-0000-0000-000000000032", id = "orphan")) + File(tmp.root, "small/00000000-0000-0000-0000-000000000032.json").setLastModified(2_000) + store.save(entry(state = EntryState.RUNNING)) + smallOps.settle("e1", 1, Settlement.Completed(ok, "u", "POST")) + val left = small.unacknowledged().map { it.eventId } + assertTrue(left.contains(named)) + assertTrue(left.contains(store.load("e1")!!.settledEventId)) + assertEquals(2, left.size) + } + + @Test + fun `a response that lands after cancel drops its record`() { + controller.enqueue(parsed()) + ops.begin("e1") + controller.cancel("e1") + events.log.clear() + assertFalse(ops.settle("e1", 1, Settlement.Completed(ok, "u", "POST"))) + val left = journal.unacknowledged().single() + assertEquals(EventJournal.KIND_CANCELLED, left.kind) + assertEquals(EntryState.CANCELLED, store.load("e1")!!.state) + assertEquals(emptyList(), events.log) + } + + @Test + fun `a settle from an older generation is dropped`() { + store.save(entry(state = EntryState.QUEUED, generation = 2)) + assertFalse(ops.settle("e1", 1, Settlement.Completed(ok, "u", "POST"))) + assertEquals(emptyList(), journal.unacknowledged()) + } + + @Test + fun `an accepted response that lands during pause still settles`() { + store.save(entry(state = EntryState.PAUSED)) + assertTrue(ops.settle("e1", 1, Settlement.Completed(ok, "u", "POST"))) + assertEquals(EntryState.COMPLETED, store.load("e1")!!.state) + } + + @Test + fun `a failure that lands during pause does not settle`() { + store.save(entry(state = EntryState.PAUSED)) + assertFalse(ops.settle("e1", 1, Settlement.Failed("http", "HTTP 400", ok.copy(code = 400), null, "u", "POST"))) + assertEquals(EntryState.PAUSED, store.load("e1")!!.state) + assertEquals(emptyList(), journal.unacknowledged()) + assertEquals(emptyList(), events.log) + } + + @Test + fun `a failed simple settle keeps the live bytes, and a chunked one keeps its accepted bytes`() { + store.save(entry(state = EntryState.RUNNING)) + ops.settle("e1", 1, Settlement.Failed("http", "HTTP 400", ok.copy(code = 400), null, "u", "POST", bytesSent = 5)) + assertEquals(5, store.load("e1")!!.bytesSent) + assertEquals(5, journal.unacknowledged().single().bytesSent) + store.save(entry(id = "c", state = EntryState.RUNNING).copy(bytesSent = 10)) + ops.settle("c", 1, Settlement.Failed("file", "gone", null, 1, "u", "PUT")) + assertEquals(10, store.load("c")!!.bytesSent) + } + + @Test + fun `park sets awaiting-auth once and wakes at expiry`() { + store.save(entry(state = EntryState.RUNNING, expiresAt = 90_000)) + assertEquals(WorkerOps.ParkResult.PARKED, ops.park("e1", 1, headerGeneration = 0)) + val e = store.load("e1")!! + assertEquals(EntryState.AWAITING_AUTH, e.state) + assertEquals(0, e.parkedGeneration) + assertEquals(listOf("state:e1:awaiting-auth"), events.log) + assertEquals(listOf("e1" to 90_000L), scheduler.wakes) + assertEquals(emptyList(), journal.unacknowledged()) + } + + @Test + fun `a 401 sent under older headers re-issues instead of parking`() { + store.save(entry(state = EntryState.RUNNING)) + controller.updateHeaders(mapOf("Authorization" to "Bearer new")) + assertEquals(WorkerOps.ParkResult.REISSUE, ops.park("e1", 1, headerGeneration = 0)) + assertEquals(EntryState.RUNNING, store.load("e1")!!.state) + } + + @Test + fun `updateHeaders after a park requeues it`() { + store.save(entry(state = EntryState.RUNNING)) + ops.park("e1", 1, 0) + controller.updateHeaders(mapOf("Authorization" to "Bearer new")) + assertEquals(EntryState.QUEUED, store.load("e1")!!.state) + assertEquals(listOf("e1"), scheduler.scheduled) + } + + @Test + fun `release queues the entry with its wake time and streak`() { + store.save(entry(state = EntryState.RUNNING)) + assertTrue(ops.release("e1", 1, nextAttemptAt = 80_000, streak = 7)) + val e = store.load("e1")!! + assertEquals(EntryState.QUEUED, e.state) + assertEquals(80_000L, e.nextAttemptAt) + assertEquals(7, e.backoffStreak) + assertEquals(listOf("e1" to 80_000L), scheduler.wakes) + assertEquals(80_000.0, events.rows.single().toMap()["nextAttemptAt"]) + } + + @Test + fun `a system stop queues a running entry and leaves a paused one`() { + store.save(entry(state = EntryState.RUNNING)) + ops.stopped("e1", 1) + assertEquals(EntryState.QUEUED, store.load("e1")!!.state) + store.save(entry(state = EntryState.PAUSED)) + ops.stopped("e1", 1) + assertEquals(EntryState.PAUSED, store.load("e1")!!.state) + assertEquals(emptyList(), journal.unacknowledged()) + } + + @Test + fun `markAccepted persists the flag and the accepted bytes`() { + store.save(entry( + state = EntryState.RUNNING, + descriptor = desc(url = null, file = "/f", parts = listOf(part(0, 10), part(10, 25))), + body = StagedBody(StagedBody.CHUNKED, "blob", null, 25), + ).copy(backoffStreak = 3)) + ops.markAccepted("e1", 1, 1) + val e = store.load("e1")!! + assertTrue(e.descriptor!!.parts!![1].accepted) + assertEquals(15, e.bytesSent) + assertEquals(0, e.backoffStreak) + } + + // MARK: - header generation of an attempt + + @Test + fun `an attempt carries the header generation of the headers it sends`() { + store.save(entry(state = EntryState.RUNNING)) + controller.updateHeaders(mapOf("Authorization" to "Bearer new")) + val e = ops.recordAttempt("e1", 1, "r1") + assertEquals(1, e.headerGeneration) + assertEquals("Bearer new", e.descriptor!!.headers["Authorization"]) + } + + @Test + fun `a 401 between the settings bump and the entry patch parks, and the patch requeues it`() { + store.save(entry(state = EntryState.RUNNING)) + // updateHeaders() has bumped the settings but not yet patched the entry. + settings.update { it.copy(headerGeneration = it.headerGeneration + 1) } + val sent = ops.recordAttempt("e1", 1, "r1") + assertEquals(0, sent.headerGeneration) + assertFalse(ops.hasNewerHeaders("e1", 1, sent.headerGeneration)) // no re-issue with the same old headers + assertEquals(WorkerOps.ParkResult.PARKED, ops.park("e1", 1, sent.headerGeneration)) + // The rest of updateHeaders() finds it parked. + controller.updateHeaders(mapOf("Authorization" to "Bearer new")) + val e = store.load("e1")!! + assertEquals(EntryState.QUEUED, e.state) + assertEquals("Bearer new", e.descriptor!!.headers["Authorization"]) + } + + @Test + fun `a re-issue sends the patched headers, and a 401 on them parks with them`() { + store.save(entry(state = EntryState.RUNNING)) + val first = ops.recordAttempt("e1", 1, "r1") + controller.updateHeaders(mapOf("Authorization" to "Bearer new")) + assertTrue(ops.hasNewerHeaders("e1", 1, first.headerGeneration)) + val second = ops.recordAttempt("e1", 1, "r2") + assertEquals("Bearer new", second.descriptor!!.headers["Authorization"]) + assertFalse(ops.hasNewerHeaders("e1", 1, second.headerGeneration)) + assertEquals(WorkerOps.ParkResult.PARKED, ops.park("e1", 1, second.headerGeneration)) + assertEquals(1, store.load("e1")!!.parkedGeneration) + } + + // MARK: - short backoff + + @Test + fun `a short backoff shows nextAttemptAt on the running row until the next attempt`() { + store.save(entry(state = EntryState.RUNNING)) + ops.backingOff("e1", 1, nextAttemptAt = 24_000) + val waiting = store.load("e1")!! + assertEquals(EntryState.RUNNING, waiting.state) + assertEquals(24_000L, waiting.nextAttemptAt) + assertEquals(24_000.0, events.rows.last().toMap()["nextAttemptAt"]) + ops.recordAttempt("e1", 1, "r2") + assertNull(store.load("e1")!!.nextAttemptAt) + assertEquals(listOf("state:e1:running", "state:e1:running"), events.log) + assertFalse(events.rows.last().toMap().containsKey("nextAttemptAt")) + } + + @Test + fun `an attempt with no pending backoff emits no state event`() { + store.save(entry(state = EntryState.RUNNING)) + ops.recordAttempt("e1", 1, "r1") + assertEquals(emptyList(), events.log) + } + + @Test + fun `a short backoff on an entry the run no longer owns changes nothing`() { + store.save(entry(state = EntryState.PAUSED)) + ops.backingOff("e1", 1, nextAttemptAt = 24_000) + assertNull(store.load("e1")!!.nextAttemptAt) + assertEquals(emptyList(), events.log) + } + + // MARK: - deliveries + + @Test + fun `a settle racing the first drain reaches JS exactly once with deliveries 1`() { + // The drain sets the listener and scans under one journal lock, and the + // append decides 0 or 1 under it. Either the drain returns the record, + // or the settle emits it live; never both, never neither. + repeat(200) { i -> + val id = "race-$i" + val owner = Any() + // The previous iteration's drain left its owner as the listener. Clear + // it, so the race starts with no listener every time. + journal.listener()?.let { journal.stopListening(it) } + assertFalse(journal.isListening()) + store.save(entry(id = id, state = EntryState.RUNNING)) + events.records.clear() + val start = CyclicBarrier(2) + var drained: List = emptyList() + val drain = Thread { start.await(); drained = controller.unacknowledged(owner).filter { it.id == id } } + drain.start() + start.await() + ops.settle(id, 1, Settlement.Completed(ok, "u", "POST")) + drain.join() + val live = events.records.filter { it.id == id } + val seen = drained + live + assertEquals("iteration $i", 1, seen.size) + assertEquals("iteration $i", 1, seen.single().deliveries) + journal.ack(listOf(seen.single().eventId)) + } + } +} diff --git a/babel.config.js b/babel.config.js new file mode 100644 index 00000000..7bcd7fce --- /dev/null +++ b/babel.config.js @@ -0,0 +1,8 @@ +// For Jest only. Metro/RN builds use each app's own babel config; the library +// itself ships TypeScript source (no build step). +module.exports = { + presets: [ + ['@babel/preset-env', { targets: { node: 'current' } }], + '@babel/preset-typescript', + ], +}; diff --git a/docs/superpowers/plans/2026-07-22-phase0-reliable-uploads.md b/docs/superpowers/plans/2026-07-22-phase0-reliable-uploads.md new file mode 100644 index 00000000..1c9526d8 --- /dev/null +++ b/docs/superpowers/plans/2026-07-22-phase0-reliable-uploads.md @@ -0,0 +1,1841 @@ +# Phase 0: Reliable Upload Statuses Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make upload outcomes impossible to lose and impossible to misread — a native, durable event journal with acknowledgment, a query API for live uploads, correct success/error classification (`completed` = accepted statuses only), distinguishable cancel reasons, and iOS background-relaunch handling (v8.0.0, breaking changes coordinated with Diana). + +**Architecture:** Terminal events (completed/error/cancelled) are written to a native on-disk journal *before* being emitted to JS, and deleted only when JS acknowledges them; delivery becomes at-least-once instead of at-most-once. A `getAllUploads()` query enumerates live uploads (WorkManager work on Android, session tasks on iOS) so consumers can reconcile on boot instead of guessing. Outcomes are classified natively (2xx/`acceptStatus` → completed; other HTTP → typed error with the response attached) instead of pushing success determination onto every consumer. The event API keeps its names, but semantics are corrected where the old ones were wrong — Diana adopts in one coordinated bump. + +**Tech Stack:** Kotlin (WorkManager 2.8.1, OkHttp 4.10, Gson 2.8.9 — all existing deps), Objective-C (background NSURLSession), TypeScript. Tests: JUnit 4 (plain JVM) for Android journal logic, Jest for the JS surface, on-device checklist via the example app + `example/server`. + +**RFC / context:** https://github.com/openspacelabs/diana/issues/8875. Research notes in `.claude/work/831bb775-1eba-45d6-b429-ad2951acda24/`. + +## Global Constraints + +- **Best practices and official platform documentation take precedence over existing code** (Dylan, 2026-07-23: "we don't need to be married to anything in the current library"). Breaking changes are allowed — Diana is the sole consumer and pins a git tag, so every break ships with a coordinated Diana change (listed in "After Phase 0"). Event names (`RNFileUploader-*`) and native module names are kept this phase only to limit churn; they rename at the Phase 1 package split. +- Outcome classification (breaking): `completed` fires only for 2xx or per-request `acceptStatus` codes; other HTTP responses emit `error` with `errorKind: 'http'` and the full response attached. Transport failures are `errorKind: 'network'`, missing files `'file'`. +- No new runtime dependencies. JUnit is test-only. Gson/OkHttp/WorkManager stay at current versions. +- Objective-C only on iOS (no Swift); podspec globs `ios/*.{h,m}`. +- Kotlin `jvmTarget = "17"` as configured in `android/build.gradle`. +- Version bump to **8.0.0** (breaking: completed = accepted statuses only; android options optional; `lib/` removed). Branch: `dylan/phase0-reliable-uploads`. +- Every commit must leave the example app buildable. Run `yarn lint:ci` before pushing; do not poll CI after push. +- Response bodies in the journal are capped at **64 KB** with a `responseBodyTruncated` flag. +- Journal event `type` values: `'completed' | 'error' | 'cancelled'`. Cancel `reason` values: `'user' | 'system'`. +- Platform-API claims in this plan were fact-checked against official docs — see `.claude/work/831bb775-1eba-45d6-b429-ad2951acda24/07-platform-facts.md`. Two findings shaped the design: **iOS `taskDescription` has no documented persistence guarantee across process death** (hence the `RNBGUTaskMap` sidecar in Task 5 — Apple DTS guidance is to persist task metadata externally keyed by `taskIdentifier`), and **WorkManager auto-prunes finished work after ~1 day**, so `getAllUploads()` only sees recent terminal work — the journal is the source of truth for outcomes. + +## Decision gate (read first) + +**Resolved 2026-07-22: BUILD — execute Tasks 1–11 as written.** The candidate wrapper target (`@kesha-antonov/react-native-background-downloader` v4.5.8) was verified at source level (`.claude/work/831bb775-1eba-45d6-b429-ad2951acda24/06-kesha-source-verification.md`). Its iOS upload path is sound (background NSURLSession, `uploadTaskWithRequest:fromFile:`, AppDelegate hook), but two dealbreakers: (1) **Android uploads run on a plain thread pool with `HttpURLConnection`** — no WorkManager, no foreground service, uploads die with the process, "resume" restarts from byte 0; (2) **no completion journal on either platform** — an upload that completes while the app is dead emits one fire-and-forget event, deletes its persisted config, and its outcome/response is unrecoverable (`getExistingUploadTasks()` cannot see it). Wrapping it would regress the two guarantees this library exists to provide. No device spike needed; Task 0 below is kept for the record. + +--- + +### Task 0: Build-vs-wrap decision gate + +**Files:** none (decision task). + +**Interfaces:** +- Produces: a go/no-go decision recorded at the top of this plan. "GO (build)" → execute Tasks 1–11 as written. "WRAP" → stop; the native tasks are re-planned as a wrapper (JS contract in Task 9 is unchanged). + +- [x] **Step 1: Read the source-verification report** + +Read `.claude/work/831bb775-1eba-45d6-b429-ad2951acda24/06-kesha-source-verification.md`. It answers, with code quotes: does the kesha-antonov iOS upload path use `backgroundSessionConfigurationWithIdentifier` + `uploadTaskWithRequest:fromFile:`; does Android survive process death; is there any dead-JS event persistence. + +- [x] **Step 2: Apply the decision rule** + +- If the report's verdict is **BUILD** (uploads don't truly survive termination, or no meaningful advantage over our fork): proceed with Tasks 1–11. No device spike needed. +- If **WRAP or PARTIAL**: run a half-day device spike before writing native code — upload a 500 MB file from the example app, force-quit mid-upload, verify resumption/completion delivery on relaunch, on both platforms. Only a passing spike flips this plan to wrapper mode. + +- [x] **Step 3: Record the decision** + +Decision: BUILD — 2026-07-22 — kesha-antonov Android uploads don't survive process death (plain thread pool, no WorkManager) and neither platform journals completions; wrapping would regress both core guarantees. + +--- + +### Task 1: Android `EventJournal` (durable terminal-event store) + +**Files:** +- Create: `android/src/main/java/com/vydia/RNUploader/EventJournal.kt` +- Modify: `android/build.gradle` (test dependency) +- Test: `android/src/test/java/com/vydia/RNUploader/EventJournalTest.kt` + +**Interfaces:** +- Consumes: nothing (pure Kotlin + Gson + java.io; no Android runtime classes → plain JVM tests). +- Produces: `EventJournal.get(context: Context): EventJournal`, `Entry` data class, `append(entry)`, `unacknowledged(): List`, `ack(eventIds: List)`. Constructor `EventJournal(dir: File)` is public for tests. `EventJournal.MAX_BODY_BYTES = 64 * 1024`. + +- [ ] **Step 1: Add the test dependency** + +In `android/build.gradle`, inside the existing `dependencies { }` block, add: + +```groovy + testImplementation 'junit:junit:4.13.2' +``` + +- [ ] **Step 2: Write the failing tests** + +Create `android/src/test/java/com/vydia/RNUploader/EventJournalTest.kt`: + +```kotlin +package com.vydia.RNUploader + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder + +class EventJournalTest { + @get:Rule + val tmp = TemporaryFolder() + + private fun entry(id: String, uploadId: String = "u1") = EventJournal.Entry( + eventId = id, + uploadId = uploadId, + type = "completed", + timestamp = System.currentTimeMillis(), + responseCode = 200, + responseBody = "ok", + responseHeaders = mapOf("x-a" to "b"), + ) + + @Test + fun `append then read returns the entry`() { + val journal = EventJournal(tmp.newFolder()) + journal.append(entry("e1")) + val events = journal.unacknowledged() + assertEquals(1, events.size) + assertEquals("e1", events[0].eventId) + assertEquals(200, events[0].responseCode) + assertEquals("ok", events[0].responseBody) + } + + @Test + fun `ack removes only the acked entry`() { + val journal = EventJournal(tmp.newFolder()) + journal.append(entry("e1")) + journal.append(entry("e2")) + journal.ack(listOf("e1")) + assertEquals(listOf("e2"), journal.unacknowledged().map { it.eventId }) + } + + @Test + fun `entries survive a new journal instance over the same dir`() { + val dir = tmp.newFolder() + EventJournal(dir).append(entry("e1")) + assertEquals(1, EventJournal(dir).unacknowledged().size) + } + + @Test + fun `oversized body is truncated and flagged`() { + val journal = EventJournal(tmp.newFolder()) + val big = "x".repeat(EventJournal.MAX_BODY_BYTES + 100) + journal.append(entry("e1").copy(responseBody = big)) + val read = journal.unacknowledged()[0] + assertTrue(read.responseBodyTruncated) + assertTrue(read.responseBody!!.length <= EventJournal.MAX_BODY_BYTES) + } + + @Test + fun `corrupt file is skipped, not fatal`() { + val dir = tmp.newFolder() + val journal = EventJournal(dir) + journal.append(entry("e1")) + java.io.File(dir, "garbage.json").writeText("{not json") + assertEquals(1, journal.unacknowledged().size) + } + + @Test + fun `entries are ordered by timestamp`() { + val journal = EventJournal(tmp.newFolder()) + journal.append(entry("late").copy(timestamp = 2000)) + journal.append(entry("early").copy(timestamp = 1000)) + assertEquals(listOf("early", "late"), journal.unacknowledged().map { it.eventId }) + } +} +``` + +- [ ] **Step 3: Run tests to verify they fail** + +Run: `cd example/RNBGUExample/android && ./gradlew :react-native-background-upload:testDebugUnitTest --tests '*EventJournalTest*'` +Expected: FAIL — `Unresolved reference: EventJournal` (compilation error). +(The example app's `settings.gradle` maps `:react-native-background-upload` to the repo's `android/` dir, so this runs against live source. Run `yarn --cwd example/RNBGUExample install` first if `node_modules` is missing.) + +> **Review addendum (2026-07-23):** during review we hardened the journal beyond the +> original block below. The committed `EventJournal.kt` is authoritative; deltas from the +> code shown here: +> 1. Constructor takes `maxEntries: Int = MAX_ENTRIES` (`MAX_ENTRIES = 1000`); `append` +> calls `pruneToMax()` which drops the oldest `.json` files (by `lastModified`) beyond +> the cap — a runaway guard for a broken/unadopted ack loop. +> 2. `append` wraps its write in try/catch and returns on failure instead of throwing — +> a journal-write failure (e.g. disk full) must never propagate into the worker, where +> it would be misclassified as a retryable error and re-run a completed upload. +> 3. `Entry` includes a `toWritableMap()` member (originally listed under Task 2) and an +> `errorKind` field. +> Two extra tests cover the cap and the non-throwing guarantee (8 tests total). + +- [ ] **Step 4: Implement `EventJournal`** + +Create `android/src/main/java/com/vydia/RNUploader/EventJournal.kt`: + +```kotlin +package com.vydia.RNUploader + +import android.content.Context +import com.google.gson.Gson +import java.io.File + +// Durable record of terminal upload events (completed / error / cancelled). +// Written BEFORE the event is emitted to JS; deleted only when JS acknowledges. +// One JSON file per event named .json — atomic-ish via tmp+rename. +class EventJournal(private val dir: File) { + + data class Entry( + val eventId: String, + val uploadId: String, + val type: String, // completed | error | cancelled + val timestamp: Long, + val responseCode: Int? = null, + val responseBody: String? = null, + val responseBodyTruncated: Boolean = false, + val responseHeaders: Map? = null, + val error: String? = null, + val errorKind: String? = null, // http | network | file | unknown + val cancelReason: String? = null, // user | system + ) + + companion object { + const val MAX_BODY_BYTES = 64 * 1024 + private val gson = Gson() + + @Volatile + private var instance: EventJournal? = null + + // Worker may run in a process where React never initialized, so the + // journal must be reachable from a bare Context, not the module. + fun get(context: Context): EventJournal = + instance ?: synchronized(this) { + instance + ?: EventJournal(File(context.filesDir, "rnbgupload-events")).also { instance = it } + } + } + + init { + dir.mkdirs() + } + + @Synchronized + fun append(entry: Entry) { + val body = entry.responseBody + val bounded = + if (body != null && body.length > MAX_BODY_BYTES) + entry.copy(responseBody = body.substring(0, MAX_BODY_BYTES), responseBodyTruncated = true) + else entry + val tmp = File(dir, "${entry.eventId}.tmp") + tmp.writeText(gson.toJson(bounded)) + tmp.renameTo(File(dir, "${entry.eventId}.json")) + } + + @Synchronized + fun unacknowledged(): List = + (dir.listFiles { f -> f.extension == "json" } ?: emptyArray()) + .mapNotNull { f -> + runCatching { gson.fromJson(f.readText(), Entry::class.java) }.getOrNull() + } + .filter { it.eventId != null } // corrupt-but-parseable guard + .sortedBy { it.timestamp } + + @Synchronized + fun ack(eventIds: List) { + eventIds.forEach { File(dir, "$it.json").delete() } + } +} +``` + +Note: the cap counts UTF-16 chars, not bytes — acceptable for a safety cap; do not "fix" it to byte-accurate splitting (risks cutting surrogate pairs). +Note: Gson instantiates the data class via `Unsafe`, so a parsed file missing `eventId` yields a null field despite the non-null type — hence the explicit `it.eventId != null` filter. + +- [ ] **Step 5: Run tests to verify they pass** + +Run: `cd example/RNBGUExample/android && ./gradlew :react-native-background-upload:testDebugUnitTest --tests '*EventJournalTest*'` +Expected: `BUILD SUCCESSFUL`, 6 tests passing. + +- [ ] **Step 6: Commit** + +```bash +git add android/src/main/java/com/vydia/RNUploader/EventJournal.kt \ + android/src/test/java/com/vydia/RNUploader/EventJournalTest.kt \ + android/build.gradle +git commit -m "feat(android): durable terminal-event journal with ack" +``` + +--- + +### Task 2: Journal wiring in `UploadWorker` + module `getUnacknowledgedEvents`/`ackEvents` + +**Files:** +- Modify: `android/src/main/java/com/vydia/RNUploader/UploadWorker.kt:153-171` +- Modify: `android/src/main/java/com/vydia/RNUploader/UploaderModule.kt` +- Modify: `android/src/main/java/com/vydia/RNUploader/EventReporter.kt` +- Modify: `android/src/main/java/com/vydia/RNUploader/Upload.kt` (acceptStatus option) + +**Interfaces:** +- Consumes: `EventJournal` from Task 1. +- Produces: React methods `getUnacknowledgedEvents(promise)` and `ackEvents(eventIds: ReadableArray, promise)`; `Upload.acceptStatus: List` option; outcome classification (`completed` = 2xx/acceptStatus, otherwise `error` with `errorKind`); `EventReporter.cancelled(uploadId, reason: String)`, `EventReporter.error(uploadId, error, kind)`, `EventReporter.httpError(uploadId, response)`; journal entries written before every terminal emit. Task 3 depends on the `reason` parameter. + +- [ ] **Step 1: Journal terminal outcomes in the worker** + +First, add the `acceptStatus` option to `Upload.kt` — a field on the data class plus its parsing in `fromReadableMap`: + +```kotlin + val acceptStatus: List, // add to the data class fields +``` + +```kotlin + // add to fromReadableMap + acceptStatus = map.getArray("acceptStatus")?.let { arr -> + (0 until arr.size()).map { i -> arr.getInt(i) } + } ?: listOf(), +``` + +Then in `UploadWorker.kt`, rename the `handleSuccess(response)` call inside `doWork` (line ~105) to `handleResponse(response)`, and replace `handleSuccess`, `handleError`, and `checkAndHandleCancellation` (lines 153–171) with: + +```kotlin + // HTTP response received. "completed" only for 2xx or per-request acceptStatus + // codes (axios validateStatus semantics — a 400 is an error, not a completion); + // anything else is a terminal error carrying the full response. + private fun handleResponse(response: UploadResponse) { + UploadProgress.complete(upload.id) + val accepted = response.code in 200..299 || upload.acceptStatus.contains(response.code) + EventJournal.get(context).append( + EventJournal.Entry( + eventId = java.util.UUID.randomUUID().toString(), + uploadId = upload.id, + type = if (accepted) "completed" else "error", + timestamp = System.currentTimeMillis(), + responseCode = response.code, + responseBody = response.body, + responseHeaders = response.headers, + errorKind = if (accepted) null else "http", + error = if (accepted) null else "HTTP ${response.code}", + ) + ) + if (accepted) EventReporter.success(upload.id, response) + else EventReporter.httpError(upload.id, response) + } + + private fun handleError(error: Throwable) { + UploadProgress.remove(upload.id) + val kind = when { + error is IOException && + runCatching { !File(upload.path).exists() }.getOrDefault(false) -> "file" + error is IOException -> "network" + else -> "unknown" + } + EventJournal.get(context).append( + EventJournal.Entry( + eventId = java.util.UUID.randomUUID().toString(), + uploadId = upload.id, + type = "error", + timestamp = System.currentTimeMillis(), + error = error.message ?: "Unknown exception", + errorKind = kind, + ) + ) + EventReporter.error(upload.id, error, kind) + } + + // Check if cancelled by user or new worker with same ID + // Worker won't rerun, perform teardown + private fun checkAndHandleCancellation(): Boolean { + if (!isStopped) return false + + val reason = if (UserCancellations.consume(upload.id)) "user" else "system" + UploadProgress.remove(upload.id) + EventJournal.get(context).append( + EventJournal.Entry( + eventId = java.util.UUID.randomUUID().toString(), + uploadId = upload.id, + type = "cancelled", + timestamp = System.currentTimeMillis(), + cancelReason = reason, + ) + ) + EventReporter.cancelled(upload.id, reason) + return true + } +``` + +(`UserCancellations` is created in Task 3; to keep this task compiling on its own, create the stub now — Task 3 fills in the caller.) + +Create `android/src/main/java/com/vydia/RNUploader/UserCancellations.kt`: + +```kotlin +package com.vydia.RNUploader + +// Upload ids the JS side explicitly cancelled. Consulted by the worker to +// distinguish user cancels from system kills. Same-process only: a user +// cancel always originates from live JS, so the set never needs to persist. +object UserCancellations { + private val ids = mutableSetOf() + + @Synchronized + fun mark(id: String) { + ids.add(id) + } + + @Synchronized + fun consume(id: String): Boolean = ids.remove(id) +} +``` + +- [ ] **Step 2: Add `reason` to the cancelled event** + +In `EventReporter.kt`, replace the `cancelled` and `error` functions and add `httpError`: + +```kotlin + fun cancelled(uploadId: String, reason: String) = + sendEvent("cancelled", Arguments.createMap().apply { + putString("id", uploadId) + putString("cancelReason", reason) + }) + + fun error(uploadId: String, exception: Throwable, kind: String) = + sendEvent("error", Arguments.createMap().apply { + putString("id", uploadId) + putString("error", exception.message ?: "Unknown exception") + putString("errorKind", kind) + }) + + fun httpError(uploadId: String, response: UploadResponse) = + sendEvent("error", Arguments.createMap().apply { + putString("id", uploadId) + putString("error", "HTTP ${response.code}") + putString("errorKind", "http") + putInt("responseCode", response.code) + putString("responseBody", response.body) + putMap("responseHeaders", Arguments.makeNativeMap(response.headers)) + }) +``` + +- [ ] **Step 3: Expose journal methods on the module** + +In `UploaderModule.kt`, add imports `com.facebook.react.bridge.Arguments`, `com.facebook.react.bridge.ReadableArray`, `com.facebook.react.bridge.WritableMap`, then add: + +```kotlin + @ReactMethod + fun getUnacknowledgedEvents(promise: Promise) { + try { + val events = EventJournal.get(reactApplicationContext).unacknowledged() + val arr = Arguments.createArray() + events.forEach { arr.pushMap(it.toWritableMap()) } + promise.resolve(arr) + } catch (exc: Throwable) { + Log.e(TAG, exc.message, exc) + promise.reject(exc) + } + } + + @ReactMethod + fun ackEvents(eventIds: ReadableArray, promise: Promise) { + try { + val ids = (0 until eventIds.size()).mapNotNull { eventIds.getString(it) } + EventJournal.get(reactApplicationContext).ack(ids) + promise.resolve(true) + } catch (exc: Throwable) { + Log.e(TAG, exc.message, exc) + promise.reject(exc) + } + } +``` + +And add the map conversion as a **member function of the `Entry` data class** in `EventJournal.kt` (a member — not an extension declared inside `EventJournal`, which would not resolve from `UploaderModule`'s scope): + +```kotlin + data class Entry( + /* …fields exactly as in Task 1… */ + ) { + fun toWritableMap(): com.facebook.react.bridge.WritableMap = + com.facebook.react.bridge.Arguments.createMap().apply { + putString("eventId", eventId) + putString("id", uploadId) + putString("type", type) + putDouble("timestamp", timestamp.toDouble()) + responseCode?.let { putInt("responseCode", it) } + responseBody?.let { putString("responseBody", it) } + putBoolean("responseBodyTruncated", responseBodyTruncated) + responseHeaders?.let { + putMap("responseHeaders", com.facebook.react.bridge.Arguments.makeNativeMap(it)) + } + error?.let { putString("error", it) } + errorKind?.let { putString("errorKind", it) } + cancelReason?.let { putString("cancelReason", it) } + } + } +``` + +This keeps the JUnit tests green: the JVM resolves the React classes lazily at first invocation, and the tests never call `toWritableMap`. + +- [ ] **Step 4: Build to verify compilation** + +Run: `cd example/RNBGUExample/android && ./gradlew :react-native-background-upload:assembleDebug` +Expected: `BUILD SUCCESSFUL`. + +- [ ] **Step 5: Re-run journal tests** + +Run: `cd example/RNBGUExample/android && ./gradlew :react-native-background-upload:testDebugUnitTest` +Expected: PASS (6 tests). + +- [ ] **Step 6: Commit** + +```bash +git add android/src/main/java/com/vydia/RNUploader/ +git commit -m "feat(android): journal terminal events; expose get/ack; cancel reason" +``` + +--- + +### Task 3: Android cancel intent, notification defaults + auto channel, remove `stopAllUploads` + +**Files:** +- Modify: `android/src/main/java/com/vydia/RNUploader/UploaderModule.kt:83-109` +- Modify: `android/src/main/java/com/vydia/RNUploader/Upload.kt` (notification fields get defaults) +- Modify: `android/src/main/java/com/vydia/RNUploader/UploadWorker.kt` (ensure channel) + +**Interfaces:** +- Consumes: `UserCancellations.mark(id)` from Task 2. +- Produces: `cancelUpload` marks user intent before cancelling; `stopAllUploads` deleted (never exposed in JS, no consumers — YAGNI); all five `notification*` options become optional with defaults; the library creates its notification channel itself (breaking: consumers no longer need notifee/channel plumbing — Task 9 makes the `android` block optional in TS). + +- [ ] **Step 1: Mark user cancels and delete dead method** + +In `UploaderModule.kt`, replace `cancelUpload` and delete `stopAllUploads` entirely: + +```kotlin + /* + * Cancels file upload + * Accepts upload ID as a first argument, this upload will be cancelled + * Event "cancelled" will be fired when upload is cancelled. + */ + @ReactMethod + fun cancelUpload(uploadId: String, promise: Promise) { + try { + UserCancellations.mark(uploadId) + workManager.cancelUniqueWork(uploadId) + promise.resolve(true) + } catch (exc: Throwable) { + exc.printStackTrace() + Log.e(TAG, exc.message, exc) + promise.reject(exc) + } + } +``` + +- [ ] **Step 2: Notification defaults + library-owned channel** + +In `Upload.kt`'s `fromReadableMap`, replace the five throwing `notification*` parses with defaults (`url` and `path` keep `MissingOptionException`): + +```kotlin + notificationId = (map.getString(Upload::notificationId.name) + ?: "react-native-background-upload").hashCode(), + notificationTitle = map.getString(Upload::notificationTitle.name) + ?: "Uploading…", + notificationTitleNoInternet = map.getString(Upload::notificationTitleNoInternet.name) + ?: "Waiting for connection…", + notificationTitleNoWifi = map.getString(Upload::notificationTitleNoWifi.name) + ?: "Waiting for Wi-Fi…", + notificationChannel = map.getString(Upload::notificationChannel.name) + ?: "background-upload", +``` + +In `UploadWorker.kt`, add (imports `android.app.NotificationChannel`) and call it in `doWork` immediately before `setForeground(getForegroundInfo())`: + +```kotlin + private fun ensureNotificationChannel() { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) return + // Idempotent per official docs: creating an existing channel is a no-op. + val channel = NotificationChannel( + upload.notificationChannel, + "Uploads", + NotificationManager.IMPORTANCE_LOW + ) + notificationManager.createNotificationChannel(channel) + } +``` + +Consumers that want a custom channel name/importance can still create the channel themselves first (creation elsewhere wins — same id is a no-op here) and pass `notificationChannel`. + +- [ ] **Step 3: Build** + +Run: `cd example/RNBGUExample/android && ./gradlew :react-native-background-upload:assembleDebug` +Expected: `BUILD SUCCESSFUL`. + +- [ ] **Step 4: Commit** + +```bash +git add android/src/main/java/com/vydia/RNUploader/ +git commit -m "feat(android): user-cancel intent; notification defaults + auto channel; drop stopAllUploads" +``` + +--- + +### Task 4: Android `getAllUploads()` + +**Files:** +- Modify: `android/src/main/java/com/vydia/RNUploader/UploaderModule.kt` (startUpload + new method) + +**Interfaces:** +- Consumes: WorkManager `getWorkInfosByTag` (verified: returns work in all states; finished records are pruned eventually, so the journal — not this call — is the source of truth for terminal outcomes). +- Produces: React method `getAllUploads(promise)` resolving `[{ id: string, state: 'pending'|'running'|'completed'|'error'|'cancelled' }]`; work requests tagged `"RNFileUploaderId:"`. + +- [ ] **Step 1: Tag work with the upload id** + +In `UploaderModule.kt` companion object add: + +```kotlin + const val ID_TAG_PREFIX = "RNFileUploaderId:" +``` + +In the private `startUpload`, add the id tag to the request builder: + +```kotlin + val request = OneTimeWorkRequestBuilder() + .addTag(WORKER_TAG) + .addTag(ID_TAG_PREFIX + upload.id) + .setInputData(workDataOf(UploadWorker.Input.Params.name to data)) + .build() +``` + +- [ ] **Step 2: Add the query method** + +Add to `UploaderModule.kt` (import `androidx.work.WorkInfo`): + +```kotlin + /** + * Enumerates uploads WorkManager still knows about. Finished work is + * auto-pruned by WorkManager after roughly ONE DAY — terminal outcomes must + * be read from getUnacknowledgedEvents(), not from this method. + */ + @ReactMethod + fun getAllUploads(promise: Promise) { + try { + val infos = workManager.getWorkInfosByTag(WORKER_TAG).get() + val arr = Arguments.createArray() + for (info in infos) { + val id = info.tags.firstOrNull { it.startsWith(ID_TAG_PREFIX) } + ?.removePrefix(ID_TAG_PREFIX) ?: continue + val map = Arguments.createMap() + map.putString("id", id) + map.putString( + "state", + when (info.state) { + WorkInfo.State.ENQUEUED, WorkInfo.State.BLOCKED -> "pending" + WorkInfo.State.RUNNING -> "running" + WorkInfo.State.SUCCEEDED -> "completed" + WorkInfo.State.FAILED -> "error" + WorkInfo.State.CANCELLED -> "cancelled" + } + ) + arr.pushMap(map) + } + promise.resolve(arr) + } catch (exc: Throwable) { + Log.e(TAG, exc.message, exc) + promise.reject(exc) + } + } +``` + +(`.get()` blocks the NativeModules thread briefly; the query is local-database-only. Acceptable — matches the existing module's synchronous style.) + +- [ ] **Step 3: Build** + +Run: `cd example/RNBGUExample/android && ./gradlew :react-native-background-upload:assembleDebug` +Expected: `BUILD SUCCESSFUL`. + +- [ ] **Step 4: Commit** + +```bash +git add android/src/main/java/com/vydia/RNUploader/UploaderModule.kt +git commit -m "feat(android): getAllUploads via id-tagged WorkManager query" +``` + +--- + +> **PIVOT (2026-07-23): iOS is now a Swift REWRITE, not edits to the Obj-C.** +> Decided with Dylan: the three Obj-C files (<500 lines) are old and hard to read, Diana +> already builds Swift under static linkage and runs new arch, and Dylan will own this. So +> Tasks 5–8 below are superseded — the Obj-C code blocks remain only as a behavior reference +> for what the Swift must reproduce. The rewrite: +> - `RNFileUploader.swift` — `@objc(VydiaRNFileUploader)` class : `RCTEventEmitter`, the two +> background `URLSession`s, the delegate, all exported methods, classification, throttle, +> bg-relaunch handler. `RNFileUploader.m` — ~30-line `RCT_EXTERN_MODULE`/`RCT_EXTERN_METHOD` +> shim (unavoidable on the bridge). `EventJournal.swift` + `TaskMap.swift` — `Codable`, +> with the cap / backup-exclusion / non-throwing-write mitigations. Delete the old +> `VydiaRNFileUploader.m`, `Helper.{h,m}`. +> - Keep the JS contract identical: module name `VydiaRNFileUploader`, events `RNFileUploader-*`. +> - Target iOS **15.1** (Diana's floor); podspec bumped from 9.0, `swift_version` + `React-Core` +> dependency + `DEFINES_MODULE` added; example app deployment target bumped to 15.1. +> - **Correctness invariant:** the background `NSURLSession` delegate API is load-bearing and +> has no modern replacement — port it verbatim (session ids, `discretionary=NO`, +> `HTTPMaximumConnectionsPerHost=1`, `waitsForConnectivity`, `uploadTask(with:fromFile:)`). +> Do NOT use `async` `URLSession.upload` (no background support). Journal stays synchronous +> (serial queue), NOT an actor, to preserve "journal before emit" ordering. +> - Verification: `xcodebuild` per checkpoint + on-device Task 11 (no plain-unit-test path). +> Started with an isolated Swift interop probe (`RNBGUProbe.{swift,m}`) to prove the pod/Swift +> setup compiles and registers before the real port; probe is deleted afterward. + +### Task 5: iOS `RNBGUEventJournal` + journal wiring + +**Files:** +- Create: `ios/RNBGUEventJournal.h` +- Create: `ios/RNBGUEventJournal.m` +- Create: `ios/RNBGUTaskMap.h` +- Create: `ios/RNBGUTaskMap.m` +- Modify: `ios/VydiaRNFileUploader.m:369-399` (didCompleteWithError) and `startUpload` + +**Interfaces:** +- Consumes: nothing new. +- Produces: `+[RNBGUEventJournal append:]` (takes `NSDictionary` containing `eventId`), `+[RNBGUEventJournal unacknowledged]` → `NSArray`, `+[RNBGUEventJournal ack:]`; `RNBGUTaskMap` persisted `sessionId:taskIdentifier → {id, acceptStatus}` sidecar (`setMeta:forKey:`, `metaForKey:`, `removeKey:`); uploader helpers `-taskMapKeyFor:task:`, `-uploadIdFor:task:`, `-acceptStatusFor:task:`; `didCompleteWithError` classifies (2xx/acceptStatus → completed, other HTTP → error with `errorKind: http`, transport → `errorKind: network`) and journals before emitting; completed events gain `eventId` + `responseHeaders`; cancelled events gain `cancelReason`. Tasks 6 and 8 use the helpers; Task 6 exports the journal to JS. + +**Why the task map exists:** Apple documents `taskDescription` only as an uninterpreted app string — there is NO guarantee it survives process death, and DTS guidance is to persist task metadata externally keyed by `taskIdentifier` (which is stable for a background session's tasks across relaunch, unique per session). `taskDescription` stays the primary id; the map is the durable fallback so a relaunch can never orphan an event under an unknown id. + +- [ ] **Step 1: Create the journal class** + +`ios/RNBGUEventJournal.h`: + +```objc +#import + +// Durable record of terminal upload events. Written before emitting to JS, +// deleted only on explicit acknowledgment from JS. One JSON file per event. +// Bounded by kMaxEntries (drops oldest) as a runaway guard; excluded from +// device backups. Mirrors the Android EventJournal contract. +@interface RNBGUEventJournal : NSObject ++ (void)append:(NSDictionary *)event; // event MUST contain @"eventId" ++ (NSArray *)unacknowledged; ++ (void)ack:(NSArray *)eventIds; +@end +``` + +`ios/RNBGUEventJournal.m`: + +```objc +#import "RNBGUEventJournal.h" + +static NSString *const kJournalDirName = @"RNFileUploaderEvents"; +// Runaway guard, mirrors Android's EventJournal.MAX_ENTRIES. Only fires if JS +// never drains the journal via ack; drops the oldest entries when exceeded. +static const NSUInteger kMaxEntries = 1000; + +@implementation RNBGUEventJournal + ++ (dispatch_queue_t)queue { + static dispatch_queue_t q; + static dispatch_once_t once; + dispatch_once(&once, ^{ + q = dispatch_queue_create("com.vydia.rnbgupload.journal", DISPATCH_QUEUE_SERIAL); + }); + return q; +} + ++ (NSURL *)dirURL { + NSURL *appSupport = [[NSFileManager.defaultManager URLsForDirectory:NSApplicationSupportDirectory + inDomains:NSUserDomainMask] firstObject]; + NSURL *dir = [appSupport URLByAppendingPathComponent:kJournalDirName isDirectory:YES]; + [NSFileManager.defaultManager createDirectoryAtURL:dir + withIntermediateDirectories:YES + attributes:nil + error:nil]; + // Application Support is backed up to iCloud/iTunes by default; the journal + // is transient device-local state, so exclude it from backups. + NSURL *mutableDir = dir; + [mutableDir setResourceValue:@YES forKey:NSURLIsExcludedFromBackupKey error:nil]; + return dir; +} + ++ (void)append:(NSDictionary *)event { + NSString *eventId = event[@"eventId"]; + if (!eventId) return; + dispatch_sync([self queue], ^{ + NSError *err = nil; + NSData *data = [NSJSONSerialization dataWithJSONObject:event options:0 error:&err]; + if (!data) return; + // writeToURL returns BOOL (never throws), so a failed journal write can + // never propagate into the upload delegate and re-trigger the upload. + NSURL *file = [[self dirURL] URLByAppendingPathComponent:[eventId stringByAppendingString:@".json"]]; + [data writeToURL:file atomically:YES]; + [self pruneToMax]; + }); +} + +// Caller already holds the serial queue. Prune oldest by content-modification +// date beyond kMaxEntries. ++ (void)pruneToMax { + NSArray *keys = @[NSURLContentModificationDateKey]; + NSArray *files = [NSFileManager.defaultManager contentsOfDirectoryAtURL:[self dirURL] + includingPropertiesForKeys:keys + options:0 + error:nil]; + NSMutableArray *jsonFiles = [NSMutableArray array]; + for (NSURL *f in files) { + if ([f.pathExtension isEqualToString:@"json"]) [jsonFiles addObject:f]; + } + if (jsonFiles.count <= kMaxEntries) return; + [jsonFiles sortUsingComparator:^NSComparisonResult(NSURL *a, NSURL *b) { + NSDate *da = nil, *db = nil; + [a getResourceValue:&da forKey:NSURLContentModificationDateKey error:nil]; + [b getResourceValue:&db forKey:NSURLContentModificationDateKey error:nil]; + return [(da ?: NSDate.distantPast) compare:(db ?: NSDate.distantPast)]; + }]; + NSUInteger toRemove = jsonFiles.count - kMaxEntries; + for (NSUInteger i = 0; i < toRemove; i++) { + [NSFileManager.defaultManager removeItemAtURL:jsonFiles[i] error:nil]; + } +} + ++ (NSArray *)unacknowledged { + __block NSMutableArray *events = [NSMutableArray array]; + dispatch_sync([self queue], ^{ + NSArray *files = [NSFileManager.defaultManager contentsOfDirectoryAtURL:[self dirURL] + includingPropertiesForKeys:nil + options:0 + error:nil]; + for (NSURL *file in files) { + if (![file.pathExtension isEqualToString:@"json"]) continue; + NSData *data = [NSData dataWithContentsOfURL:file]; + if (!data) continue; + id parsed = [NSJSONSerialization JSONObjectWithData:data options:0 error:nil]; + if ([parsed isKindOfClass:[NSDictionary class]] && parsed[@"eventId"]) { + [events addObject:parsed]; + } + } + [events sortUsingComparator:^NSComparisonResult(NSDictionary *a, NSDictionary *b) { + return [(a[@"timestamp"] ?: @0) compare:(b[@"timestamp"] ?: @0)]; + }]; + }); + return events; +} + ++ (void)ack:(NSArray *)eventIds { + dispatch_sync([self queue], ^{ + for (NSString *eventId in eventIds) { + NSURL *file = [[self dirURL] URLByAppendingPathComponent:[eventId stringByAppendingString:@".json"]]; + [NSFileManager.defaultManager removeItemAtURL:file error:nil]; + } + }); +} + +@end +``` + +- [ ] **Step 2: Create the persisted task map** + +`ios/RNBGUTaskMap.h`: + +```objc +#import + +// Durable sessionId:taskIdentifier -> uploadId mapping. taskDescription has no +// documented persistence guarantee across process death; this map does. +@interface RNBGUTaskMap : NSObject ++ (void)setMeta:(NSDictionary *)meta forKey:(NSString *)key; // meta: {id, acceptStatus} ++ (NSDictionary * _Nullable)metaForKey:(NSString *)key; ++ (void)removeKey:(NSString *)key; +@end +``` + +`ios/RNBGUTaskMap.m`: + +```objc +#import "RNBGUTaskMap.h" + +static NSString *const kTaskMapFile = @"RNFileUploaderTaskMap.json"; + +@implementation RNBGUTaskMap + ++ (dispatch_queue_t)queue { + static dispatch_queue_t q; + static dispatch_once_t once; + dispatch_once(&once, ^{ + q = dispatch_queue_create("com.vydia.rnbgupload.taskmap", DISPATCH_QUEUE_SERIAL); + }); + return q; +} + ++ (NSURL *)fileURL { + NSURL *appSupport = [[NSFileManager.defaultManager URLsForDirectory:NSApplicationSupportDirectory + inDomains:NSUserDomainMask] firstObject]; + [NSFileManager.defaultManager createDirectoryAtURL:appSupport + withIntermediateDirectories:YES + attributes:nil + error:nil]; + return [appSupport URLByAppendingPathComponent:kTaskMapFile]; +} + +// Callers hold the serial queue via dispatch_sync; these two are queue-private. ++ (NSMutableDictionary *)readMap { + NSData *data = [NSData dataWithContentsOfURL:[self fileURL]]; + if (!data) return [NSMutableDictionary dictionary]; + id parsed = [NSJSONSerialization JSONObjectWithData:data options:0 error:nil]; + return [parsed isKindOfClass:[NSDictionary class]] + ? [parsed mutableCopy] + : [NSMutableDictionary dictionary]; +} + ++ (void)writeMap:(NSDictionary *)map { + NSData *data = [NSJSONSerialization dataWithJSONObject:map options:0 error:nil]; + if (data) [data writeToURL:[self fileURL] atomically:YES]; +} + ++ (void)setMeta:(NSDictionary *)meta forKey:(NSString *)key { + if (!meta || !key) return; + dispatch_sync([self queue], ^{ + NSMutableDictionary *map = [self readMap]; + map[key] = meta; + [self writeMap:map]; + }); +} + ++ (NSDictionary *)metaForKey:(NSString *)key { + __block NSDictionary *result = nil; + dispatch_sync([self queue], ^{ + id value = [self readMap][key]; + if ([value isKindOfClass:[NSDictionary class]]) result = value; + }); + return result; +} + ++ (void)removeKey:(NSString *)key { + dispatch_sync([self queue], ^{ + NSMutableDictionary *map = [self readMap]; + [map removeObjectForKey:key]; + [self writeMap:map]; + }); +} + +@end +``` + +Add the uploader helpers to `VydiaRNFileUploader.m` (with `#import "RNBGUTaskMap.h"`): + +```objc +- (NSString *)taskMapKeyFor:(NSURLSession *)session task:(NSURLSessionTask *)task { + return [NSString stringWithFormat:@"%@:%lu", + session.configuration.identifier ?: @"", + (unsigned long)task.taskIdentifier]; +} + +// taskDescription is the primary id; the persisted map is the fallback for +// tasks observed after a relaunch where taskDescription may not have survived. +- (NSString *)uploadIdFor:(NSURLSession *)session task:(NSURLSessionTask *)task { + return task.taskDescription + ?: [RNBGUTaskMap metaForKey:[self taskMapKeyFor:session task:task]][@"id"] + ?: @"unknown"; +} + +- (NSArray *)acceptStatusFor:(NSURLSession *)session task:(NSURLSessionTask *)task { + NSArray *accept = [RNBGUTaskMap metaForKey:[self taskMapKeyFor:session task:task]][@"acceptStatus"]; + return [accept isKindOfClass:[NSArray class]] ? accept : @[]; +} +``` + +And record the mapping in `startUpload`, immediately after the taskDescription assignment: + +```objc + [RNBGUTaskMap setMeta:@{ + @"id": uploadTask.taskDescription, + @"acceptStatus": options[@"acceptStatus"] ?: @[], + } forKey:[self taskMapKeyFor:session task:uploadTask]]; +``` + +(The map persists `acceptStatus` alongside the id so classification still works for tasks that complete after a relaunch, when the original `startUpload` options are gone.) + +- [ ] **Step 3: Journal terminal events in `didCompleteWithError`** + +In `VydiaRNFileUploader.m`, add `#import "RNBGUEventJournal.h"` and a static set near the other statics: + +```objc +static NSMutableSet *_userCancelledIds = nil; +``` + +Initialize it in `init` alongside `_responsesData`: + +```objc + _userCancelledIds = [NSMutableSet set]; +``` + +Replace the entire `didCompleteWithError` delegate method (lines 369–399) with: + +```objc +- (void)URLSession:(NSURLSession *)session + task:(NSURLSessionTask *)task +didCompleteWithError:(NSError *)error { + NSString *uploadId = [self uploadIdFor:session task:task]; + NSMutableDictionary *data = [NSMutableDictionary dictionaryWithObjectsAndKeys:uploadId, @"id", nil]; + NSURLSessionDataTask *uploadTask = (NSURLSessionDataTask *)task; + NSHTTPURLResponse *response = (NSHTTPURLResponse *)uploadTask.response; + if (response != nil) { + [data setObject:[NSNumber numberWithInteger:response.statusCode] forKey:@"responseCode"]; + NSMutableDictionary *headers = [NSMutableDictionary dictionary]; + [response.allHeaderFields enumerateKeysAndObjectsUsingBlock:^(id key, id val, BOOL *stop) { + headers[[key description]] = [val description]; + }]; + [data setObject:headers forKey:@"responseHeaders"]; + } + //Add data that was collected earlier by the didReceiveData method + NSMutableData *responseData = _responsesData[@(task.taskIdentifier)]; + if (responseData) { + [_responsesData removeObjectForKey:@(task.taskIdentifier)]; + NSString *body = [[NSString alloc] initWithData:responseData encoding:NSUTF8StringEncoding] ?: @""; + // Journal cap: 64KB, flag truncation + static const NSUInteger kMaxBodyChars = 64 * 1024; + if (body.length > kMaxBodyChars) { + [data setObject:[body substringToIndex:kMaxBodyChars] forKey:@"responseBody"]; + [data setObject:@YES forKey:@"responseBodyTruncated"]; + } else { + [data setObject:body forKey:@"responseBody"]; + } + } else { + [data setObject:[NSNull null] forKey:@"responseBody"]; + } + + [data setObject:[[NSUUID UUID] UUIDString] forKey:@"eventId"]; + [data setObject:@([[NSDate date] timeIntervalSince1970] * 1000) forKey:@"timestamp"]; + + NSString *eventName; + if (error == nil) { + // "completed" only for 2xx or per-request acceptStatus codes (axios + // validateStatus semantics); other HTTP responses are terminal errors. + NSInteger status = response ? response.statusCode : 0; + NSArray *accept = [self acceptStatusFor:session task:task]; + if ((status >= 200 && status < 300) || [accept containsObject:@(status)]) { + eventName = @"completed"; + } else { + eventName = @"error"; + [data setObject:@"http" forKey:@"errorKind"]; + [data setObject:[NSString stringWithFormat:@"HTTP %ld", (long)status] forKey:@"error"]; + } + } else { + [data setObject:error.localizedDescription forKey:@"error"]; + if (error.code == NSURLErrorCancelled) { + eventName = @"cancelled"; + BOOL userCancelled; + @synchronized (_userCancelledIds) { + userCancelled = [_userCancelledIds containsObject:uploadId]; + [_userCancelledIds removeObject:uploadId]; + } + [data setObject:(userCancelled ? @"user" : @"system") forKey:@"cancelReason"]; + } else { + eventName = @"error"; + [data setObject:@"network" forKey:@"errorKind"]; + } + } + + // Journal BEFORE emitting: the emit is best-effort (JS may be dead), + // the journal entry is the durable record until JS acks it. + NSMutableDictionary *journalEntry = [data mutableCopy]; + [journalEntry setObject:eventName forKey:@"type"]; + if (journalEntry[@"responseBody"] == [NSNull null]) [journalEntry removeObjectForKey:@"responseBody"]; + [RNBGUEventJournal append:journalEntry]; + [RNBGUTaskMap removeKey:[self taskMapKeyFor:session task:task]]; + + [self _sendEventWithName:[@"RNFileUploader-" stringByAppendingString:eventName] body:data]; +} +``` + +- [ ] **Step 4: Build the example app** + +Run: `cd example/RNBGUExample && yarn install && (cd ios && pod install) && yarn ios` (builds and launches on a simulator; alternatively build `ios/RNBGUExample.xcworkspace` in Xcode). +Expected: build succeeds. New files are picked up by the podspec glob `ios/*.{h,m}` — no pbxproj edits needed for consumers; `pod install` regenerates the pod target. + +- [ ] **Step 5: Commit** + +```bash +git add ios/RNBGUEventJournal.h ios/RNBGUEventJournal.m ios/RNBGUTaskMap.h ios/RNBGUTaskMap.m ios/VydiaRNFileUploader.m +git commit -m "feat(ios): journal terminal events; persisted task map; response headers; cancel reason" +``` + +--- + +### Task 6: iOS `getUnacknowledgedEvents` / `ackEvents` / `getAllUploads` exports + +**Files:** +- Modify: `ios/VydiaRNFileUploader.m` (new RCT_EXPORT_METHODs; extend `cancelUpload`) + +**Interfaces:** +- Consumes: `RNBGUEventJournal` (Task 5), `_userCancelledIds` (Task 5). +- Produces: `getUnacknowledgedEvents()`, `ackEvents(ids)`, `getAllUploads()` — same JS names as Android (Task 2/4). `getAllUploads` returns live session tasks only: `[{ id, state: 'running'|'pending', bytesSent, totalBytes }]`. + +- [ ] **Step 1: Add the export methods** + +Add to `VydiaRNFileUploader.m`: + +```objc +RCT_EXPORT_METHOD(getUnacknowledgedEvents:(RCTPromiseResolveBlock)resolve reject:(RCTPromiseRejectBlock)reject) { + resolve([RNBGUEventJournal unacknowledged]); +} + +RCT_EXPORT_METHOD(ackEvents:(NSArray *)eventIds resolve:(RCTPromiseResolveBlock)resolve reject:(RCTPromiseRejectBlock)reject) { + [RNBGUEventJournal ack:eventIds]; + resolve(@YES); +} + +/* + * Enumerates upload tasks the OS sessions still know about (running/suspended). + * Terminal outcomes are NOT visible here — read getUnacknowledgedEvents for those. + */ +RCT_EXPORT_METHOD(getAllUploads:(RCTPromiseResolveBlock)resolve reject:(RCTPromiseRejectBlock)reject) { + NSMutableArray *sessions = [NSMutableArray array]; + if (_urlSession) [sessions addObject:_urlSession]; + if (_wifiOnlyUrlSession) [sessions addObject:_wifiOnlyUrlSession]; + + NSMutableArray *result = [NSMutableArray array]; + dispatch_group_t group = dispatch_group_create(); + + for (NSURLSession *session in sessions) { + dispatch_group_enter(group); + [session getTasksWithCompletionHandler:^(NSArray *dataTasks, NSArray *uploadTasks, NSArray *downloadTasks) { + for (NSURLSessionTask *task in uploadTasks) { + NSString *taskUploadId = [self uploadIdFor:session task:task]; + if ([taskUploadId isEqualToString:@"unknown"]) continue; + NSString *state = task.state == NSURLSessionTaskStateRunning ? @"running" : @"pending"; + @synchronized (result) { + [result addObject:@{ + @"id": taskUploadId, + @"state": state, + @"bytesSent": @(task.countOfBytesSent), + @"totalBytes": @(task.countOfBytesExpectedToSend), + }]; + } + } + dispatch_group_leave(group); + }]; + } + + dispatch_group_notify(group, dispatch_get_main_queue(), ^{ + resolve(result); + }); +} +``` + +- [ ] **Step 2: Track user cancels in `cancelUpload` and fix its task matching** + +At the top of the existing `cancelUpload` method body (before the sessions loop), add: + +```objc + @synchronized (_userCancelledIds) { + [_userCancelledIds addObject:cancelUploadId]; + } +``` + +In the same method, the matching line inside the tasks loop compares `taskDescription` directly — a task observed after a relaunch may have lost it. Replace: + +```objc + if ([uploadTask.taskDescription isEqualToString:cancelUploadId]){ +``` + +with: + +```objc + if ([[self uploadIdFor:session task:uploadTask] isEqualToString:cancelUploadId]){ +``` + +- [ ] **Step 3: Build** + +Same command as Task 5 Step 3. Expected: build succeeds. + +- [ ] **Step 4: Commit** + +```bash +git add ios/VydiaRNFileUploader.m +git commit -m "feat(ios): expose journal get/ack and getAllUploads; track user cancels" +``` + +--- + +### Task 7: iOS background-relaunch completion handler + +**Files:** +- Create: `ios/VydiaRNFileUploader.h` +- Modify: `ios/VydiaRNFileUploader.m` (class extension removal + delegate method + static handler store) +- Modify: `README.md` (AppDelegate instructions — full text in Task 10) + +**Interfaces:** +- Consumes: nothing new. +- Produces: `+[VydiaRNFileUploader setBackgroundSessionCompletionHandler:forIdentifier:]` for AppDelegates; `URLSessionDidFinishEventsForBackgroundURLSession:` implemented. Session identifiers in play: `ReactNativeBackgroundUpload`, `ReactNativeBackgroundUpload_WifiOnly`. + +- [ ] **Step 1: Create the public header** + +`ios/VydiaRNFileUploader.h`: + +```objc +#import +#import +#import + +@interface VydiaRNFileUploader : RCTEventEmitter + +/** + * Call from AppDelegate's application:handleEventsForBackgroundURLSession:completionHandler: + * so iOS can relaunch the app for uploads that finish while it is terminated. + * The handler is invoked after all pending session events have been delivered + * (and journaled), on the main queue. + */ ++ (void)setBackgroundSessionCompletionHandler:(void (^)(void))handler + forIdentifier:(NSString *)identifier; + +@end +``` + +In `VydiaRNFileUploader.m`, delete the inline `@interface VydiaRNFileUploader : RCTEventEmitter …` block (lines 8–9) and replace with `#import "VydiaRNFileUploader.h"`. + +- [ ] **Step 2: Store and fire the handler** + +In `VydiaRNFileUploader.m`, add near the other statics: + +```objc +static NSMutableDictionary *_bgCompletionHandlers = nil; +``` + +Add the class method and delegate method: + +```objc ++ (void)setBackgroundSessionCompletionHandler:(void (^)(void))handler + forIdentifier:(NSString *)identifier { + @synchronized (self) { + if (!_bgCompletionHandlers) _bgCompletionHandlers = [NSMutableDictionary dictionary]; + _bgCompletionHandlers[identifier] = [handler copy]; + } +} + +- (void)URLSessionDidFinishEventsForBackgroundURLSession:(NSURLSession *)session { + NSString *identifier = session.configuration.identifier; + if (!identifier) return; + void (^handler)(void); + @synchronized ([self class]) { + handler = _bgCompletionHandlers[identifier]; + [_bgCompletionHandlers removeObjectForKey:identifier]; + } + if (handler) { + dispatch_async(dispatch_get_main_queue(), handler); + } +} +``` + +Known limitation to note in the README (Task 10): the sessions are recreated inside the module's `init`, which runs when the React bridge starts. RN apps initialize the bridge during a background relaunch, so events are delivered and journaled; if a host app defers bridge startup in background, events wait in `nsurlsessiond` until the next real launch and are journaled then. Nothing is lost either way — the journal is the contract. + +- [ ] **Step 3: Build** + +Same build as Task 5 Step 3. Expected: build succeeds. + +- [ ] **Step 4: Commit** + +```bash +git add ios/VydiaRNFileUploader.h ios/VydiaRNFileUploader.m +git commit -m "feat(ios): background-relaunch completion handler support" +``` + +--- + +### Task 8: iOS progress throttle, UUID ids, dead-code removal + +**Files:** +- Modify: `ios/VydiaRNFileUploader.m` + +**Interfaces:** +- Consumes: nothing new. +- Produces: progress events throttled to 500ms per task (parity with Android); default upload ids are UUIDs; `startUpload` loses the multipart/PHAsset/appGroup branches (never functional for background use; TS types already forbid them). + +- [ ] **Step 1: Throttle progress events** + +Add near the other statics: + +```objc +static NSMutableDictionary *_lastProgressAt = nil; +``` + +Initialize in `init`: `_lastProgressAt = [NSMutableDictionary dictionary];` + +Replace the body of `didSendBodyData` (lines 401–418) with: + +```objc + float progress = -1; + if (totalBytesExpectedToSend > 0) { + progress = 100.0 * (float)totalBytesSent / (float)totalBytesExpectedToSend; + } + + // Throttle to 500ms per task (parity with Android). Always deliver 100%. + NSString *taskId = [self uploadIdFor:session task:task]; + NSTimeInterval now = [[NSDate date] timeIntervalSince1970]; + @synchronized (_lastProgressAt) { + NSNumber *last = _lastProgressAt[taskId]; + if (progress < 100 && last && now - last.doubleValue < 0.5) return; + _lastProgressAt[taskId] = @(now); + } + + NSDictionary *data = @{ @"id": taskId, @"progress": [NSNumber numberWithFloat:progress] }; + [self _sendEventWithName:@"RNFileUploader-progress" body:data]; +``` + +Also remove the task's entry in `didCompleteWithError` — add one line right after the `data` dictionary is created: + +```objc + @synchronized (_lastProgressAt) { [_lastProgressAt removeObjectForKey:uploadId]; } +``` + +(`uploadId` is the resolved id local added to `didCompleteWithError` in Task 5.) + +- [ ] **Step 2: UUID default ids** + +In `startUpload`, delete the `static int uploadId` counter usage (lines 124–128 and 201) and replace the taskDescription assignment: + +```objc + uploadTask.taskDescription = customUploadId ?: [[NSUUID UUID] UUIDString]; +``` + +Delete the `static int uploadId = 0;` declaration (line 16). + +- [ ] **Step 3: Delete dead branches** + +In `startUpload`: +- Delete the `assets-library` copy block (lines 160–174) and the `copyAssetToFile:` method (lines 81–109) and the `#import `. +- Delete the multipart branch: the `if ([uploadType isEqualToString:@"multipart"]) { … } else { … }` collapses to the raw path only; reject non-raw types explicitly: + +```objc + if (uploadType != nil && ![uploadType isEqualToString:@"raw"]) { + return reject(@"RN Uploader", @"Only type: 'raw' is supported", nil); + } + if (parameters.count > 0) { + return reject(@"RN Uploader", @"'parameters' requires multipart, which is not supported", nil); + } + NSURLSessionDataTask *uploadTask = (NSURLSessionDataTask *)[session uploadTaskWithRequest:request + fromFile:[NSURL URLWithString:fileURI]]; +``` + +- Delete `createBodyWithBoundary:…` (lines 281–319), `guessMIMETypeFromFileName:` (lines 67–79), the `#import `, and the `appGroup` handling (lines 136, 179–181 — it was a no-op: NSURLSession copies its configuration at creation, so mutating it later never had any effect). +- Delete the now-unused local variables (`fieldName`, `appGroup`, `parameters` stays only for the rejection check above). + +- [ ] **Step 4: Build and smoke-test** + +Build as in Task 5 Step 3, then run the example app on a simulator, press Upload against `example/server` (start it with `yarn --cwd example/server install && yarn --cwd example/server start`; it listens on port **3000**. Set `UPLOAD_URL` in `example/RNBGUExample/App.tsx` to `http://localhost:3000/upload`). +Expected: progress fires at ~2/sec max; completed event includes `eventId`, `responseHeaders`. + +- [ ] **Step 5: Commit** + +```bash +git add ios/VydiaRNFileUploader.m +git commit -m "feat(ios): throttle progress, UUID ids, remove dead multipart/PHAsset/appGroup code" +``` + +--- + +### Task 9: JS API surface + types + Jest harness + +**Files:** +- Modify: `src/index.ts`, `src/types.ts` +- Create: `jest.config.js`, `babel.config.js`, `src/__tests__/index.test.ts` +- Modify: `package.json` (test script + devDeps) + +**Interfaces:** +- Consumes: native methods from Tasks 2/4/6 (`getUnacknowledgedEvents`, `ackEvents`, `getAllUploads`). +- Produces (public API, all additive): + +```ts +export interface JournaledEvent { + eventId: string; + id: string; // upload id + type: 'completed' | 'error' | 'cancelled'; + timestamp: number; + responseCode?: number; + responseBody?: string; + responseBodyTruncated?: boolean; + responseHeaders?: Record; + error?: string; + errorKind?: 'http' | 'network' | 'file' | 'unknown'; + cancelReason?: 'user' | 'system'; +} +export interface UploadSnapshot { + id: string; + state: 'pending' | 'running' | 'completed' | 'error' | 'cancelled'; + bytesSent?: number; // iOS only + totalBytes?: number; // iOS only +} +getUnacknowledgedEvents(): Promise +ackEvents(eventIds: string[]): Promise +getAllUploads(): Promise +// CompletedData gains: responseHeaders?: Record; eventId?: string +// cancelled EventData gains: cancelReason?: 'user' | 'system' +// ErrorData gains: errorKind + responseCode/responseBody/responseHeaders (http errors) +// UploadOptions gains: acceptStatus?: number[]; android becomes optional +``` + +- [ ] **Step 1: Add Jest tooling** + +In `package.json` devDependencies add `"jest": "^29.7.0"`, `"babel-jest": "^29.7.0"`, `"@babel/preset-typescript": "^7.24.0"` and a script `"test": "jest"`. + +Create `babel.config.js` (root; the example app has its own config and Metro reads from the example dir, so this only affects Jest): + +```js +module.exports = { + presets: [['@babel/preset-env', {targets: {node: 'current'}}], '@babel/preset-typescript'], +}; +``` + +Create `jest.config.js`: + +```js +module.exports = { + testEnvironment: 'node', + testMatch: ['**/__tests__/**/*.test.ts'], +}; +``` + +Run: `yarn install` + +- [ ] **Step 2: Write the failing test** + +Create `src/__tests__/index.test.ts`: + +```ts +// Jest only allows out-of-scope variables in mock factories when their names +// start with "mock" — do not rename these. +const mockStartUpload = jest.fn(async () => 'id-1'); +const mockGetUnacknowledgedEvents = jest.fn(async () => [ + {eventId: 'e1', id: 'u1', type: 'completed', timestamp: 1, responseCode: 200}, +]); +const mockAckEvents = jest.fn(async () => true); +const mockGetAllUploads = jest.fn(async () => [{id: 'u1', state: 'running'}]); + +jest.mock('react-native', () => ({ + Platform: {OS: 'ios'}, + DeviceEventEmitter: {addListener: jest.fn()}, + NativeModules: { + VydiaRNFileUploader: { + addListener: jest.fn(), + startUpload: mockStartUpload, + getUnacknowledgedEvents: mockGetUnacknowledgedEvents, + ackEvents: mockAckEvents, + getAllUploads: mockGetAllUploads, + }, + }, +})); + +import Upload from '../index'; + +describe('journal + query API', () => { + it('exposes getUnacknowledgedEvents', async () => { + const events = await Upload.getUnacknowledgedEvents(); + expect(events[0].eventId).toBe('e1'); + }); + + it('exposes ackEvents', async () => { + await Upload.ackEvents(['e1']); + expect(mockAckEvents).toHaveBeenCalledWith(['e1']); + }); + + it('exposes getAllUploads', async () => { + const uploads = await Upload.getAllUploads(); + expect(uploads[0].state).toBe('running'); + }); + + it('still prefixes file paths in startUpload', async () => { + await Upload.startUpload({ + url: 'https://x', + path: '/tmp/f.bin', + method: 'POST', + type: 'raw', + android: { + notificationId: 'n', + notificationTitle: 't', + notificationTitleNoWifi: 'w', + notificationTitleNoInternet: 'i', + notificationChannel: 'c', + }, + }); + expect(mockStartUpload).toHaveBeenCalledWith( + expect.objectContaining({path: 'file:///tmp/f.bin'}), + ); + }); +}); +``` + +- [ ] **Step 3: Run to verify failure** + +Run: `yarn test` +Expected: FAIL — `Upload.getUnacknowledgedEvents is not a function`. + +- [ ] **Step 4: Implement the JS surface** + +In `src/types.ts` add (and extend `CompletedData`/the cancelled overload): + +```ts +export interface JournaledEvent { + eventId: string; + id: string; + type: 'completed' | 'error' | 'cancelled'; + timestamp: number; + responseCode?: number; + responseBody?: string; + responseBodyTruncated?: boolean; + responseHeaders?: Record; + error?: string; + errorKind?: 'http' | 'network' | 'file' | 'unknown'; + cancelReason?: 'user' | 'system'; +} + +export interface UploadSnapshot { + id: string; + state: 'pending' | 'running' | 'completed' | 'error' | 'cancelled'; + bytesSent?: number; + totalBytes?: number; +} + +export interface CancelledData extends EventData { + cancelReason?: 'user' | 'system'; +} +``` + +Replace `ErrorData` with (HTTP errors now arrive here, carrying the full response): + +```ts +export interface ErrorData extends EventData { + error: string; + errorKind?: 'http' | 'network' | 'file' | 'unknown'; + responseCode?: number; + responseBody?: string; + responseHeaders?: Record; +} +``` + +In `UploadOptions`, add `acceptStatus` and make the `android` block optional (all its fields optional too — native now supplies defaults and creates the channel): + +```ts + /** + * Non-2xx HTTP statuses to treat as successful completion, e.g. [409] when + * retries make duplicate-create conflicts expected. Everything else non-2xx + * emits an 'error' event with errorKind 'http' and the response attached. + */ + acceptStatus?: number[]; + android?: Partial; +``` + +Change `CompletedData` to: + +```ts +export interface CompletedData extends EventData { + eventId?: string; + responseCode: number; + responseBody: string; + responseHeaders?: Record; +} +``` + +Change the `cancelled` overload in `AddListener` to use `CancelledData`. + +In `src/index.ts` add: + +```ts +/** + * Terminal events (completed/error/cancelled) are journaled natively before + * being emitted, so they survive the app being killed. Read them on startup, + * process them, then acknowledge — unacked events are re-delivered here forever. + */ +const getUnacknowledgedEvents = (): Promise => + NativeModule.getUnacknowledgedEvents(); + +const ackEvents = (eventIds: string[]): Promise => + NativeModule.ackEvents(eventIds); + +/** + * Uploads the OS still knows about. On iOS this is live session tasks only; + * terminal outcomes come from getUnacknowledgedEvents, not from here. + */ +const getAllUploads = (): Promise => NativeModule.getAllUploads(); +``` + +Import the new types, and replace the default export with: + +```ts +export default { + startUpload, + cancelUpload, + addListener, + getUnacknowledgedEvents, + ackEvents, + getAllUploads, + ios, + android, +}; +``` + +- [ ] **Step 5: Run tests + typecheck** + +Run: `yarn test && yarn tsc --noEmit` +Expected: 4 tests PASS; typecheck clean. (The `typecheck` script replaces `build` in Task 10.) + +- [ ] **Step 6: Commit** + +```bash +git add src/ jest.config.js babel.config.js package.json yarn.lock +git commit -m "feat(js): journal get/ack, getAllUploads, typed errors, acceptStatus" +``` + +--- + +### Task 10: Packaging, CI, README, CHANGELOG + +**Files:** +- Modify: `package.json`, `.github/workflows/node.yml`, `README.md`, `CHANGELOG.md` + +**Interfaces:** +- Consumes: everything prior. +- Produces: v7.6.0; CI runs lint + jest + android unit tests + a `lib/` drift check. + +- [ ] **Step 1: Version + prepack safety** + +In `package.json`: set `"version": "8.0.0"`. Delete the committed `lib/` directory (`git rm -r lib`) — it's a hand-regenerated build artifact that has already drifted once. Metro consumes TS source directly (`"main": "src/index"`), and TypeScript reads types from source too: set `"typings": "src/index.ts"` and drop the `tsc-alias` build in favor of `"typecheck": "tsc --noEmit"` (nothing consumes emitted JS anymore). Diana's one deep import (`react-native-background-upload/lib/types` in `fileTransfers/models.ts`) moves to the root import on adoption. + +- [ ] **Step 2: CI** + +Replace the `node-lint-tests` job steps in `.github/workflows/node.yml` after "install node_modules" with: + +```yaml + - name: node lint + run: yarn lint:ci + + - name: js tests + run: yarn test + + - name: typecheck + run: yarn typecheck + + - name: setup java + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 17 + + - name: android unit tests + run: | + cd example/RNBGUExample/android + ./gradlew :react-native-background-upload:testDebugUnitTest +``` + +- [ ] **Step 3: README** + +Make these edits (keep the rest): +- Intro: replace "on Android it uses CoroutineWorker and Ktor" with "on Android it uses WorkManager (CoroutineWorker) and OkHttp". +- Delete the "Multipart Uploads — COMING SOON" section, the `useUtf8Charset` row, the `field`/`parameters`/`appGroup` rows, and the stale `notification` object table. Replace the notification table with: + +```markdown +### `android` options (required) + +| Name | Type | Required | Description | +| --- | --- | --- | --- | +| `notificationChannel` | string | Yes | Existing notification channel id (create it with notifee or similar before uploading) | +| `notificationId` | string | Yes | Stable id for the progress notification | +| `notificationTitle` | string | Yes | Title while uploading | +| `notificationTitleNoWifi` | string | Yes | Title while waiting for wifi (`wifiOnly: true`) | +| `notificationTitleNoInternet` | string | Yes | Title while waiting for connectivity | +| `maxRetries` | number | No (default 5) | Retry budget for non-network errors | +``` +- Add an **"iOS Setup (AppDelegate)"** section: + +````markdown +### iOS: background completion handler (required) + +Add to your `AppDelegate.mm` so uploads that finish while the app is terminated +relaunch it and get journaled: + +```objc +#import + +- (void)application:(UIApplication *)application +handleEventsForBackgroundURLSession:(NSString *)identifier + completionHandler:(void (^)(void))completionHandler { + [VydiaRNFileUploader setBackgroundSessionCompletionHandler:completionHandler + forIdentifier:identifier]; +} +``` +```` + +- Add a **"Reliable delivery (v7.6+)"** section: + +````markdown +## Reliable delivery (v7.6+) + +Terminal events (`completed` / `error` / `cancelled`) are journaled natively +*before* being emitted, so they survive app death, JS reloads, and background +relaunches. Events stay in the journal until you acknowledge them. + +On every app start: + +```js +const events = await Upload.getUnacknowledgedEvents(); +for (const e of events) { + // e: { eventId, id, type, timestamp, responseCode?, responseBody?, responseHeaders?, error?, cancelReason? } + handleOutcome(e); +} +await Upload.ackEvents(events.map((e) => e.eventId)); + +// Then reconcile anything still in flight: +const live = await Upload.getAllUploads(); +``` + +Notes: +- **Breaking in v8:** `completed` fires only for 2xx (plus any per-request `acceptStatus` + codes). Other HTTP responses emit `error` with `errorKind: 'http'` and the full + response (`responseCode`, `responseBody`, `responseHeaders`) attached. +- `errorKind` distinguishes HTTP errors, transport failures (`'network'`), and missing + files (`'file'`) — retry transport failures, don't retry client errors. +- `cancelReason` distinguishes user cancels (`'user'`) from system kills (`'system'`). +- Duplicate journal entries per upload id are possible if the process dies at + exactly the wrong moment (Android re-runs the worker); dedupe by `id`, keep latest. +- Android: `getAllUploads()` sees finished work for only ~1 day (WorkManager + auto-pruning) — read outcomes from the journal, use `getAllUploads()` for live state. +- iOS: in rare cases the OS fails to deliver the response body for a background + upload (known platform quirk) — `responseCode` is always authoritative; + treat `responseBody` as best-effort. +```` + +- [ ] **Step 4: CHANGELOG** + +Prepend to `CHANGELOG.md`: + +```markdown +## 8.0.0 + +Breaking: +- `completed` now fires only for 2xx responses (plus per-request `acceptStatus` codes, + e.g. `acceptStatus: [409]`). Other HTTP responses emit `error` with + `errorKind: 'http'` and the full response attached. +- `error` events are typed: `errorKind: 'http' | 'network' | 'file' | 'unknown'`. +- Removed committed `lib/` output; typings now point at `src/index.ts` + (deep imports of `lib/*` break — import from the package root). +- iOS: removed non-functional multipart, PHAsset (`assets-library://`), and `appGroup` + code paths. Android: removed unexposed `stopAllUploads`. + +Added: +- Native durable event journal: `getUnacknowledgedEvents()` / `ackEvents(ids)` — terminal + events survive app death and JS reloads (at-least-once delivery). +- `getAllUploads()` on both platforms. +- `cancelled` events carry `cancelReason: 'user' | 'system'`. +- iOS: `handleEventsForBackgroundURLSession` support (see README AppDelegate setup), + `responseHeaders` on completed events, progress throttled to 500ms, UUID default ids, + persisted task map so ids survive relaunch. +- Android: the `android` options block is now optional — sensible notification defaults, + and the library creates its notification channel itself. +``` + +- [ ] **Step 5: Lint, commit, tag** + +```bash +yarn lint:ci && yarn test +git rm -r lib +git add package.json .github/workflows/node.yml README.md CHANGELOG.md +git commit -m "chore: v8.0.0 — docs, CI (jest + android tests + typecheck), drop committed lib/" +git tag v8.0.0 # after PR merge, not on the branch +``` + +--- + +### Task 11: On-device E2E verification (manual checklist) + +**Files:** +- Modify: `example/RNBGUExample/App.tsx` (debug buttons) + +**Interfaces:** +- Consumes: the full new API. +- Produces: a verified release. **Do not mark this plan complete without this task — the whole point is behavior when the app dies, which only devices can prove.** + +- [ ] **Step 1: Add journal debug buttons to the example app** + +In `example/RNBGUExample/App.tsx`, next to the existing Cancel button, add: + +```tsx +