From 196451de56fb2d7256138c963c44056959998c7e Mon Sep 17 00:00:00 2001 From: Dylan Murphy Date: Thu, 24 Sep 2026 14:30:31 -0400 Subject: [PATCH 1/3] v10 slice 2: Android queue store, executor, and events Android implements the v10 contract pinned in the codegen spec. The v9 chunked manifest store generalizes into QueueStore: one durable entry per mutate() (id, key, vars, descriptor, staged body, state, attempts, bytes, expiresAt, header generation, deliveries) written tmp+fsync+rename before any attempt. BodyStaging writes JSON and multipart bodies to library files, copies single-file bodies, and moves chunked sources as v9 did. WorkManager runs one worker per entry. WorkerOps applies the retry table from the plan: accept rules with bodyIncludes, transient network/5xx/408/ 429 with jittered backoff and nextAttemptAt, 401/403 parking with a header generation that updateHeaders bumps, per-request exempt lists, expiry at expiresAt with bytes kept. The transfer semaphore (4) and chunked window (3) stay. Pause gates the queue; cancel settles a live entry and forgets a settled one; ack is void and idempotent and forgets only a matching generation. The journal writes every outcome before onSettled emits it; onState carries a full RequestRow, onProgress is throttled 1 s / 10 min, onAttempt is live-only. RequestIndex backs the synchronous getRequests. First launch imports v9 journal entries as legacy rows and cancels v9 work. Logic lives in JVM-testable classes (QueueController, WorkerOps, EnqueueRules, EntryTransitions); the module and workers are thin shells. 217 unit tests, including crash-mid-write cases for the store. Co-Authored-By: Claude Fable 5.1 --- android/consumer-rules.pro | 55 +- .../openspace/backgroundupload/AtomicFiles.kt | 53 ++ .../backgroundupload/AttemptEvent.kt | 91 +++ .../openspace/backgroundupload/BodyStaging.kt | 212 ++++++ .../backgroundupload/ChunkedEngine.kt | 106 +-- .../backgroundupload/ChunkedManifest.kt | 298 --------- .../backgroundupload/ChunkedParts.kt | 60 ++ .../backgroundupload/ChunkedUploadWorker.kt | 488 ++++---------- .../backgroundupload/ChunkedWorkerGate.kt | 40 -- .../ai/openspace/backgroundupload/Diag.kt | 19 + .../backgroundupload/EnqueueRules.kt | 156 +++++ .../backgroundupload/EntryParsing.kt | 184 ++++++ .../backgroundupload/EntryTransitions.kt | 86 +++ .../openspace/backgroundupload/EntryWorker.kt | 297 +++++++++ .../backgroundupload/EventJournal.kt | 278 +++++--- .../backgroundupload/EventReporter.kt | 88 ++- .../openspace/backgroundupload/JsonBridge.kt | 176 +++++ .../backgroundupload/LegacyImport.kt | 101 +++ .../backgroundupload/ProgressThrottle.kt | 55 ++ .../backgroundupload/QueueController.kt | 422 ++++++++++++ .../openspace/backgroundupload/QueueEntry.kt | 228 +++++++ .../backgroundupload/QueueSettings.kt | 97 +++ .../openspace/backgroundupload/QueueStore.kt | 215 ++++++ .../backgroundupload/RequestIndex.kt | 49 ++ .../backgroundupload/RetryClassifier.kt | 83 +++ .../openspace/backgroundupload/Scheduler.kt | 111 ++++ .../ai/openspace/backgroundupload/Upload.kt | 107 --- .../backgroundupload/UploadOutcome.kt | 3 +- .../openspace/backgroundupload/UploadUtils.kt | 139 ++-- .../backgroundupload/UploadWorker.kt | 354 +++------- .../backgroundupload/UploaderModule.kt | 451 ++++--------- .../backgroundupload/UserCancellations.kt | 17 - .../openspace/backgroundupload/WorkerGate.kt | 32 + .../openspace/backgroundupload/WorkerOps.kt | 271 ++++++++ .../backgroundupload/AckReleaseTest.kt | 69 -- .../backgroundupload/BodyStagingTest.kt | 207 ++++++ .../backgroundupload/ChunkedEngineTest.kt | 124 +--- .../backgroundupload/ChunkedManifestTest.kt | 384 ----------- .../backgroundupload/ChunkedPartsTest.kt | 56 ++ .../backgroundupload/ChunkedWorkerGateTest.kt | 57 -- .../backgroundupload/EntryParsingTest.kt | 124 ++++ .../backgroundupload/EntryTransitionsTest.kt | 176 +++++ .../backgroundupload/EventJournalTest.kt | 167 +++-- .../backgroundupload/JsonBridgeTest.kt | 81 +++ .../backgroundupload/LegacyImportTest.kt | 82 +++ .../backgroundupload/QueueControllerTest.kt | 617 ++++++++++++++++++ .../backgroundupload/QueueEntryTest.kt | 103 +++ .../backgroundupload/QueueSettingsTest.kt | 68 ++ .../backgroundupload/QueueStoreTest.kt | 245 +++++++ .../backgroundupload/RetryClassifierTest.kt | 102 +++ .../backgroundupload/SmallPartsTest.kt | 167 +++++ .../openspace/backgroundupload/TestSupport.kt | 123 ++++ .../backgroundupload/UploadStatesTest.kt | 84 --- .../openspace/backgroundupload/UploadTest.kt | 112 ---- .../backgroundupload/WorkerGateTest.kt | 51 ++ .../backgroundupload/WorkerOpsTest.kt | 283 ++++++++ 56 files changed, 6321 insertions(+), 2583 deletions(-) create mode 100644 android/src/main/java/ai/openspace/backgroundupload/AtomicFiles.kt create mode 100644 android/src/main/java/ai/openspace/backgroundupload/AttemptEvent.kt create mode 100644 android/src/main/java/ai/openspace/backgroundupload/BodyStaging.kt delete mode 100644 android/src/main/java/ai/openspace/backgroundupload/ChunkedManifest.kt create mode 100644 android/src/main/java/ai/openspace/backgroundupload/ChunkedParts.kt delete mode 100644 android/src/main/java/ai/openspace/backgroundupload/ChunkedWorkerGate.kt create mode 100644 android/src/main/java/ai/openspace/backgroundupload/Diag.kt create mode 100644 android/src/main/java/ai/openspace/backgroundupload/EnqueueRules.kt create mode 100644 android/src/main/java/ai/openspace/backgroundupload/EntryParsing.kt create mode 100644 android/src/main/java/ai/openspace/backgroundupload/EntryTransitions.kt create mode 100644 android/src/main/java/ai/openspace/backgroundupload/EntryWorker.kt create mode 100644 android/src/main/java/ai/openspace/backgroundupload/JsonBridge.kt create mode 100644 android/src/main/java/ai/openspace/backgroundupload/LegacyImport.kt create mode 100644 android/src/main/java/ai/openspace/backgroundupload/ProgressThrottle.kt create mode 100644 android/src/main/java/ai/openspace/backgroundupload/QueueController.kt create mode 100644 android/src/main/java/ai/openspace/backgroundupload/QueueEntry.kt create mode 100644 android/src/main/java/ai/openspace/backgroundupload/QueueSettings.kt create mode 100644 android/src/main/java/ai/openspace/backgroundupload/QueueStore.kt create mode 100644 android/src/main/java/ai/openspace/backgroundupload/RequestIndex.kt create mode 100644 android/src/main/java/ai/openspace/backgroundupload/RetryClassifier.kt create mode 100644 android/src/main/java/ai/openspace/backgroundupload/Scheduler.kt delete mode 100644 android/src/main/java/ai/openspace/backgroundupload/Upload.kt delete mode 100644 android/src/main/java/ai/openspace/backgroundupload/UserCancellations.kt create mode 100644 android/src/main/java/ai/openspace/backgroundupload/WorkerGate.kt create mode 100644 android/src/main/java/ai/openspace/backgroundupload/WorkerOps.kt delete mode 100644 android/src/test/java/ai/openspace/backgroundupload/AckReleaseTest.kt create mode 100644 android/src/test/java/ai/openspace/backgroundupload/BodyStagingTest.kt delete mode 100644 android/src/test/java/ai/openspace/backgroundupload/ChunkedManifestTest.kt create mode 100644 android/src/test/java/ai/openspace/backgroundupload/ChunkedPartsTest.kt delete mode 100644 android/src/test/java/ai/openspace/backgroundupload/ChunkedWorkerGateTest.kt create mode 100644 android/src/test/java/ai/openspace/backgroundupload/EntryParsingTest.kt create mode 100644 android/src/test/java/ai/openspace/backgroundupload/EntryTransitionsTest.kt create mode 100644 android/src/test/java/ai/openspace/backgroundupload/JsonBridgeTest.kt create mode 100644 android/src/test/java/ai/openspace/backgroundupload/LegacyImportTest.kt create mode 100644 android/src/test/java/ai/openspace/backgroundupload/QueueControllerTest.kt create mode 100644 android/src/test/java/ai/openspace/backgroundupload/QueueEntryTest.kt create mode 100644 android/src/test/java/ai/openspace/backgroundupload/QueueSettingsTest.kt create mode 100644 android/src/test/java/ai/openspace/backgroundupload/QueueStoreTest.kt create mode 100644 android/src/test/java/ai/openspace/backgroundupload/RetryClassifierTest.kt create mode 100644 android/src/test/java/ai/openspace/backgroundupload/SmallPartsTest.kt create mode 100644 android/src/test/java/ai/openspace/backgroundupload/TestSupport.kt delete mode 100644 android/src/test/java/ai/openspace/backgroundupload/UploadStatesTest.kt delete mode 100644 android/src/test/java/ai/openspace/backgroundupload/UploadTest.kt create mode 100644 android/src/test/java/ai/openspace/backgroundupload/WorkerGateTest.kt create mode 100644 android/src/test/java/ai/openspace/backgroundupload/WorkerOpsTest.kt diff --git a/android/consumer-rules.pro b/android/consumer-rules.pro index 754b74ca..d7627d06 100644 --- a/android/consumer-rules.pro +++ b/android/consumer-rules.pro @@ -1,32 +1,43 @@ # These rules go to consumers through consumerProguardFiles. Thus a minified # release build of the host app keeps these guarantees. # -# Gson persists Upload, NotificationConfig, and EventJournal.Entry. Upload goes -# into WorkManager input data. NotificationConfig goes into SharedPreferences. -# Entry goes into the on-disk event journal. The library reads them back later, -# across app restarts AND across app updates. Gson finds fields by name through -# reflection. Gson also needs the generic Signature attribute to rebuild typed -# collections. Thus, if R8 renames a field or removes Signature, it corrupts the -# persisted state silently: +# Gson persists the queue entry (entry.json), the queue settings +# (settings.json), the settled-outcome journal, NotificationConfig +# (SharedPreferences), and reads the v9 journal and v9 chunked manifests at +# the first v10 launch. The library reads them back later, across app +# restarts AND across app updates. Gson finds fields by name through +# reflection, and it needs the generic Signature attribute to rebuild typed +# collections. Thus, if R8 renames a field or removes Signature, it corrupts +# the persisted state silently: # -# * Upload.accept is a List. Without Signature, Gson decodes the -# elements as bare maps. Then no rule ever matches, and a configured accept -# status (for example 409) is reported as an http error, not as a completed -# upload. ChunkedManifest.parts has the same shape and the same failure. -# * A journal Entry from an older build fails to parse if field names changed. -# The library then drops the Entry as malformed. This loses the terminal -# outcomes that the journal exists to keep. A ChunkedManifest is the resume -# record for a chunked upload, and it fails in the same way. +# * Descriptor.accept is a List and Descriptor.parts a +# List. Without Signature, Gson decodes the elements as bare maps. +# Then no accept rule matches, and every chunked part reads as unsent. +# * A record from an older build fails to parse if field names changed. The +# library drops it as malformed. That loses the outcomes the journal +# exists to keep, and the entries the queue exists to run. +# * EntryState is an enum persisted by its @SerializedName wire string. # -# Debug builds are not minified and round-trip correctly. Thus neither failure +# Debug builds are not minified and round-trip correctly, so neither failure # is reproducible without R8. Keep these rules. -keepattributes Signature -keepattributes *Annotation* --keep class ai.openspace.backgroundupload.Upload { *; } --keep class ai.openspace.backgroundupload.Upload$* { *; } --keep class ai.openspace.backgroundupload.NotificationConfig { *; } --keep class ai.openspace.backgroundupload.EventJournal$Entry { *; } --keep class ai.openspace.backgroundupload.ChunkedManifest { *; } --keep class ai.openspace.backgroundupload.ChunkedManifest$* { *; } +# v10 queue +-keep class ai.openspace.backgroundupload.QueueEntry { *; } +-keep class ai.openspace.backgroundupload.EntryState { *; } +-keep class ai.openspace.backgroundupload.Descriptor { *; } +-keep class ai.openspace.backgroundupload.FormPart { *; } +-keep class ai.openspace.backgroundupload.RetryOverride { *; } +-keep class ai.openspace.backgroundupload.StagedBody { *; } +-keep class ai.openspace.backgroundupload.Part { *; } +-keep class ai.openspace.backgroundupload.QueueSettings { *; } +-keep class ai.openspace.backgroundupload.RetryDefaults { *; } +-keep class ai.openspace.backgroundupload.EventJournal$SettledRecord { *; } +-keep class ai.openspace.backgroundupload.EventJournal$Response { *; } -keep class ai.openspace.backgroundupload.UploadOutcome$AcceptRule { *; } +-keep class ai.openspace.backgroundupload.NotificationConfig { *; } + +# v9 files read once at the first v10 launch +-keep class ai.openspace.backgroundupload.LegacyImport$V9Entry { *; } +-keep class ai.openspace.backgroundupload.LegacyManifest { *; } diff --git a/android/src/main/java/ai/openspace/backgroundupload/AtomicFiles.kt b/android/src/main/java/ai/openspace/backgroundupload/AtomicFiles.kt new file mode 100644 index 00000000..bf291b53 --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/AtomicFiles.kt @@ -0,0 +1,53 @@ +package ai.openspace.backgroundupload + +import java.io.File +import java.io.FileOutputStream +import java.io.IOException +import java.nio.channels.FileChannel +import java.nio.file.StandardOpenOption + +/** + * Write-ahead file writes. Every durable file in the library goes through + * [writeAtomically]: write a tmp sibling, fsync it, rename it over the + * target. A crash at any point leaves either the old target or the new one, + * never a partial file. + */ +internal object AtomicFiles { + const val TMP_SUFFIX = ".tmp" + + fun tmpFor(target: File) = File(target.parentFile, target.name + TMP_SUFFIX) + + /** Throws IOException when the target could not be replaced. The old target is then intact. */ + fun writeAtomically(target: File, write: (FileOutputStream) -> Unit) { + val parent = target.parentFile ?: throw IOException("no parent directory for ${target.path}") + if (!parent.isDirectory && !parent.mkdirs()) { + throw IOException("could not create ${parent.path}") + } + val tmp = tmpFor(target) + try { + FileOutputStream(tmp).use { out -> + write(out) + out.flush() + out.fd.sync() + } + if (!tmp.renameTo(target)) throw IOException("could not rename ${tmp.path} to ${target.name}") + } catch (error: Throwable) { + tmp.delete() + throw error + } + syncDirectory(parent) + } + + fun writeText(target: File, text: String) = + writeAtomically(target) { it.write(text.toByteArray(Charsets.UTF_8)) } + + /** + * Makes a rename durable. This works on Linux (Android). Some file systems + * do not allow it, so a failure is ignored: the rename itself is still atomic. + */ + fun syncDirectory(dir: File) { + runCatching { + FileChannel.open(dir.toPath(), StandardOpenOption.READ).use { it.force(true) } + } + } +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/AttemptEvent.kt b/android/src/main/java/ai/openspace/backgroundupload/AttemptEvent.kt new file mode 100644 index 00000000..37a65f0a --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/AttemptEvent.kt @@ -0,0 +1,91 @@ +package ai.openspace.backgroundupload + +import com.facebook.react.bridge.WritableMap + +/** + * One HTTP attempt, before the library interprets it. Live only: never + * journaled. [outcome] is `completed` when the response is accepted and + * `error` otherwise, so a 401 is `error` with httpCode 401 even though the + * entry parks. + */ +data class AttemptEvent( + val id: String, + val key: String, + val requestId: String, + val attempt: Int, + val url: String, + val method: String, + val partIndex: Int?, + val outcome: String, + val httpCode: Int?, + val responseBody: String?, + val responseBodyTruncated: Boolean?, + val responseHeaders: Map?, + val errorKind: String?, + val errorMessage: String?, + val cancelReason: String?, + val at: Long, +) { + companion object { + const val MAX_BODY_CHARS = 4 * 1024 + + fun ofResponse( + entry: QueueEntry, + requestId: String, + url: String, + partIndex: Int?, + response: UploadResponse, + accepted: Boolean, + at: Long, + ): AttemptEvent { + val (body, truncated) = EventJournal.capBody(response.body, MAX_BODY_CHARS) + return AttemptEvent( + id = entry.id, key = entry.key, requestId = requestId, attempt = entry.attempts, + url = url, method = entry.descriptor?.method ?: "POST", partIndex = partIndex, + outcome = if (accepted) "completed" else "error", + httpCode = response.code, responseBody = body, responseBodyTruncated = truncated, + responseHeaders = response.headers, + errorKind = if (accepted) null else "http", + errorMessage = if (accepted) null else "HTTP ${response.code}", + cancelReason = null, at = at, + ) + } + + fun ofFailure( + entry: QueueEntry, + requestId: String, + url: String, + partIndex: Int?, + errorKind: String, + message: String, + at: Long, + ) = AttemptEvent( + id = entry.id, key = entry.key, requestId = requestId, attempt = entry.attempts, + url = url, method = entry.descriptor?.method ?: "POST", partIndex = partIndex, + outcome = "error", httpCode = null, responseBody = null, responseBodyTruncated = null, + responseHeaders = null, errorKind = errorKind, errorMessage = message, + cancelReason = null, at = at, + ) + } + + fun toMap(): Map = LinkedHashMap().apply { + put("id", id) + put("key", key) + put("requestId", requestId) + put("attempt", attempt.toDouble()) + put("url", url) + put("method", method) + partIndex?.let { put("partIndex", it.toDouble()) } + put("outcome", outcome) + httpCode?.let { put("httpCode", it.toDouble()) } + responseBody?.let { put("responseBody", it) } + responseBodyTruncated?.let { put("responseBodyTruncated", it) } + responseHeaders?.let { put("responseHeaders", it) } + errorKind?.let { put("errorKind", it) } + errorMessage?.let { put("errorMessage", it) } + cancelReason?.let { put("cancelReason", it) } + put("at", at.toDouble()) + } + + fun toWritableMap(): WritableMap = JsonBridge.toWritableMap(toMap()) +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/BodyStaging.kt b/android/src/main/java/ai/openspace/backgroundupload/BodyStaging.kt new file mode 100644 index 00000000..e65d4647 --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/BodyStaging.kt @@ -0,0 +1,212 @@ +package ai.openspace.backgroundupload + +import java.io.BufferedOutputStream +import java.io.File +import java.io.OutputStream +import java.nio.file.Files +import java.nio.file.StandardCopyOption +import java.util.UUID + +/** + * Writes a request body into the entry directory before the entry is saved. + * After enqueue resolves, every byte the request needs is in the library + * directory, so the caller may delete its source. + * + * | Descriptor | Staged file | Content-Type | + * | none | none | none | + * | data | body-.json | application/json unless the caller set one | + * | form | body-.multipart| multipart/form-data; boundary=… (replaces the caller's) | + * | file | file- (copy) | the caller's own | + * | file + parts | blob / blob- (move) | the caller's own | + * + * Each staged file has a name per generation, so a replace writes a new file + * next to the old one. The old body is deleted only after the new entry is + * saved. Every source is checked before anything is written. A chunked blob + * of generation 1 keeps the v9 name `blob`, so v9 blobs are found in place. + */ +object BodyStaging { + const val JSON_CONTENT_TYPE = "application/json" + private const val BUFFER = 64 * 1024 + private val CRLF = "\r\n".toByteArray() + + data class Staged(val body: StagedBody, val headers: Map) + + /** + * @param ownedBlob the chunked blob the entry (or a v9 manifest) already + * owns. The parts run over it when the caller's file is gone (v9 recreate). + * @param keepOwned run over [ownedBlob] even when the caller's file is + * present. Only for the same parts, whose accepted flags carry over. + */ + fun stage( + d: Descriptor, + dir: File, + generation: Int, + ownedBlob: File? = null, + keepOwned: Boolean = false, + ): Staged { + checkSources(d, dir, generation, ownedBlob, keepOwned) + val body = when (d.bodyKind) { + StagedBody.NONE -> StagedBody(StagedBody.NONE, null, null, 0) + StagedBody.JSON -> { + val name = "body-$generation.json" + val target = File(dir, name) + writeJson(d.dataJson!!, target) + StagedBody(StagedBody.JSON, name, null, target.length()) + } + StagedBody.MULTIPART -> { + val name = "body-$generation.multipart" + val target = File(dir, name) + val boundary = newBoundary() + writeMultipart(d.form!!, boundary, target) + StagedBody(StagedBody.MULTIPART, name, boundary, target.length()) + } + StagedBody.FILE -> { + val name = "file-$generation" + val target = File(dir, name) + copyFile(File(d.file!!), target) + StagedBody(StagedBody.FILE, name, null, target.length()) + } + else -> { + val parts = d.parts!! + val source = File(d.file!!) + val runOver = chunkedInput(d, dir, generation, ownedBlob, keepOwned) + // Checked before the move, so a rejected plan moves nothing. + if (!ChunkedParts.tilesExactly(parts, runOver.length())) { + throw QueueException( + QueueException.E_INVALID, + "parts must tile the file exactly: [0, ${runOver.length()})", + ) + } + val blob = if (runOver.path == source.path) { + File(dir, blobName(generation)).also { takeOwnership(source, it) } + } else runOver + StagedBody(StagedBody.CHUNKED, blob.name, null, ChunkedParts.totalBytes(parts)) + } + } + return Staged(body, headersFor(d.headers, body)) + } + + /** The Content-Type rule of the table above. */ + fun headersFor(headers: Map, body: StagedBody): Map = + when (body.kind) { + StagedBody.JSON -> + if (HeaderMap.contains(headers, "Content-Type")) headers + else headers + ("Content-Type" to JSON_CONTENT_TYPE) + StagedBody.MULTIPART -> + HeaderMap.without(headers, "Content-Type") + + ("Content-Type" to "multipart/form-data; boundary=${body.boundary}") + else -> headers + } + + /** The chunked blob name of [generation]. Generation 1 keeps the v9 name. */ + fun blobName(generation: Int) = + if (generation <= 1) QueueStore.BLOB_FILE else "${QueueStore.BLOB_FILE}-$generation" + + /** + * The file a chunked plan runs over, before anything moves: + * 1. [ownedBlob] when [keepOwned] and it exists; + * 2. the caller's file when present (a new file wins over an old blob); + * 3. this generation's blob, left by a crash after the move; + * 4. [ownedBlob], when the caller's file was moved away at an earlier enqueue. + */ + internal fun chunkedInput(d: Descriptor, dir: File, generation: Int, ownedBlob: File?, keepOwned: Boolean): File { + val source = File(d.file!!) + val leftover = File(dir, blobName(generation)) + return when { + keepOwned && ownedBlob != null && ownedBlob.exists() -> ownedBlob + source.isFile -> source + leftover.exists() -> leftover + ownedBlob != null && ownedBlob.exists() -> ownedBlob + else -> throw QueueException(QueueException.E_FILE_MISSING, "file does not exist: ${source.path}") + } + } + + /** Every source must exist and be readable before any write. */ + internal fun checkSources(d: Descriptor, dir: File, generation: Int, ownedBlob: File?, keepOwned: Boolean) { + d.form?.forEach { part -> + part.path?.let { requireReadable(File(it), "form part '${part.name}'") } + } + when (d.bodyKind) { + StagedBody.FILE -> requireReadable(File(d.file!!), "file") + StagedBody.CHUNKED -> chunkedInput(d, dir, generation, ownedBlob, keepOwned) + } + } + + private fun requireReadable(file: File, what: String) { + if (!file.isFile || !file.canRead()) { + throw QueueException(QueueException.E_FILE_MISSING, "$what does not exist or can not be read: ${file.path}") + } + } + + internal fun writeJson(dataJson: String, target: File) = + AtomicFiles.writeText(target, dataJson) + + /** + * RFC 7578. Per part: the boundary line, Content-Disposition with the name + * (and a filename for a file part), Content-Type, a blank line, the bytes, + * CRLF. Then the closing delimiter. File parts stream from disk. + */ + internal fun writeMultipart(form: List, boundary: String, target: File) { + AtomicFiles.writeAtomically(target) { raw -> + val out = BufferedOutputStream(raw, BUFFER) + for (part in form) { + out.ascii("--$boundary") + out.write(CRLF) + val disposition = StringBuilder("Content-Disposition: form-data; name=\"") + .append(escapeQuoted(part.name)).append('"') + if (part.path != null) { + val fileName = part.fileName ?: File(part.path).name + disposition.append("; filename=\"").append(escapeQuoted(fileName)).append('"') + } + out.write(disposition.toString().toByteArray(Charsets.UTF_8)) + out.write(CRLF) + out.write("Content-Type: ${part.contentType}".toByteArray(Charsets.UTF_8)) + out.write(CRLF) + out.write(CRLF) + if (part.path != null) { + File(part.path).inputStream().use { it.copyTo(out, BUFFER) } + } else { + out.write((part.string ?: "").toByteArray(Charsets.UTF_8)) + } + out.write(CRLF) + } + out.ascii("--$boundary--") + out.write(CRLF) + out.flush() + } + } + + /** The WHATWG form encoding of a quoted name: `"`, CR and LF are percent-encoded. */ + internal fun escapeQuoted(value: String): String = + value.replace("\"", "%22").replace("\r", "%0D").replace("\n", "%0A") + + internal fun copyFile(source: File, target: File) { + source.inputStream().use { input -> + AtomicFiles.writeAtomically(target) { out -> input.copyTo(out, BUFFER) } + } + } + + /** + * Moves the caller's file to [blob] (v9 verbatim). A crash between the move + * and the entry save leaves the bytes at the blob path with no entry; a + * retry whose source is gone adopts them ([chunkedInput] step 3). + */ + internal fun takeOwnership(source: File, blob: File) { + if (source.absoluteFile == blob.absoluteFile) return + if (!source.exists()) { + if (blob.exists()) return + throw QueueException(QueueException.E_FILE_MISSING, "file does not exist: ${source.path}") + } + blob.parentFile?.mkdirs() + if (blob.exists()) blob.delete() + if (!source.renameTo(blob)) { + // renameTo can not cross file systems. Files.move falls back to copy + delete. + Files.move(source.toPath(), blob.toPath(), StandardCopyOption.REPLACE_EXISTING) + } + blob.parentFile?.let { AtomicFiles.syncDirectory(it) } + } + + internal fun newBoundary(): String = "----RNBGU" + UUID.randomUUID().toString().replace("-", "") + + private fun OutputStream.ascii(text: String) = write(text.toByteArray(Charsets.US_ASCII)) +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/ChunkedEngine.kt b/android/src/main/java/ai/openspace/backgroundupload/ChunkedEngine.kt index e398c750..bbbee474 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/ChunkedEngine.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/ChunkedEngine.kt @@ -6,108 +6,22 @@ import kotlinx.coroutines.sync.Semaphore import kotlinx.coroutines.sync.withPermit /** - * The pure scheduling half of chunked execution: the window, the retry - * policy, and the backoff. It is kept free of Android and OkHttp types. Thus - * the highest-consequence invariants (at most WINDOW parts in flight, and - * never two requests for one part index) are unit-testable on a plain JVM. - * [ChunkedUploadWorker] supplies the part executor. + * The pure scheduling half of chunked execution: the window. It has no + * Android or OkHttp types, so its two invariants (at most WINDOW parts in + * flight, never two requests for one part index) are unit-testable on a + * plain JVM. The retry decisions moved to [RetryClassifier] in v10. */ object ChunkedEngine { - // The number of parts of one upload in flight at one time. This is a library - // constant, not an option. If soak data shows that a different value is - // better, this constant changes, not the API. + // Parts of one upload in flight at one time. A library constant, not an option. const val WINDOW = 3 - // The library retries a non-accepted, non-transient HTTP response this many - // times per part. Then the response becomes a terminal error and stalls the - // upload. The budget is small on purpose. A response that the server repeats - // (401, 400) does not change without a new startUpload. Only transient - // failures retry without a limit. - const val PART_HTTP_RETRIES = 3 - - // The poll interval while the network is unusable (offline, or waiting for - // wifi). The interval is constant, not exponential. We wait for conditions - // here; we do not back off a server. And expiresAt bounds the total wait. - const val CONNECTIVITY_POLL_MS = 10_000L - - private const val BACKOFF_BASE_MS = 1_000L - private const val BACKOFF_CAP_MS = 60_000L - - // A 5xx means that the server failed, not that the request is wrong. Thus it - // retries like a transport failure: without a limit, until expiresAt. - fun isTransientHttp(code: Int) = code in 500..599 - - /** What a starting worker must do for the manifest that it finds (or does not find). */ - enum class StartAction { - /** - * No manifest exists. The upload was completed and acknowledged, or it was - * explicitly removed, while this run sat in the queue. Both are legitimate - * ends, already reported (or deliberately not reported). Exit with success - * and in silence. A journaled terminal here would be a spurious 'file' - * error for an upload that nobody owns any more. - */ - NO_MANIFEST, - - /** - * Every part is already accepted: this is a trailing resume run. Re-report - * the journaled completion (never mint a second terminal event) and stop. - * Start no foreground service and no transfers. - */ - ALREADY_COMPLETE, - - /** Pending parts remain. Run the engine. */ - RUN, - } - - fun startAction(manifest: ChunkedManifest?): StartAction = when { - manifest == null -> StartAction.NO_MANIFEST - manifest.allAccepted -> StartAction.ALREADY_COMPLETE - else -> StartAction.RUN - } - - /** How a run that found (or produced) an all-accepted manifest reports the completion. */ - sealed class CompletionReport { - /** An unacknowledged 'completed' entry exists. Re-emit it. Never mint a second entry. */ - data class ReEmit(val entry: EventJournal.Entry) : CompletionReport() - - /** A fresh completion with no journal entry yet. Journal and emit a new entry. */ - object Mint : CompletionReport() - - /** - * A trailing run with nothing unacknowledged: the completion was journaled - * AND acknowledged. Nobody is owed an event. This occurs when the trailing - * run races ackEvents, which deletes the journal entry just before the - * manifest. An event minted here would be a duplicate 'completed' for an - * upload that the consumer already settled. - */ - object None : CompletionReport() - } - - fun completionReport( - unacked: List, - uploadId: String, - freshCompletion: Boolean, - ): CompletionReport { - val existing = unacked.firstOrNull { it.uploadId == uploadId && it.type == "completed" } - return when { - existing != null -> CompletionReport.ReEmit(existing) - freshCompletion -> CompletionReport.Mint - else -> CompletionReport.None - } - } - - /** Exponential backoff for transient failures: 1s, 2s, 4s, and more, capped at 60s. */ - fun backoffMs(attempt: Int): Long = - (BACKOFF_BASE_MS shl (attempt - 1).coerceIn(0, 6)).coerceAtMost(BACKOFF_CAP_MS) - /** - * Runs [executePart] exactly one time per index, with at most [window] parts - * at one time. One coroutine per part index is what guarantees that no two - * requests for the same part are in flight (concurrent PUTs of one partNum - * are verified unsafe on the server side). An executor that throws cancels - * the remaining parts, and the error propagates. Terminal classification is - * the caller's job. + * Runs [executePart] exactly one time per index, with at most [window] + * parts at one time. One coroutine per index is what guarantees no two + * requests for one part are in flight (concurrent PUTs of one partNum are + * unsafe on the server). An executor that throws cancels the other parts, + * and the error propagates. */ suspend fun run( partIndexes: List, diff --git a/android/src/main/java/ai/openspace/backgroundupload/ChunkedManifest.kt b/android/src/main/java/ai/openspace/backgroundupload/ChunkedManifest.kt deleted file mode 100644 index 6bbe523b..00000000 --- a/android/src/main/java/ai/openspace/backgroundupload/ChunkedManifest.kt +++ /dev/null @@ -1,298 +0,0 @@ -package ai.openspace.backgroundupload - -import android.content.Context -import com.facebook.react.bridge.ReadableMap -import com.google.gson.Gson -import java.io.File -import java.util.Base64 - -/** - * The durable record of one chunked upload: the moved source file, the parts - * that the consumer authored, and which of them the server has accepted. - * [ChunkedManifestStore] persists it as JSON at startUpload, BEFORE the work - * is enqueued. Thus a worker rescheduled after process death (or a startUpload - * after a crash, a stop, or a reauth) resumes from it without a call into JS. - * This manifest IS the resume mechanism. - * - * The data shape is kept free of Android and React types (Gson round-trips - * it, and JVM tests construct it directly). The ReadableMap parsing lives in - * the companion, like [Upload]'s. - */ -data class ChunkedManifest( - val id: String, - /** The library-owned copy of the bytes (the consumer's file, renamed in). */ - val sourcePath: String, - val parts: List, - val accept: List, - /** Epoch ms. After this time, the upload stops with errorKind 'expired'. */ - val expiresAt: Long, - val wifiOnly: Boolean, - val noNotification: Boolean, - val createdAt: Long, -) { - /** - * One part, exactly as the consumer authored it. The library sends the file - * bytes [start, end) as the body of a PUT to [url], with [headers] - * unchanged. It never derives or edits a protocol field. - */ - data class Part( - val url: String, - val headers: Map, - val start: Long, - val end: Long, // exclusive - val accepted: Boolean = false, - ) { - val size get() = end - start - } - - val showsNotification get() = !noNotification - val totalBytes get() = parts.sumOf { it.size } - val acceptedBytes get() = parts.filter { it.accepted }.sumOf { it.size } - - /** The server's auto-publish condition. It is the only thing that 'completed' may mean. */ - val allAccepted get() = parts.all { it.accepted } - - fun isExpired(now: Long) = now >= expiresAt - - fun pendingIndexes() = parts.indices.filter { !parts[it].accepted } - - fun withPartAccepted(index: Int) = copy( - parts = parts.mapIndexed { i, part -> if (i == index) part.copy(accepted = true) else part }, - ) - - class ReconcileException(message: String) : IllegalArgumentException(message) - - /** - * A startUpload re-call with an existing id is one of two things: - * - * **Resume** — the incoming parts are the SAME array (identical count, - * ranges, and urls). The headers, the accept rules, expiresAt, and the flags - * come from the new call. This is how fresh auth reaches stalled parts, and - * how a salvage extends the deadline. The accepted part statuses, the moved - * source, and createdAt survive from this manifest. A resume is permitted at - * any time, running or not. A running worker re-reads the stored copy before - * every attempt. - * - * **Recreate** — a DIFFERENT parts array. The consumer re-authored the - * upload under a fresh server uploadId after the old one died (it expired - * past the server's 31-day window, or it is otherwise unrecoverable). The - * owned bytes are kept. The parts are replaced as a whole, and every part - * status resets to unsent. The headers, the accept rules, and expiresAt come - * from the new call. The new ranges must tile exactly [0, blobSize). A - * partial or overlapping cover would silently upload wrong bytes. A recreate - * is accepted only while the upload is NOT running (stalled on a terminal - * error, expired, or cancelled). A different parts array while a worker - * executes is a consumer bug, not a recreate, because the in-flight requests - * belong to the old parts. - */ - fun reconcile(incoming: ChunkedManifest, running: Boolean, blobSize: Long): ChunkedManifest { - if (samePartsAs(incoming)) { - // Accepted flags follow the RANGE, not the array index. samePartsAs is - // order-independent, so the same tile can sit at a different index. - val acceptedStarts = parts.filter { it.accepted }.map { it.start }.toSet() - return incoming.copy( - sourcePath = sourcePath, - createdAt = createdAt, - parts = incoming.parts.map { it.copy(accepted = it.start in acceptedStarts) }, - ) - } - if (running) throw ReconcileException( - "chunked upload '$id' is running; a different parts array is only accepted once it stops", - ) - if (!tilesExactly(incoming.parts, blobSize)) throw ReconcileException( - "chunked upload '$id' recreate parts must tile exactly [0, $blobSize)", - ) - return incoming.copy(sourcePath = sourcePath, createdAt = createdAt) - } - - // Order-independent, like tilesExactly. The same tiles, authored in a - // different order, are the SAME upload (a resume), never a recreate. - private fun samePartsAs(incoming: ChunkedManifest): Boolean { - if (incoming.parts.size != parts.size) return false - val stored = parts.sortedBy { it.start } - val fresh = incoming.parts.sortedBy { it.start } - return stored.indices.all { i -> - fresh[i].url == stored[i].url && - fresh[i].start == stored[i].start && - fresh[i].end == stored[i].end - } - } - - companion object { - /** - * Whether [parts] cover [0, size) exactly: no gap, no overlap, and nothing - * past the end. Order-independent, like everything else about parts. - */ - fun tilesExactly(parts: List, size: Long): Boolean { - if (parts.isEmpty()) return false - val sorted = parts.sortedBy { it.start } - var cursor = 0L - for (part in sorted) { - if (part.start != cursor || part.end <= part.start) return false - cursor = part.end - } - return cursor == size - } - - /** - * Validates a first-call (create) manifest against the just-owned bytes. - * Like a recreate, the parts must tile exactly [0, blobSize). A partial or - * overlapping cover would silently upload wrong bytes. It throws BEFORE - * the manifest is saved. Thus the moved blob stays adoptable by a - * corrected retry (see UploaderModule.takeOwnership's orphan branch). - */ - fun validatedForCreate(incoming: ChunkedManifest, blobSize: Long): ChunkedManifest { - if (!tilesExactly(incoming.parts, blobSize)) throw ReconcileException( - "chunked upload '${incoming.id}' parts must tile exactly [0, $blobSize)", - ) - return incoming - } - - /** @param sourcePath the library-owned destination, not the consumer's path. */ - fun fromReadableMap(map: ReadableMap, sourcePath: String, createdAt: Long): ChunkedManifest { - val partsArr = map.getArray("parts") ?: throw Upload.MissingOptionException("parts") - if (partsArr.size() == 0) throw IllegalArgumentException("parts must be a non-empty array") - if (!map.hasKey("expiresAt")) throw Upload.MissingOptionException("expiresAt") - return ChunkedManifest( - id = map.getString("id") ?: throw Upload.MissingOptionException("id"), - sourcePath = sourcePath, - parts = (0 until partsArr.size()).map { i -> - val part = partsArr.getMap(i) ?: throw Upload.MissingOptionException("parts[$i]") - val range = part.getMap("range") ?: throw Upload.MissingOptionException("parts[$i].range") - Part( - url = part.getString("url") ?: throw Upload.MissingOptionException("parts[$i].url"), - headers = parseHeaderMap(part.getMap("headers")), - start = range.getDouble("start").toLong(), - end = range.getDouble("end").toLong(), - ) - }, - accept = parseAcceptRules(map.getArray("accept")), - expiresAt = map.getDouble("expiresAt").toLong(), - wifiOnly = if (map.hasKey("wifiOnly")) map.getBoolean("wifiOnly") else false, - noNotification = if (map.hasKey("noNotification")) map.getBoolean("noNotification") else false, - createdAt = createdAt, - ) - } - } -} - -/** - * A file-backed store: one directory per upload id, which holds - * `manifest.json` and `blob` (the moved source bytes). It has the same - * durability pattern as [EventJournal]: tmp+rename writes, and corrupt files - * read as absent. It is reachable from a bare Context, because the worker can - * run in a process where React never initialized. - */ -class ChunkedManifestStore(private val dir: File) { - - companion object { - private val gson = Gson() - - @Volatile - private var instance: ChunkedManifestStore? = null - - fun get(context: Context): ChunkedManifestStore = - instance ?: synchronized(this) { - instance ?: ChunkedManifestStore(File(context.filesDir, "rnbgupload-chunked")) - .also { instance = it } - } - } - - init { - dir.mkdirs() - } - - // Upload ids come from the consumer, and they can contain path separators or - // other filesystem-hostile characters. Thus the directory name is an encoding - // of the id, never the id itself. The id is read back from the manifest, not - // decoded from the name. - private fun uploadDir(id: String) = - File(dir, Base64.getUrlEncoder().withoutPadding().encodeToString(id.toByteArray())) - - private fun manifestFile(id: String) = File(uploadDir(id), "manifest.json") - - /** Where startUpload moves the source file for this id. */ - fun blobFile(id: String) = File(uploadDir(id), "blob") - - @Synchronized - fun load(id: String): ChunkedManifest? { - val file = manifestFile(id) - if (!file.exists()) return null - val parsed = runCatching { gson.fromJson(file.readText(), ChunkedManifest::class.java) } - .getOrNull() - return validated(parsed) - } - - /** Throws on a write failure. A manifest that did not persist must fail the startUpload call. */ - @Synchronized - fun save(manifest: ChunkedManifest) { - val dir = uploadDir(manifest.id) - dir.mkdirs() - val tmp = File(dir, "manifest.tmp") - tmp.writeText(gson.toJson(manifest)) - if (!tmp.renameTo(manifestFile(manifest.id))) { - throw java.io.IOException("failed to persist chunked manifest for '${manifest.id}'") - } - } - - /** - * An atomic read-modify-write. Thus a worker that marks a part accepted can - * never clobber a concurrent startUpload's fresh headers (or another part's - * flag). Returns null, without a throw, when the manifest is gone or the - * write failed. A caller that can continue from memory does that. - */ - @Synchronized - fun update(id: String, transform: (ChunkedManifest) -> ChunkedManifest): ChunkedManifest? = - runCatching { - val manifest = load(id) ?: return null - val next = transform(manifest) - save(next) - next - }.getOrNull() - - /** - * An atomic create-or-transform. The store lock spans load, [transform], and - * save. Thus nothing — a running worker's markAccepted included — can write - * between them and be erased. startUpload's load, reconcile, and save must - * go through here, not as three separate calls. [transform] receives null - * when no manifest exists. Unlike [update], a transform that throws (a - * reconcile rejection) or a failed write propagates, because startUpload - * must fail loudly, not continue from memory. - */ - @Synchronized - fun compute(id: String, transform: (ChunkedManifest?) -> ChunkedManifest): ChunkedManifest { - val next = transform(load(id)) - save(next) - return next - } - - /** Whether a manifest is stored for this id (without parsing it). */ - @Synchronized - fun contains(id: String): Boolean = manifestFile(id).exists() - - /** Deletes the manifest AND the moved bytes. Does nothing for an unknown id (a simple upload). */ - @Synchronized - fun remove(id: String) { - uploadDir(id).deleteRecursively() - } - - @Synchronized - fun all(): List = - (dir.listFiles { f -> f.isDirectory } ?: emptyArray()) - .mapNotNull { d -> - val file = File(d, "manifest.json") - if (!file.exists()) return@mapNotNull null - validated(runCatching { gson.fromJson(file.readText(), ChunkedManifest::class.java) }.getOrNull()) - } - - // Gson does not use the constructor. Thus a corrupt or field-renamed file can - // make non-null Kotlin fields null. Reject a file that lacks a field that the - // engine relies on. Normalize an absent accept list; do not reject it. - @Suppress("SENSELESS_COMPARISON") - private fun validated(m: ChunkedManifest?): ChunkedManifest? { - if (m == null || m.id == null || m.sourcePath == null || m.parts == null) return null - if (m.parts.isEmpty()) return null - if (m.parts.any { it == null || it.url == null || it.headers == null }) return null - return if (m.accept == null) m.copy(accept = listOf()) else m - } -} diff --git a/android/src/main/java/ai/openspace/backgroundupload/ChunkedParts.kt b/android/src/main/java/ai/openspace/backgroundupload/ChunkedParts.kt new file mode 100644 index 00000000..5d8ac2e0 --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/ChunkedParts.kt @@ -0,0 +1,60 @@ +package ai.openspace.backgroundupload + +/** + * One part of a chunked upload, as the caller wrote it. The library sends the + * file bytes [start, end) to [url]. [accepted] is set when the server + * accepted the part. The field names match the v9 manifest, so a v9 + * manifest.json reads into this class unchanged. + */ +data class Part( + val url: String, + val headers: Map, + val start: Long, + val end: Long, // exclusive + val accepted: Boolean = false, +) { + val size get() = end - start +} + +/** Pure rules over a parts list. Moved from the v9 ChunkedManifest with the same meaning. */ +object ChunkedParts { + + /** Whether [parts] cover [0, size) exactly: no gap, no overlap, nothing past the end. Order does not matter. */ + fun tilesExactly(parts: List, size: Long): Boolean { + if (parts.isEmpty()) return false + var cursor = 0L + for (part in parts.sortedBy { it.start }) { + if (part.start != cursor || part.end <= part.start) return false + cursor = part.end + } + return cursor == size + } + + /** + * The same upload: the same count, ranges, and urls, in any order. Headers + * are not compared, because a resume sends fresh headers. + */ + fun sameParts(a: List, b: List): Boolean { + if (a.size != b.size) return false + val x = a.sortedBy { it.start } + val y = b.sortedBy { it.start } + return x.indices.all { i -> + x[i].url == y[i].url && x[i].start == y[i].start && x[i].end == y[i].end + } + } + + /** The [incoming] parts with the accepted flags of [stored]. A flag follows the range start, not the index. */ + fun carryAccepted(stored: List, incoming: List): List { + val acceptedStarts = stored.filter { it.accepted }.map { it.start }.toSet() + return incoming.map { it.copy(accepted = it.start in acceptedStarts) } + } + + fun totalBytes(parts: List): Long = parts.sumOf { it.size } + + fun acceptedBytes(parts: List): Long = parts.filter { it.accepted }.sumOf { it.size } + + fun pendingIndexes(parts: List): List = parts.indices.filter { !parts[it].accepted } + + fun withAccepted(parts: List, index: Int): List = + parts.mapIndexed { i, part -> if (i == index) part.copy(accepted = true) else part } +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/ChunkedUploadWorker.kt b/android/src/main/java/ai/openspace/backgroundupload/ChunkedUploadWorker.kt index 33750073..1b1fed34 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/ChunkedUploadWorker.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/ChunkedUploadWorker.kt @@ -1,410 +1,178 @@ package ai.openspace.backgroundupload -import android.app.NotificationManager import android.content.Context -import androidx.work.CoroutineWorker -import androidx.work.ForegroundInfo -import androidx.work.ListenableWorker import androidx.work.WorkerParameters import kotlinx.coroutines.CancellationException -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.delay import kotlinx.coroutines.sync.withPermit -import kotlinx.coroutines.withContext import java.io.File -import java.io.IOException import java.util.UUID import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.atomic.AtomicLong +/** The WorkManager class for a chunked entry. The run is [EntryWorker]'s. */ +class ChunkedUploadWorker(context: Context, params: WorkerParameters) : EntryWorker(context, params) + /** - * Executes one chunked upload from its durable [ChunkedManifest]. The input - * data carries only the upload id. The manifest is the record: startUpload - * persists it before this work is enqueued. Thus a worker rescheduled after - * process death resumes from disk, with no JS involved. + * The parts of one chunked entry, at most [ChunkedEngine.WINDOW] at a time, + * each part through the shared 4-request semaphore. One logical entry has + * one progress stream (byte-weighted) and one outcome: completed only when + * every part is accepted. * - * One logical upload has one event stream: byte-weighted aggregate progress, - * and one terminal event. 'completed' is journaled only when every part is - * accepted. Every other terminal keeps the manifest and the bytes, so a later - * startUpload can resume. The bytes are deleted only when a 'completed' event - * is ACKED (see UploaderModule.ackEvents). + * Per part: accepted → persist the flag; auth → the whole entry parks (the + * sibling parts stop); transient → a short backoff waits in the part while + * the siblings go on, a long one releases the whole worker (accepted parts + * are kept); terminal → the entry fails with that part's index. */ -class ChunkedUploadWorker(private val context: Context, params: WorkerParameters) : - CoroutineWorker(context, params) { - - companion object { - /** - * The key for the upload id in the worker's input data. It is a string - * literal for the same reason as [UploadWorker.PARAMS_KEY]: WorkManager's - * database persists it across builds, and it must survive R8 renames and - * refactors. - */ - const val ID_KEY = "chunkedUploadId" - - /** How often a starting worker re-checks [ChunkedWorkerGate] for its id. */ - private const val GATE_POLL_MS = 100L - } - - private lateinit var uploadId: String - private val store by lazy { ChunkedManifestStore.get(context) } - private val config by lazy { NotificationConfig.load(context) } - private val notificationManager = - context.getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager +internal class ChunkedTransfer(private val host: EntryWorker) { - // The latest known manifest. Part executors re-read the stored copy before - // every attempt (see latest()). Thus a reconciling startUpload's fresh - // headers, and an extended expiresAt, reach a worker that already runs. - @Volatile - private var manifest: ChunkedManifest? = null - - @Volatile - private var connectivity = Connectivity.Ok + /** A terminal part failure. Not a CancellationException, so it stops the sibling parts. */ + private class PartFailed(val settlement: Settlement.Failed) : Exception(settlement.message) - // In-flight bytes per part index, for byte-weighted aggregate progress. private val partSent = ConcurrentHashMap() + private val acceptedHere = ConcurrentHashMap.newKeySet() private val acceptedBytes = AtomicLong(0) + private var total = 0L - private class ExpiredException : Exception("upload expired") - - private class SourceMissingException(path: String) : - IOException("chunked source file missing: $path") - - private class PartRejectedException(val partIndex: Int, val response: UploadResponse) : - Exception("part $partIndex rejected with HTTP ${response.code}") - - private class PartBeyondEofException(val partIndex: Int, message: String) : Exception(message) - - override suspend fun doWork(): Result = withContext(Dispatchers.IO) { - uploadId = inputData.getString(ID_KEY) ?: throw Throwable("No upload id") - - // Acquire the per-id execution gate BEFORE the first manifest read. A - // cancel-then-start can start this worker while the cancelled one still - // winds down, and two PUTs of one partNum are unsafe. Also, the manifest - // read occurs only after the gate is held. That is what makes the module's - // recreate check race-free (see ChunkedWorkerGate and - // ChunkedManifestStore.compute). - try { - while (!ChunkedWorkerGate.tryAcquire(uploadId, this@ChunkedUploadWorker)) { - delay(GATE_POLL_MS) - } - } catch (error: CancellationException) { - // Cancelled while waiting. A user cancel still owes its terminal event. - checkAndHandleCancellation() - throw error - } - try { - runUpload() - } finally { - ChunkedWorkerGate.release(uploadId, this@ChunkedUploadWorker) - } - } - - private suspend fun runUpload(): Result { - val initial = store.load(uploadId) - when (ChunkedEngine.startAction(initial)) { - // The upload was completed-and-acknowledged, or it was removed, while - // this run sat in the queue. Both are legitimate and already settled. - // Exit in silence. A terminal journaled here would be a spurious error - // for an upload that nobody owns. - ChunkedEngine.StartAction.NO_MANIFEST -> return Result.success() - // A trailing resume of a finished-but-unacknowledged upload. Re-report - // the journaled completion. Skip the foreground service and the engine. - ChunkedEngine.StartAction.ALREADY_COMPLETE -> { - manifest = initial - journalCompleted(freshCompletion = false) - return Result.success() - } - ChunkedEngine.StartAction.RUN -> Unit + @Volatile + private var lastAcceptedUrl: String? = null + + suspend fun run(start: QueueEntry): Settlement { + val d0 = start.descriptor!! + val parts = d0.parts!! + lastAcceptedUrl = parts.lastOrNull { it.accepted }?.url + val pending = ChunkedParts.pendingIndexes(parts) + // A run over an all-accepted entry that has not settled yet. + if (pending.isEmpty()) return Settlement.Completed(null, lastAcceptedUrl ?: d0.reportUrl, d0.method) + val blob = host.bodyFile(start) + if (blob == null || !blob.exists()) { + return Settlement.Failed("file", "the chunked file is missing", null, null, d0.reportUrl, d0.method) } - checkNotNull(initial) // RUN implies a manifest - manifest = initial - acceptedBytes.set(initial.acceptedBytes) - UploadProgress.add(uploadId, initial.totalBytes) - UploadProgress.set(uploadId, initial.acceptedBytes) + total = ChunkedParts.totalBytes(parts) + acceptedBytes.set(ChunkedParts.acceptedBytes(parts)) + UploadProgress.add(host.entryId, total) + UploadProgress.set(host.entryId, acceptedBytes.get()) - // Initialization. A failure here is terminal: journaled, never retried. - // The EXCEPTION is a refused foreground start, which the transfer - // survives. try { - if (initial.showsNotification) { - ensureNotificationChannel(notificationManager, config) - setForeground(getForegroundInfo()) - } - } catch (error: Throwable) { - if (!isForegroundStartDenied(error)) { - if (!checkAndHandleCancellation()) { - UploadProgress.remove(uploadId) - handleFailure(error) - } - return terminalErrorResult() - } - // The app is in the background, and API 31+ refused the foreground - // start. This is the usual state for a WorkManager relaunch (a reboot, - // or a quota resume). The upload runs correctly without foreground - // priority. A failure here would brick every headless resume. - } - - return try { - ChunkedEngine.run(initial.pendingIndexes()) { index -> executePart(index) } - // Every executor returned. An executor returns only when its part was - // accepted. That is exactly the server's auto-publish condition. - UploadProgress.complete(uploadId) - journalCompleted(freshCompletion = true) - Result.success() - } catch (error: Throwable) { - if (checkAndHandleCancellation()) throw error - UploadProgress.remove(uploadId) - handleFailure(error) - terminalErrorResult() + ChunkedEngine.run(pending) { index -> executePart(index, blob, start.backoffStreak) } + } catch (failed: PartFailed) { + return failed.settlement } + // Every executor returned, and an executor returns only when its part was + // accepted: the server's auto-publish condition. + return Settlement.Completed(null, lastAcceptedUrl ?: d0.reportUrl, d0.method) } - /** - * Uploads one part until it is accepted, or throws. Terminal conditions - * (expiry, a missing source, or a non-accepted response out of retries) - * propagate and cancel the sibling parts. Everything transient retries here, - * bounded only by expiresAt. - */ - private suspend fun executePart(index: Int) { - var rejections = 0 - var transientAttempts = 0 + private suspend fun executePart(index: Int, blob: File, initialStreak: Int) { + var streak = initialStreak while (true) { - val current = latest() - val part = current.parts[index] - if (part.accepted) return - if (current.isExpired(System.currentTimeMillis())) throw ExpiredException() - - // A range past the blob's EOF can never transmit. The read would fail - // on every attempt until expiry. Thus it is a terminal 'file' error - // immediately (iOS classifies it the same way). length() is 0 for a - // missing file. That case falls through to the transfer, which - // classifies it as source-missing. The failed-probe-reads-as-network - // default stays intact. - val blobLength = runCatching { File(current.sourcePath).length() }.getOrDefault(0L) - if (blobLength > 0L && part.end > blobLength) throw PartBeyondEofException( - index, - "part $index range [${part.start}, ${part.end}) exceeds source size $blobLength", - ) - - if (!validateAndReportConnectivity(current.wifiOnly)) { - delay(ChunkedEngine.CONNECTIVITY_POLL_MS) - continue + if (index in acceptedHere) return + val latest = host.ops.latest(host.entryId, host.generation) + val stored = latest.descriptor!!.parts!![index] + if (stored.accepted) return + if (RetryClassifier.isExpired(host.now(), latest.expiresAt)) throw EntryWorker.ExpiredException() + + // A range past EOF can never be sent. length() is 0 for a missing file; + // that case falls through to the transfer, which classifies it as file. + val blobLength = runCatching { blob.length() }.getOrDefault(0L) + if (blobLength > 0L && stored.end > blobLength) { + throw PartFailed( + Settlement.Failed( + "file", + "part $index range [${stored.start}, ${stored.end}) exceeds the file size $blobLength", + null, index, stored.url, latest.descriptor.method, + ), + ) } + host.waitForNetwork() + + val requestId = UUID.randomUUID().toString() + val entry = host.ops.recordAttempt(host.entryId, host.generation, requestId) + // The generation of the headers this attempt sends: both come from the same entry. + val headerGeneration = entry.headerGeneration + val d = entry.descriptor!! + val part = d.parts!![index] + val policy = host.policy(entry) val response = try { transferSemaphore.withPermit { - okhttpUploadPart(uploadHttpClient, part, File(current.sourcePath)) { sent -> - onPartProgress(index, sent) - } + okhttpSend( + uploadHttpClient, + TransferRequest( + part.url, d.method, host.headersFor(d, part, requestId), + rangeRequestBody(blob, part.start, part.end), + ), + ) { sent -> onPartProgress(index, sent) } } } catch (error: CancellationException) { throw error - } catch (error: IOException) { + } catch (error: Throwable) { onPartProgress(index, 0L) - // The default is fileExists=true. Thus a failed probe reads as - // network, not file. - val fileExists = runCatching { File(current.sourcePath).exists() }.getOrDefault(true) - if (!fileExists) throw SourceMissingException(current.sourcePath) - transientAttempts++ - delay(ChunkedEngine.backoffMs(transientAttempts)) - continue + val fileExists = runCatching { blob.exists() }.getOrDefault(true) + val message = error.message ?: error.javaClass.simpleName + EventReporter.attempt( + AttemptEvent.ofFailure( + entry, requestId, part.url, index, RetryClassifier.failureKind(error, fileExists), message, host.now(), + ), + ) + when (val verdict = RetryClassifier.classifyFailure(error, fileExists)) { + is RetryClassifier.Verdict.Terminal -> throw PartFailed( + Settlement.Failed(verdict.errorKind, verdict.message, null, index, part.url, d.method), + ) + else -> { + streak++ + host.backoffOrRelease(policy, streak, entry.expiresAt) + continue + } + } } - if (UploadOutcome.isAccepted(response.code, response.body, current.accept)) { - markAccepted(index) - return - } - onPartProgress(index, 0L) - if (ChunkedEngine.isTransientHttp(response.code)) { - transientAttempts++ - delay(ChunkedEngine.backoffMs(transientAttempts)) - continue + val verdict = RetryClassifier.classifyResponse(response.code, response.body, d.accept, policy.exempt) + EventReporter.attempt( + AttemptEvent.ofResponse( + entry, requestId, part.url, index, response, verdict == RetryClassifier.Verdict.Accepted, host.now(), + ), + ) + when (verdict) { + RetryClassifier.Verdict.Accepted -> { + markAccepted(index, part) + return + } + RetryClassifier.Verdict.Auth -> { + if (host.ops.hasNewerHeaders(host.entryId, host.generation, headerGeneration)) { + streak = 0 + continue + } + throw EntryWorker.ParkException(headerGeneration) + } + RetryClassifier.Verdict.Transient -> { + onPartProgress(index, 0L) + streak++ + host.backoffOrRelease(policy, streak, entry.expiresAt) + } + is RetryClassifier.Verdict.Terminal -> throw PartFailed( + Settlement.Failed("http", "HTTP ${response.code} on part $index", response, index, part.url, d.method), + ) } - rejections++ - if (rejections > ChunkedEngine.PART_HTTP_RETRIES) throw PartRejectedException(index, response) - delay(ChunkedEngine.backoffMs(rejections)) } } - // The stored copy is the truth: a reconcile can have replaced the headers - // or expiresAt. Fall back to the in-memory copy only when the read fails. - private fun latest(): ChunkedManifest = - store.load(uploadId)?.also { manifest = it } ?: manifest!! - - private fun markAccepted(index: Int) { - // Persist the flag first, atomically against concurrent flips and - // reconciles. This is best-effort. A lost flag only re-sends this part on - // a later resume, and the consumer's accept rules absorb that ('already - // completed'). That is better than a failure of an upload that the server - // accepted. - manifest = store.update(uploadId) { it.withPartAccepted(index) } - ?: manifest?.withPartAccepted(index) - manifest?.parts?.get(index)?.let { acceptedBytes.addAndGet(it.size) } + private fun markAccepted(index: Int, part: Part) { + // Remembered here too, so a lost flag write does not re-send the part in this run. + acceptedHere += index + host.ops.markAccepted(host.entryId, host.generation, index) + acceptedBytes.addAndGet(part.size) + lastAcceptedUrl = part.url partSent.remove(index) - reportProgress() + report() } private fun onPartProgress(index: Int, sent: Long) { if (sent == 0L) partSent.remove(index) else partSent[index] = sent - reportProgress() + report() } - private fun reportProgress() { - val total = manifest?.totalBytes ?: return + private fun report() { val sent = (acceptedBytes.get() + partSent.values.sum()).coerceAtMost(total) - UploadProgress.set(uploadId, sent) - EventReporter.progress(uploadId, sent, total) - updateNotification() + host.reportProgress(sent, total) } - - // A resume of a finished-but-unacknowledged upload (all parts accepted, - // 'completed' journaled, and the consumer re-called startUpload before the - // ack) must not mint a second terminal event. Re-emit the journaled one. - // Then a live listener still hears it, with the eventId that the consumer - // will acknowledge. And a trailing run whose completion was already ACKED - // reports nothing at all. See ChunkedEngine.CompletionReport. - private fun journalCompleted(freshCompletion: Boolean) { - val report = ChunkedEngine.completionReport( - EventJournal.get(context).unacknowledged(), - uploadId, - freshCompletion, - ) - when (report) { - is ChunkedEngine.CompletionReport.ReEmit -> EventReporter.emit(report.entry) - // No response fields, because no single response represents N accepted - // parts. - ChunkedEngine.CompletionReport.Mint -> journalAndEmit( - EventJournal.Entry( - eventId = UUID.randomUUID().toString(), - uploadId = uploadId, - type = "completed", - timestamp = System.currentTimeMillis(), - ), - ) - ChunkedEngine.CompletionReport.None -> Unit - } - } - - private fun handleFailure(error: Throwable) { - val entry = when (error) { - is ExpiredException -> errorEntry( - error = "upload expired before every part was accepted", - errorKind = "expired", - ) - is PartRejectedException -> { - val (body, truncated) = EventJournal.capBody(error.response.body) - errorEntry( - error = "HTTP ${error.response.code} on part ${error.partIndex}", - errorKind = "http", - partIndex = error.partIndex, - responseCode = error.response.code, - responseBody = body, - responseBodyTruncated = truncated, - responseHeaders = error.response.headers, - ) - } - is SourceMissingException -> errorEntry(error = error.message!!, errorKind = "file") - is PartBeyondEofException -> errorEntry( - error = error.message!!, - errorKind = "file", - partIndex = error.partIndex, - ) - else -> { - val fileExists = manifest?.let { m -> - runCatching { File(m.sourcePath).exists() }.getOrDefault(true) - } ?: true - errorEntry( - error = error.message ?: "Unknown exception", - errorKind = UploadOutcome.errorKind(error, fileExists), - ) - } - } - journalAndEmit(entry) - } - - private fun errorEntry( - error: String, - errorKind: String, - partIndex: Int? = null, - responseCode: Int? = null, - responseBody: String? = null, - responseBodyTruncated: Boolean = false, - responseHeaders: Map? = null, - ) = EventJournal.Entry( - eventId = UUID.randomUUID().toString(), - uploadId = uploadId, - type = "error", - timestamp = System.currentTimeMillis(), - error = error, - errorKind = errorKind, - partIndex = partIndex, - responseCode = responseCode, - responseBody = responseBody, - responseBodyTruncated = responseBodyTruncated, - responseHeaders = responseHeaders, - ) - - // The semantics are the same as UploadWorker's. Only a user cancel is - // terminal (journaled, cancelReason 'user'). A system stop emits nothing, - // because WorkManager will re-run this upload, and the manifest resumes it. - // The manifest and the bytes are kept in both cases. stopUpload's contract - // is that the next startUpload resumes. - private fun checkAndHandleCancellation(): Boolean { - if (!isStopped) return false - - UploadProgress.remove(uploadId) - - if (!UserCancellations.consume(uploadId)) return true - - journalAndEmit( - EventJournal.Entry( - eventId = UUID.randomUUID().toString(), - uploadId = uploadId, - type = "cancelled", - timestamp = System.currentTimeMillis(), - cancelReason = "user", - ), - ) - return true - } - - private fun journalAndEmit(entry: EventJournal.Entry) { - // A terminal event for an id whose manifest is gone would report an - // upload that nobody owns any more. Either removeUpload deleted it mid-run - // (its work cancel races the in-flight PUT's IOException), or a completed - // ack released it. Suppress the event; iOS's removedIds has the same idea. - // A user cancel keeps its manifest, so real 'cancelled' events pass - // through. - if (!store.contains(uploadId)) return - EventReporter.journalAndEmit(context, entry) - } - - private fun validateAndReportConnectivity(wifiOnly: Boolean): Boolean { - connectivity = validateConnectivity(context, wifiOnly) - updateNotification() - return connectivity == Connectivity.Ok - } - - private fun updateNotification() { - if (manifest?.showsNotification != true) return - notificationManager.notify( - config.systemNotificationId, - buildUploadNotification(context, config, connectivity), - ) - } - - override suspend fun getForegroundInfo(): ForegroundInfo = - uploadForegroundInfo(config, buildUploadNotification(context, config, connectivity)) } - -/** - * The Result that a chunked run returns after it journals a terminal error: - * SUCCESS, deliberately. The journal and the manifest are the upload's outcome - * record, never the WorkManager row state. A row that finishes FAILED destroys - * every appended dependent: WorkManager marks the dependents of a failed - * prerequisite FAILED without a run. Thus a resume enqueued during the failing - * run's teardown window would silently never run (see the APPEND_OR_REPLACE - * note in UploaderModule.enqueueChunkedUpload). getAllUploads derives a - * chunked upload's state from its manifest (allAccepted), not from row states. - */ -internal fun terminalErrorResult(): ListenableWorker.Result = ListenableWorker.Result.success() diff --git a/android/src/main/java/ai/openspace/backgroundupload/ChunkedWorkerGate.kt b/android/src/main/java/ai/openspace/backgroundupload/ChunkedWorkerGate.kt deleted file mode 100644 index 424c627b..00000000 --- a/android/src/main/java/ai/openspace/backgroundupload/ChunkedWorkerGate.kt +++ /dev/null @@ -1,40 +0,0 @@ -package ai.openspace.backgroundupload - -import java.util.concurrent.ConcurrentHashMap - -/** - * At most one [ChunkedUploadWorker] EXECUTES per upload id, process-wide. - * - * The unique-work chain almost guarantees this, but not across a cancel. - * cancelUniqueWork marks the row CANCELLED immediately, while the cancelled - * worker's coroutine still winds down. Thus a startUpload that arrives right - * after a cancelUpload can enqueue (and start) a replacement worker while the - * old worker still has a part PUT in flight. Two concurrent PUTs of one - * partNum are verified unsafe on the server side. A starting worker acquires - * its id here, and a successor waits for the release. - * - * This is also the truthful "is this upload running" for the recreate rule. - * A worker registers before its first manifest read, and it releases in a - * finally block. WorkManager's row state stays RUNNING for a moment after - * doWork returns. This gate does not: it never reports a finished run as - * running. - * - * The gate is same-process only, like [UserCancellations]. A worker in a dead - * process holds nothing, and WorkManager runs our workers in the app process. - */ -object ChunkedWorkerGate { - private val holders = ConcurrentHashMap() - - /** True when [token] now holds the id, or already held it. False while another token holds it. */ - fun tryAcquire(id: String, token: Any): Boolean { - val current = holders.putIfAbsent(id, token) - return current == null || current === token - } - - /** Releases only when [token] is the holder. Thus a stale release cannot evict a successor. */ - fun release(id: String, token: Any) { - holders.remove(id, token) - } - - fun isRunning(id: String): Boolean = holders.containsKey(id) -} diff --git a/android/src/main/java/ai/openspace/backgroundupload/Diag.kt b/android/src/main/java/ai/openspace/backgroundupload/Diag.kt new file mode 100644 index 00000000..14382887 --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/Diag.kt @@ -0,0 +1,19 @@ +package ai.openspace.backgroundupload + +import android.util.Log + +/** + * Logging that is safe in the JVM unit tests. There, android.util.Log is a + * stub that throws, so every call is wrapped. + */ +internal object Diag { + const val TAG = "RNFileUploader" + + fun warn(message: String, error: Throwable? = null) { + runCatching { Log.w(TAG, message, error) } + } + + fun error(message: String, error: Throwable? = null) { + runCatching { Log.e(TAG, message, error) } + } +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/EnqueueRules.kt b/android/src/main/java/ai/openspace/backgroundupload/EnqueueRules.kt new file mode 100644 index 00000000..5b4fdf64 --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/EnqueueRules.kt @@ -0,0 +1,156 @@ +package ai.openspace.backgroundupload + +/** + * The same-id rules of enqueue() (plan 5.4), as pure functions. [decide] + * picks the action; the builders make the next entry from the staged body. + * + * | Stored entry | Action | + * | none, no v9 manifest | Create | + * | none, v9 manifest | AdoptV9: keep the blob and accepted parts | + * | legacy row | Replace (generation + 1) | + * | same body, completed, record present | ReEmit: deliveries + 1, no re-run | + * | same body, completed, record gone | Replace (it was acked) | + * | same body, any other state | Resume (a settled one reopens: gen + 1) | + * | different body, running | RejectRunning (E_RUNNING) | + * | different body, otherwise | Replace (generation + 1, attempts 0) | + */ +object EnqueueRules { + + sealed class Action { + object Create : Action() + data class AdoptV9(val manifest: LegacyManifest) : Action() + data class ReEmit(val eventId: String) : Action() + object Resume : Action() + object Replace : Action() + object RejectRunning : Action() + } + + fun decide( + existing: QueueEntry?, + v9: LegacyManifest?, + incoming: Descriptor, + hasRecord: (eventId: String) -> Boolean, + ): Action { + if (existing == null) return if (v9 != null) Action.AdoptV9(v9) else Action.Create + if (existing.legacy) return Action.Replace + if (existing.sameBodyAs(incoming)) { + if (existing.state == EntryState.COMPLETED) { + val eventId = existing.settledEventId + return if (eventId != null && hasRecord(eventId)) Action.ReEmit(eventId) else Action.Replace + } + return Action.Resume + } + return if (existing.state == EntryState.RUNNING) Action.RejectRunning else Action.Replace + } + + /** + * The generation to stage a body for before the store lock is taken, or + * null to stage under the lock. Only a copied body (JSON, multipart, + * file), and only when no worker can change the decision meanwhile: a new + * id, or a replace of an entry that a worker can not take (not queued, + * not running). A chunked body is a move, which is fast, over a blob that + * a worker may be reading, so it always stages under the lock. + */ + fun preStageGeneration(existing: QueueEntry?, action: Action, incoming: Descriptor): Int? { + val copied = incoming.bodyKind.let { + it == StagedBody.JSON || it == StagedBody.MULTIPART || it == StagedBody.FILE + } + if (!copied) return null + return when (action) { + Action.Create -> 1 + Action.Replace -> existing + ?.takeIf { it.state != EntryState.QUEUED && it.state != EntryState.RUNNING } + ?.let { it.generation + 1 } + else -> null + } + } + + /** The parts an adopted v9 manifest runs with: its accepted flags when the parts are the same. */ + fun adoptedParts(v9: LegacyManifest, incoming: List): List = + if (ChunkedParts.sameParts(v9.parts, incoming)) ChunkedParts.carryAccepted(v9.parts, incoming) + else incoming + + private fun initialState(paused: Boolean) = if (paused) EntryState.PAUSED else EntryState.QUEUED + + /** A new entry (Create, AdoptV9). [parts] carries adopted accepted flags. */ + fun created( + p: EntryParsing.Parsed, + staged: BodyStaging.Staged, + parts: List?, + paused: Boolean, + headerGeneration: Int, + now: Long, + ): QueueEntry { + val runParts = parts ?: p.descriptor.parts + return QueueEntry( + id = p.id, + key = p.key, + varsJson = p.varsJson, + descriptor = p.descriptor.copy(headers = staged.headers, parts = runParts), + body = staged.body, + state = initialState(paused), + attempts = 0, + bytesSent = runParts?.let { ChunkedParts.acceptedBytes(it) } ?: 0L, + totalBytes = staged.body.totalBytes, + expiresAt = p.expiresAt, + createdAt = now, + updatedAt = now, + headerGeneration = headerGeneration, + generation = 1, + ) + } + + /** + * Same body. New headers, expiresAt, vars, accept, retry, and notification + * flag replace the stored ones; the body and accepted parts stay. A settled + * entry reopens with a fresh generation. A running one stays running (the + * worker reads the new headers before its next attempt). + */ + fun resumed( + existing: QueueEntry, + p: EntryParsing.Parsed, + paused: Boolean, + headerGeneration: Int, + now: Long, + ): QueueEntry { + val stored = existing.descriptor!! + val body = existing.body!! + val parts = stored.parts?.let { ChunkedParts.carryAccepted(it, p.descriptor.parts!!) } + val running = existing.state == EntryState.RUNNING + val reopen = existing.isSettled + return existing.copy( + key = p.key, + varsJson = p.varsJson, + descriptor = stored.copy( + headers = BodyStaging.headersFor(p.descriptor.headers, body), + parts = parts, + accept = p.descriptor.accept, + retry = p.descriptor.retry, + noNotification = p.descriptor.noNotification, + ), + state = if (running) EntryState.RUNNING else initialState(paused), + bytesSent = parts?.let { ChunkedParts.acceptedBytes(it) } ?: if (running) existing.bytesSent else 0L, + expiresAt = p.expiresAt, + updatedAt = now, + nextAttemptAt = null, + backoffStreak = 0, + headerGeneration = headerGeneration, + parkedGeneration = null, + generation = if (reopen) existing.generation + 1 else existing.generation, + settledEventId = if (reopen) null else existing.settledEventId, + ) + } + + /** Different body (or over a legacy or acked row). A new life over the same id. */ + fun replaced( + existing: QueueEntry, + p: EntryParsing.Parsed, + staged: BodyStaging.Staged, + paused: Boolean, + headerGeneration: Int, + now: Long, + ): QueueEntry = created(p, staged, null, paused, headerGeneration, now).copy( + createdAt = existing.createdAt, + generation = existing.generation + 1, + ) +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/EntryParsing.kt b/android/src/main/java/ai/openspace/backgroundupload/EntryParsing.kt new file mode 100644 index 00000000..81d06fb9 --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/EntryParsing.kt @@ -0,0 +1,184 @@ +package ai.openspace.backgroundupload + +import com.facebook.react.bridge.ReadableArray +import com.facebook.react.bridge.ReadableMap +import com.facebook.react.bridge.ReadableType +import okhttp3.Headers +import okhttp3.HttpUrl.Companion.toHttpUrlOrNull +import java.net.URI + +/** + * Turns the EnqueueEntry `{ id, key, vars, descriptor }` into Kotlin values. + * JS has already validated the descriptor. Native checks only what it needs + * to run, and rejects anything else with E_INVALID. + * + * A null value is read as absent. The bridge turns a JS `undefined` into + * null, so native can not tell `data: null` from `data: undefined`. + */ +object EntryParsing { + class InvalidEntryException(message: String) : IllegalArgumentException(message) + + data class Parsed( + val id: String, + val key: String, + val varsJson: String, + val descriptor: Descriptor, + val expiresAt: Long, + ) + + private val METHODS = setOf("POST", "PUT", "PATCH", "DELETE", "GET") + + fun parse(entry: ReadableMap): Parsed { + val id = entry.string("id")?.takeIf { it.isNotEmpty() } ?: invalid("id is required") + val key = entry.string("key")?.takeIf { it.isNotEmpty() } ?: invalid("key is required") + val varsJson = JsonBridge.toJson(JsonBridge.valueOf(entry, "vars")) + val d = entry.map("descriptor") ?: invalid("descriptor is required") + val expiresAt = d.number("expiresAt")?.toLong() ?: invalid("descriptor.expiresAt is required") + return Parsed(id, key, varsJson, descriptor(d), expiresAt) + } + + fun descriptor(d: ReadableMap): Descriptor { + val method = (d.string("method") ?: "POST").uppercase() + if (method !in METHODS) invalid("method $method is not supported") + + val parts = d.array("parts")?.let { parseParts(it) } + val url = d.string("url") + if (url == null && parts == null) invalid("url is required unless parts is set") + url?.let { requireHttpUrl(it, "url") } + + val dataJson = if (d.isSet("data")) JsonBridge.toJson(JsonBridge.valueOf(d, "data")) else null + val form = d.array("form")?.let { parseForm(it) } + val file = d.string("file")?.let { stripFileScheme(it) } + val kinds = listOfNotNull(dataJson?.let { "data" }, form?.let { "form" }, file?.let { "file" }) + if (kinds.size > 1) invalid("at most one of data, form, file; got ${kinds.joinToString()}") + if (parts != null && file == null) invalid("parts requires file") + if (method == "GET" && kinds.isNotEmpty()) invalid("a GET request can not carry a body") + + val headers = parseHeaderMap(d.map("headers")) + requireValidHeaders(headers, "headers") + + return Descriptor( + url = url, + method = method, + headers = headers, + dataJson = dataJson, + form = form, + file = file, + parts = parts, + accept = parseAcceptRules(d.array("accept")), + retry = d.map("retry")?.let { parseRetry(it) }, + noNotification = d.map("android")?.bool("noNotification") ?: false, + ) + } + + /** updateHeaders(patch): the header map, checked the same way as a descriptor's. */ + fun headerPatch(patch: ReadableMap): Map = + parseHeaderMap(patch).also { requireValidHeaders(it, "updateHeaders") } + + /** `file:///a%20b` → `/a b`. A plain path is returned as it is. */ + fun stripFileScheme(path: String): String { + if (!path.startsWith("file://")) return path + return runCatching { URI(path).path }.getOrNull() ?: path.removePrefix("file://") + } + + private fun parseParts(arr: ReadableArray): List { + if (arr.size() == 0) invalid("parts must be a non-empty array") + return (0 until arr.size()).map { i -> + val p = arr.getMap(i) ?: invalid("parts[$i] must be an object") + val url = p.string("url") ?: invalid("parts[$i].url is required") + requireHttpUrl(url, "parts[$i].url") + val range = p.map("range") ?: invalid("parts[$i].range is required") + val start = range.number("start")?.toLong() ?: invalid("parts[$i].range.start is required") + val end = range.number("end")?.toLong() ?: invalid("parts[$i].range.end is required") + if (start < 0 || end <= start) invalid("parts[$i].range must satisfy 0 <= start < end") + val headers = parseHeaderMap(p.map("headers")) + requireValidHeaders(headers, "parts[$i].headers") + Part(url = url, headers = headers, start = start, end = end) + } + } + + private fun parseForm(arr: ReadableArray): List { + if (arr.size() == 0) invalid("form must be a non-empty array") + return (0 until arr.size()).map { i -> + val p = arr.getMap(i) ?: invalid("form[$i] must be an object") + val name = p.string("name") ?: invalid("form[$i].name is required") + val contentType = p.string("contentType") ?: invalid("form[$i].contentType is required") + val string = p.string("string") + val path = p.string("path")?.let { stripFileScheme(it) } + if ((string == null) == (path == null)) invalid("form[$i] must set exactly one of string, path") + FormPart(name, contentType, string, path, p.string("fileName")) + } + } + + private fun parseRetry(r: ReadableMap): RetryOverride { + val backoff = r.map("backoff") + val exempt = r.map("terminalHttp")?.array("exempt")?.let { arr -> + (0 until arr.size()).mapNotNull { i -> + if (arr.getType(i) == ReadableType.Number) arr.getDouble(i).toInt() else null + } + } + return RetryOverride( + baseMs = backoff?.number("baseMs")?.toLong(), + maxMs = backoff?.number("maxMs")?.toLong(), + jitter = backoff?.number("jitter"), + exempt = exempt, + ) + } + + internal fun parseAcceptRules(arr: ReadableArray?): List { + if (arr == null) return listOf() + return (0 until arr.size()).mapNotNull { i -> + val rule = arr.getMap(i) ?: return@mapNotNull null + val status = rule.number("status") ?: return@mapNotNull null + UploadOutcome.AcceptRule(status.toInt(), rule.string("bodyIncludes")) + } + } + + /** Header values keep their text. A number is written as JSON would write it. */ + internal fun parseHeaderMap(map: ReadableMap?): Map { + if (map == null) return mapOf() + val out = LinkedHashMap() + JsonBridge.fromReadable(map).forEach { (k, v) -> + when (v) { + null -> Unit + is Double -> out[k] = JsonBridge.numberText(v) + else -> out[k] = v.toString() + } + } + return out + } + + private fun requireHttpUrl(url: String, where: String) { + if (url.toHttpUrlOrNull() == null) invalid("$where is not an http(s) url: $url") + } + + // OkHttp throws on a header name or value it can not send. Check it here, + // so the error is an enqueue rejection and not a failure at attempt time. + private fun requireValidHeaders(headers: Map, where: String) { + try { + val builder = Headers.Builder() + headers.forEach { (k, v) -> builder.add(k, v) } + } catch (e: IllegalArgumentException) { + invalid("$where: ${e.message}") + } + } + + private fun invalid(message: String): Nothing = throw InvalidEntryException(message) + + private fun ReadableMap.isSet(key: String) = hasKey(key) && getType(key) != ReadableType.Null + + private fun ReadableMap.string(key: String): String? = + if (hasKey(key) && getType(key) == ReadableType.String) getString(key) else null + + private fun ReadableMap.number(key: String): Double? = + if (hasKey(key) && getType(key) == ReadableType.Number) getDouble(key) else null + + private fun ReadableMap.bool(key: String): Boolean? = + if (hasKey(key) && getType(key) == ReadableType.Boolean) getBoolean(key) else null + + private fun ReadableMap.map(key: String): ReadableMap? = + if (hasKey(key) && getType(key) == ReadableType.Map) getMap(key) else null + + private fun ReadableMap.array(key: String): ReadableArray? = + if (hasKey(key) && getType(key) == ReadableType.Array) getArray(key) else null +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/EntryTransitions.kt b/android/src/main/java/ai/openspace/backgroundupload/EntryTransitions.kt new file mode 100644 index 00000000..94a297fa --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/EntryTransitions.kt @@ -0,0 +1,86 @@ +package ai.openspace.backgroundupload + +/** + * The state changes of one entry, as pure functions. [QueueController] and + * [WorkerOps] apply them inside `QueueStore.compute`, so each one is atomic + * against the other side. + */ +object EntryTransitions { + + /** A worker takes a queued entry. */ + fun toRunning(e: QueueEntry, now: Long) = + e.copy(state = EntryState.RUNNING, nextAttemptAt = null, updatedAt = now) + + /** A 401/403 under the current header generation. No backoff. */ + fun toParked(e: QueueEntry, headerGeneration: Int, now: Long) = e.copy( + state = EntryState.AWAITING_AUTH, + parkedGeneration = headerGeneration, + backoffStreak = 0, + updatedAt = now, + ) + + /** + * A short backoff that the worker waits out itself. The row stays running + * and shows when the next attempt is due. + */ + fun toBackingOff(e: QueueEntry, nextAttemptAt: Long, now: Long) = + e.copy(nextAttemptAt = nextAttemptAt, updatedAt = now) + + /** One attempt starts: attempts + 1, its X-Request-Id, and no pending backoff. */ + fun toAttempt(e: QueueEntry, requestId: String, now: Long) = e.copy( + attempts = e.attempts + 1, + lastRequestId = requestId, + nextAttemptAt = null, + updatedAt = now, + ) + + /** A backoff too long to wait inside the worker. The streak is kept so the next wait keeps growing. */ + fun toReleased(e: QueueEntry, nextAttemptAt: Long, streak: Int, now: Long) = e.copy( + state = EntryState.QUEUED, + nextAttemptAt = nextAttemptAt, + backoffStreak = streak, + updatedAt = now, + ) + + fun toSettled(e: QueueEntry, state: EntryState, eventId: String, bytesSent: Long, now: Long) = e.copy( + state = state, + settledEventId = eventId, + bytesSent = bytesSent, + nextAttemptAt = null, + parkedGeneration = null, + updatedAt = now, + ) + + /** + * pause(). parkedGeneration is kept so resume() can return the entry to + * awaiting-auth. nextAttemptAt is cleared: resume() retries at once. + */ + fun toPaused(e: QueueEntry, now: Long) = + e.copy(state = EntryState.PAUSED, nextAttemptAt = null, updatedAt = now) + + /** resume(): back to awaiting-auth only when no updateHeaders() came in between. */ + fun toResumed(e: QueueEntry, headerGeneration: Int, now: Long): QueueEntry = + if (e.parkedGeneration != null && e.parkedGeneration == headerGeneration) { + e.copy(state = EntryState.AWAITING_AUTH, updatedAt = now) + } else { + e.copy(state = EntryState.QUEUED, parkedGeneration = null, updatedAt = now) + } + + /** A system stop of a running worker. WorkManager runs the row again. No outcome. */ + fun toStopped(e: QueueEntry, now: Long) = e.copy(state = EntryState.QUEUED, updatedAt = now) + + /** updateHeaders() on a parked entry. */ + fun toUnparked(e: QueueEntry, paused: Boolean, now: Long) = e.copy( + state = if (paused) EntryState.PAUSED else EntryState.QUEUED, + parkedGeneration = null, + updatedAt = now, + ) + + /** Whether a worker of [generation] may still settle [e]. Not after a cancel, a replace, or a settle. */ + fun canSettle(e: QueueEntry?, generation: Int) = + e != null && e.generation == generation && e.isLive + + /** Whether a worker of [generation] still owns the running entry. */ + fun isOwnedRun(e: QueueEntry?, generation: Int) = + e != null && e.generation == generation && e.state == EntryState.RUNNING +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/EntryWorker.kt b/android/src/main/java/ai/openspace/backgroundupload/EntryWorker.kt new file mode 100644 index 00000000..35f67957 --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/EntryWorker.kt @@ -0,0 +1,297 @@ +package ai.openspace.backgroundupload + +import android.app.NotificationManager +import android.content.Context +import androidx.work.CoroutineWorker +import androidx.work.ForegroundInfo +import androidx.work.WorkerParameters +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.delay +import kotlinx.coroutines.withContext +import java.io.File +import java.io.IOException +import kotlin.math.max +import kotlin.math.min + +/** + * Runs one queue entry. The input data holds only the entry id; the worker + * reads the entry from the store at start and again before every attempt, + * so fresh headers and a new expiresAt reach a running worker with no + * restart. + * + * The run: acquire the per-id gate, take the entry (queued → running), run + * the transfer, then settle, park, or release. The body kind picks the + * transfer: [SimpleTransfer] or [ChunkedTransfer]. [UploadWorker] and + * [ChunkedUploadWorker] are the two class names WorkManager knows; both run + * this same code, so a kind change under a queued run is safe. + * + * Every run returns success (see [WorkManagerScheduler] for why). A v9 row, + * which has no entry id, exits at once in silence. + */ +open class EntryWorker(protected val context: Context, params: WorkerParameters) : + CoroutineWorker(context, params) { + + companion object { + private const val GATE_POLL_MS = 100L + /** The poll while the network is unusable (offline, or waiting for wifi). */ + const val CONNECTIVITY_POLL_MS = 10_000L + /** The poll while a short backoff remainder runs out before the run starts. */ + private const val WAIT_POLL_MS = 10_000L + } + + /** A 401/403 under the headers of [headerGeneration] (the entry's value the attempt sent). */ + class ParkException(val headerGeneration: Int) : Exception("awaiting auth") + + /** A backoff too long to wait here. */ + class BackoffException(val nextAttemptAt: Long, val streak: Int) : Exception("released for backoff") + + class ExpiredException : Exception("expired before completion") + + /** The queue was paused between the module's pause and the work cancel reaching us. */ + class PausedException : Exception("queue paused") + + internal lateinit var entryId: String + internal var generation = 0 + internal val store by lazy { QueueStore.get(context) } + internal val ops by lazy { + WorkerOps( + store, + EventJournal.get(context), + QueueSettingsStore.get(context), + EventReporter, + WorkManagerScheduler(context), + ) + } + private val config by lazy { NotificationConfig.load(context) } + private val notificationManager = + context.getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager + + @Volatile + private var connectivity = Connectivity.Ok + + @Volatile + private var showsNotification = false + + final override suspend fun doWork(): Result = withContext(Dispatchers.IO) { + val id = inputData.getString(WorkManagerScheduler.ENTRY_ID_KEY) ?: return@withContext Result.success() + entryId = id + // Acquire before the first store read: a cancel-then-enqueue can start + // this run while the old one still winds down. + while (!WorkerGate.tryAcquire(id, this@EntryWorker)) delay(GATE_POLL_MS) + try { + runEntry() + Result.success() + } catch (error: CancellationException) { + throw error + } catch (error: Throwable) { + // A store failure (disk full at start or during an attempt). Nothing + // was settled; try the run again later. + Diag.error("run of '$id' failed before it could settle; retrying", error) + Result.retry() + } finally { + WorkerGate.release(id, this@EntryWorker) + } + } + + private suspend fun runEntry() { + val initial = store.load(entryId) ?: return // forgotten while queued + if (initial.legacy) return + if (initial.state == EntryState.AWAITING_AUTH) { + // The expiry wake of a parked entry. + if (RetryClassifier.isExpired(now(), initial.expiresAt)) { + ops.settle(entryId, initial.generation, expired(initial)) + } + return + } + if (initial.state != EntryState.QUEUED && initial.state != EntryState.RUNNING) return + if (ops.settings().paused) return + if (!waitUntilDue(initial)) return + val entry = ops.begin(entryId) ?: return + generation = entry.generation + showsNotification = entry.descriptor?.noNotification == false + + var current = entry + var first = true + while (true) { + try { + if (!first) current = ops.latest(entryId, generation) + first = false + startForeground() + val settlement = transfer(current) + endProgress(completed = settlement is Settlement.Completed) + ops.settle(entryId, generation, settlement) + return + } catch (park: ParkException) { + endProgress(completed = false) + // REISSUE: updateHeaders() landed while this attempt was in flight. + if (ops.park(entryId, generation, park.headerGeneration) != WorkerOps.ParkResult.REISSUE) return + } catch (backoff: BackoffException) { + endProgress(completed = false) + ops.release(entryId, generation, backoff.nextAttemptAt, backoff.streak) + return + } catch (error: ExpiredException) { + endProgress(completed = false) + ops.settle(entryId, generation, expired(current)) + return + } catch (error: NotOwnedException) { + endProgress(completed = false) + return + } catch (error: PausedException) { + endProgress(completed = false) + return + } catch (error: CancellationException) { + // A system stop moves a running entry back to queued. A pause or a + // cancel already moved it; then this does nothing. + endProgress(completed = false) + ops.stopped(entryId, generation) + throw error + } catch (error: IOException) { + // A store write failed (disk full, directory briefly unwritable). + // The transfers classify every network IOException themselves, so + // one that lands here is storage: transient, no outcome. Back to + // queued; doWork returns retry. + endProgress(completed = false) + ops.stopped(entryId, generation) + throw error + } catch (error: Throwable) { + endProgress(completed = false) + val d = current.descriptor + ops.settle( + entryId, + generation, + Settlement.Failed( + errorKind = "unknown", + message = error.message ?: error.javaClass.simpleName, + response = null, + partIndex = null, + url = d?.reportUrl ?: "", + method = d?.method ?: "POST", + ), + ) + return + } + } + } + + private suspend fun transfer(entry: QueueEntry): Settlement = + if (entry.body?.kind == StagedBody.CHUNKED) ChunkedTransfer(this).run(entry) + else SimpleTransfer(this).run(entry) + + private fun expired(entry: QueueEntry) = Settlement.Failed( + errorKind = "expired", + message = "expired before completion", + response = null, + partIndex = null, + url = entry.descriptor?.reportUrl ?: "", + method = entry.descriptor?.method ?: "POST", + ) + + /** + * Sleeps out a short backoff remainder. False when the wait is long (the + * wake run comes back for it) or the entry is no longer queued. + */ + private suspend fun waitUntilDue(initial: QueueEntry): Boolean { + var e = initial + while (true) { + val at = e.nextAttemptAt ?: return true + val remaining = at - now() + if (remaining <= 0) return true + if (remaining > RetryClassifier.IN_WORKER_BACKOFF_MAX_MS) return false + delay(min(remaining, WAIT_POLL_MS)) + e = store.load(entryId) ?: return false + if (e.state != EntryState.QUEUED && e.state != EntryState.RUNNING) return false + } + } + + // MARK: - helpers for the transfers + + internal fun now() = System.currentTimeMillis() + + /** + * Waits until the network fits the queue's wifi-only setting. Re-reads + * the settings and the entry at every poll. + */ + internal suspend fun waitForNetwork() { + while (true) { + val s = ops.settings() + if (s.paused) throw PausedException() + val entry = ops.latest(entryId, generation) + if (RetryClassifier.isExpired(now(), entry.expiresAt)) throw ExpiredException() + connectivity = validateConnectivity(context, s.wifiOnly) + updateNotification() + if (connectivity == Connectivity.Ok) return + delay(CONNECTIVITY_POLL_MS) + } + } + + /** The descriptor's headers, the part's over them, and X-Request-Id over all. */ + internal fun headersFor(d: Descriptor, part: Part?, requestId: String): Map { + val withPart = if (part == null) d.headers else HeaderMap.merge(d.headers, part.headers) + return HeaderMap.merge(withPart, mapOf("X-Request-Id" to requestId)) + } + + internal fun policy(entry: QueueEntry) = + RetryClassifier.policy(ops.settings().retry, entry.descriptor?.retry) + + /** + * A short backoff waits here, with the row still running and showing + * nextAttemptAt; a long one throws [BackoffException] to release the worker. + */ + internal suspend fun backoffOrRelease(policy: RetryClassifier.Policy, streak: Int, expiresAt: Long) { + val backoff = RetryClassifier.backoffMs(policy, streak) + val now = now() + if (backoff <= RetryClassifier.IN_WORKER_BACKOFF_MAX_MS) { + val wait = min(backoff, max(0L, expiresAt - now)) + ops.backingOff(entryId, generation, now + wait) + delay(wait) + return + } + throw BackoffException(RetryClassifier.nextAttemptAt(now, backoff, expiresAt), streak) + } + + internal fun reportProgress(sent: Long, total: Long) { + UploadProgress.set(entryId, sent) + EventReporter.progress(entryId, sent, total) + updateNotification() + } + + internal fun bodyFile(entry: QueueEntry): File? = store.bodyFile(entry) + + private fun endProgress(completed: Boolean) { + if (completed) UploadProgress.complete(entryId) else UploadProgress.remove(entryId) + EventReporter.flushProgress(entryId) + EventReporter.dropProgress(entryId) + } + + // MARK: - notification + + // v9 rules. A suppressed notification means no foreground mode. A denied + // foreground start (API 31+, app in the background: the usual case for a + // WorkManager relaunch) is not a failure; the transfer runs without + // foreground priority. Any other failure is logged and the run goes on. + private suspend fun startForeground() { + if (!showsNotification) return + try { + ensureNotificationChannel(notificationManager, config) + setForeground(getForegroundInfo()) + } catch (error: CancellationException) { + throw error + } catch (error: Throwable) { + if (!isForegroundStartDenied(error)) Diag.warn("foreground start failed; running without it", error) + } + } + + private fun updateNotification() { + if (!showsNotification) return + runCatching { + notificationManager.notify( + config.systemNotificationId, + buildUploadNotification(context, config, connectivity), + ) + } + } + + override suspend fun getForegroundInfo(): ForegroundInfo = + uploadForegroundInfo(config, buildUploadNotification(context, config, connectivity)) +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/EventJournal.kt b/android/src/main/java/ai/openspace/backgroundupload/EventJournal.kt index 2dd718b3..03535d68 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/EventJournal.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/EventJournal.kt @@ -1,82 +1,139 @@ package ai.openspace.backgroundupload import android.content.Context +import com.facebook.react.bridge.WritableMap import com.google.gson.Gson import java.io.File -// Durable record of terminal upload events (completed / error / cancelled). -// Written BEFORE the event is emitted to JS; deleted only when JS acknowledges. -// One JSON file per event named .json — tmp+rename keeps each write -// self-contained so a crash mid-append can never corrupt other entries. -// -// `maxEntries` is a runaway guard: the design assumes JS drains the journal via -// ack() on every boot, but if that loop breaks (or a consumer hasn't adopted it -// yet) the directory would grow without bound. When exceeded we drop the OLDEST -// entries. Set high enough that legitimate heavy offline use won't hit it — this -// only fires in the pathological "nothing ever acks" case. -class EventJournal( - private val dir: File, - private val maxEntries: Int = MAX_ENTRIES, -) { - - data class Entry( +/** + * The durable record of settled outcomes (completed, error, cancelled). A + * record is written BEFORE the outcome is emitted to JS and deleted only when + * JS acknowledges it. One JSON file per record, `.json`, written + * with tmp + fsync + rename, so a crash mid-write can not corrupt another + * record. + * + * [maxEntries] is a runaway guard: if nothing ever acknowledges, the oldest + * records are dropped. It only fires in that broken case. + */ +class EventJournal(private val dir: File, private val maxEntries: Int = MAX_ENTRIES) { + + /** RawResponse. [status] is null for a chunked completion. */ + data class Response( + val status: Int?, + val headers: Map?, + val body: String?, + val bodyTruncated: Boolean, + ) { + fun toMap(): Map = LinkedHashMap().apply { + status?.let { put("status", it.toDouble()) } + headers?.let { put("headers", it) } + body?.let { put("body", it) } + put("bodyTruncated", bodyTruncated) + } + + companion object { + fun of(response: UploadResponse): Response { + val (body, truncated) = capBody(response.body) + return Response(response.code, response.headers, body, truncated) + } + + /** A chunked completion: N parts, no one response. */ + val NONE = Response(null, null, null, false) + } + } + + /** One settled outcome, in the SettledEvent shape plus [generation]. */ + data class SettledRecord( val eventId: String, - val uploadId: String, - val type: String, // completed | error | cancelled - val timestamp: Long, - val responseCode: Int? = null, - val responseBody: String? = null, - val responseBodyTruncated: Boolean = false, - val responseHeaders: Map? = null, - val error: String? = null, - val errorKind: String? = null, // http | network | file | expired | unknown - val cancelReason: String? = null, // user | system - // Chunked uploads: the index of the failing part, when one part's response - // caused the error. - val partIndex: Int? = null, + val id: String, + val key: String, + val varsJson: String, + val at: Long, + val attempts: Int, + val requestId: String?, + /** + * How many times this outcome reached JS: 1 after a live emit, 0 when it + * was journaled with JS dead; +1 per later delivery (replay, re-emit). + */ + val deliveries: Int, + /** The entry state this outcome puts it in. */ + val state: String, + val bytesSent: Long, + val totalBytes: Long, + val url: String, + val method: String, + val partIndex: Int?, + /** completed | error | cancelled */ + val kind: String, + /** completed; also error with errorKind http. */ + val response: Response?, + val errorKind: String?, + val message: String?, + val cancelReason: String?, + /** The entry life this belongs to (same-id rule 6). */ + val generation: Int, ) { - fun toWritableMap(): com.facebook.react.bridge.WritableMap = - com.facebook.react.bridge.Arguments.createMap().apply { - putString("eventId", eventId) - putString("id", uploadId) - putString("type", type) - putDouble("timestamp", timestamp.toDouble()) - responseCode?.let { putInt("responseCode", it) } - responseBody?.let { putString("responseBody", it) } - if (responseBodyTruncated) putBoolean("responseBodyTruncated", true) - responseHeaders?.let { - putMap("responseHeaders", com.facebook.react.bridge.Arguments.makeNativeMap(it)) - } - error?.let { putString("error", it) } - errorKind?.let { putString("errorKind", it) } - cancelReason?.let { putString("cancelReason", it) } - partIndex?.let { putInt("partIndex", it) } + fun toMap(): Map = LinkedHashMap().apply { + put("eventId", eventId) + put("id", id) + put("key", key) + put("vars", runCatching { JsonBridge.parse(varsJson) }.getOrNull()) + put("at", at.toDouble()) + put("attempts", attempts.toDouble()) + requestId?.let { put("requestId", it) } + put("deliveries", deliveries.toDouble()) + put("state", state) + put("bytesSent", bytesSent.toDouble()) + put("totalBytes", totalBytes.toDouble()) + put("url", url) + put("method", method) + partIndex?.let { put("partIndex", it.toDouble()) } + put("kind", kind) + when (kind) { + KIND_COMPLETED -> put("response", (response ?: Response.NONE).toMap()) + KIND_ERROR -> put("error", LinkedHashMap().apply { + put("errorKind", errorKind ?: "unknown") + put("message", message ?: "") + response?.let { put("response", it.toMap()) } + partIndex?.let { put("partIndex", it.toDouble()) } + }) + KIND_CANCELLED -> put("cancelReason", cancelReason ?: "user") } + } + + fun toWritableMap(): WritableMap = JsonBridge.toWritableMap(toMap()) } companion object { - const val MAX_BODY_CHARS = 64 * 1024 + const val KIND_COMPLETED = "completed" + const val KIND_ERROR = "error" + const val KIND_CANCELLED = "cancelled" + + /** 1 MB, the RawResponse cap. */ + const val MAX_BODY_CHARS = 1_048_576 const val MAX_ENTRIES = 1000 private val gson = Gson() - // Char-count cap (not byte-accurate: splitting on a byte boundary risks - // cutting a surrogate pair; a slightly loose cap is fine as a safety limit). - // Returns the (possibly truncated) body and whether truncation occurred. - // Single source of truth so the journaled copy and the live-emitted copy match. - fun capBody(body: String?): Pair = - if (body != null && body.length > MAX_BODY_CHARS) - body.substring(0, MAX_BODY_CHARS) to true - else body to false + // Event ids are UUIDs that native mints. ackEvents takes ids from JS, and + // an id is a file name here, so anything else is ignored. + private val EVENT_ID = Regex("^[A-Za-z0-9-]{1,64}$") + + fun isValidEventId(id: String) = EVENT_ID.matches(id) + + /** + * A char-count cap. It is not byte-exact: a cut on a byte boundary could + * split a surrogate pair. Returns the body and whether it was cut. + */ + fun capBody(body: String?, max: Int = MAX_BODY_CHARS): Pair = + if (body != null && body.length > max) body.substring(0, max) to true else body to false @Volatile private var instance: EventJournal? = null - // The worker may run in a process where React never initialized, so the - // journal must be reachable from a bare Context, not the module. + /** v10 records live in `rnbgupload-settled`. The v9 `rnbgupload-events` is read once by [LegacyImport]. */ fun get(context: Context): EventJournal = instance ?: synchronized(this) { - instance - ?: EventJournal(File(context.filesDir, "rnbgupload-events")).also { instance = it } + instance ?: EventJournal(File(context.filesDir, "rnbgupload-settled")).also { instance = it } } } @@ -84,61 +141,92 @@ class EventJournal( dir.mkdirs() } + private fun fileFor(eventId: String) = File(dir, "$eventId.json") + + /** + * Never throws. The worker calls this right after the server accepted the + * request. A thrown IOException would look like a transient failure and + * re-send the request. Losing one record is the lesser harm, so a failure + * returns false and the caller goes on. + */ @Synchronized - fun append(entry: Entry) { - // Defensive cap in case a caller didn't pre-cap; idempotent when it did. - val (body, truncated) = capBody(entry.responseBody) - val bounded = - if (truncated) entry.copy(responseBody = body, responseBodyTruncated = true) else entry - // A journal write must NEVER throw into the caller. The worker calls this - // right after a successful upload; a propagated IOException (e.g. disk full) - // would be classified as a retryable error and re-run the upload, sending - // duplicate data to the server. Losing one journal entry is the lesser evil. - try { - val tmp = File(dir, "${entry.eventId}.tmp") - tmp.writeText(gson.toJson(bounded)) - tmp.renameTo(File(dir, "${entry.eventId}.json")) + fun append(record: SettledRecord): Boolean { + val bounded = record.response?.let { r -> + val (body, truncated) = capBody(r.body) + if (truncated) record.copy(response = r.copy(body = body, bodyTruncated = true)) else record + } ?: record + val written = try { + AtomicFiles.writeText(fileFor(record.eventId), gson.toJson(bounded)) + true } catch (t: Throwable) { - t.printStackTrace() - return + Diag.error("journal append failed for ${record.eventId}", t) + false } pruneToMax() + return written } - // Keep the directory bounded. Prune by file modification time (no parsing) - // rather than the entry's own timestamp — cheaper, and close enough since a - // file's mtime is when it was journaled. Guarded: a prune failure must not - // propagate for the same reason append() must not. + // Prunes by file time (no parsing). Guarded for the same reason as append. private fun pruneToMax() { try { - // Sweep orphaned .tmp files (writeText succeeded but rename failed). - dir.listFiles { f -> f.extension == "tmp" }?.forEach { it.delete() } + dir.listFiles { f -> f.name.endsWith(AtomicFiles.TMP_SUFFIX) }?.forEach { it.delete() } val files = dir.listFiles { f -> f.extension == "json" } ?: return if (files.size <= maxEntries) return - files.sortedBy { it.lastModified() } - .take(files.size - maxEntries) - .forEach { it.delete() } + files.sortedBy { it.lastModified() }.take(files.size - maxEntries).forEach { it.delete() } } catch (t: Throwable) { - t.printStackTrace() + Diag.error("journal prune failed", t) } } + /** Every record, oldest first. Corrupt files are skipped. */ @Synchronized - @Suppress("SENSELESS_COMPARISON") // Gson can inject null into a non-null field - fun unacknowledged(): List = + fun unacknowledged(): List = (dir.listFiles { f -> f.extension == "json" } ?: emptyArray()) - .mapNotNull { f -> - runCatching { gson.fromJson(f.readText(), Entry::class.java) }.getOrNull() - } - // Gson bypasses the constructor, so a file missing a field yields null - // despite the non-null Kotlin type. Check every field JS relies on being - // present, not just eventId — an entry reaching JS with a null `type` - // would fall silently through a `switch (event.type)`. - .filter { it.eventId != null && it.uploadId != null && it.type != null } - .sortedBy { it.timestamp } + .mapNotNull { read(it) } + .sortedBy { it.at } + + @Synchronized + fun find(eventId: String): SettledRecord? = + if (isValidEventId(eventId)) read(fileFor(eventId)) else null + + @Synchronized + fun forEntry(id: String): List = unacknowledged().filter { it.id == id } + + /** + * One more delivery of [eventId]: rewrites the record with deliveries + 1 + * and returns it. Null when the record is gone. When the rewrite fails the + * incremented record is still returned, so the delivery goes ahead. + */ + @Synchronized + fun incrementDeliveries(eventId: String): SettledRecord? { + val record = find(eventId) ?: return null + val next = record.copy(deliveries = record.deliveries + 1) + try { + AtomicFiles.writeText(fileFor(eventId), gson.toJson(next)) + } catch (t: Throwable) { + Diag.error("journal deliveries update failed for $eventId", t) + } + return next + } + /** Idempotent. Unknown and malformed ids are ignored. */ @Synchronized fun ack(eventIds: List) { - eventIds.forEach { File(dir, "$it.json").delete() } + eventIds.filter { isValidEventId(it) }.forEach { fileFor(it).delete() } + } + + @Suppress("SENSELESS_COMPARISON", "USELESS_ELVIS") + private fun read(file: File): SettledRecord? { + if (!file.exists()) return null + val r = runCatching { gson.fromJson(file.readText(), SettledRecord::class.java) }.getOrNull() + ?: return null + // Gson does not run constructors. Check every field JS relies on. + if (r.eventId == null || r.id == null || r.key == null || r.kind == null || r.state == null) return null + return r.copy( + varsJson = r.varsJson ?: "null", + url = r.url ?: "", + method = r.method ?: "POST", + deliveries = r.deliveries.coerceAtLeast(0), + ) } } diff --git a/android/src/main/java/ai/openspace/backgroundupload/EventReporter.kt b/android/src/main/java/ai/openspace/backgroundupload/EventReporter.kt index 46460e1e..68121854 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/EventReporter.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/EventReporter.kt @@ -1,48 +1,72 @@ package ai.openspace.backgroundupload -import android.content.Context import com.facebook.react.bridge.Arguments -// Sends live events to JS through the module's codegen event emitters. Terminal -// outcomes are journaled before they reach here, so when JS is absent (headless -// worker, mid-reload) dropping the live event costs nothing — the consumer picks -// it up from getUnacknowledgedEvents instead. -object EventReporter { - - // Journal first, then emit. The journal is the durable record; it survives - // when JS is dead. The live emit is best-effort. The two carry the identical - // payload. Thus a consumer can acknowledge a live event by its eventId. - fun journalAndEmit(context: Context, entry: EventJournal.Entry) { - EventJournal.get(context).append(entry) - emit(entry) +/** The events that queue transitions produce. [EventReporter] sends them to JS; tests record them. */ +interface QueueEvents { + fun state(row: RequestRow) + + /** The caller journaled [record] first. */ + fun settled(record: EventJournal.SettledRecord) + + /** + * Whether a live emit can reach a JS listener now. A record journaled + * while no listener is there (JS dead, or alive but not yet subscribed) + * starts at 0 deliveries, so its first real delivery (the replay) counts + * as 1. + */ + fun canDeliver(): Boolean +} + +/** + * Sends live events to JS through the module's codegen emitters. With no + * module (a headless worker, a reload) an event is dropped. Settled outcomes + * are journaled before they reach here, so a dropped one is replayed from + * the journal. + */ +object EventReporter : QueueEvents { + + private val throttle = ProgressThrottle { id, sent, total -> + val module = UploaderModule.instance ?: return@ProgressThrottle + module.emitProgress(Arguments.createMap().apply { + putString("id", id) + putDouble("bytesSent", sent.toDouble()) + putDouble("totalBytes", total.toDouble()) + }) } - // Emit a terminal event from its journal entry, so the live event carries the - // exact same payload (incl. eventId) as the journaled copy — letting a consumer - // ackEvents([eventId]) right after handling a live event, and keeping iOS/Android - // event shapes identical. - fun emit(entry: EventJournal.Entry) { + override fun state(row: RequestRow) { val module = UploaderModule.instance ?: return - val params = entry.toWritableMap() - when (entry.type) { - "completed" -> module.emitCompletedEvent(params) - "cancelled" -> module.emitCancelledEvent(params) - else -> module.emitErrorEvent(params) - } + module.emitState(JsonBridge.toWritableMap(row.toMap())) } - fun progress(uploadId: String, bytesSentTotal: Long, contentLength: Long) { + override fun settled(record: EventJournal.SettledRecord) { val module = UploaderModule.instance ?: return - module.emitProgressEvent(Arguments.createMap().apply { - putString("id", uploadId) - // Guard against a zero-byte file (contentLength == 0) producing NaN. - val pct = if (contentLength <= 0) 0.0 else bytesSentTotal.toDouble() * 100 / contentLength - putDouble("progress", pct) // 0-100 - }) + module.emitSettled(record.toWritableMap()) + } + + override fun canDeliver(): Boolean = UploaderModule.instance?.listening == true + + /** Moves the row's bytesSent in memory and emits through the throttle. */ + fun progress(id: String, sent: Long, total: Long) { + RequestIndex.shared.setBytes(id, sent) + throttle.offer(id, sent, total, isForeground()) + } + + fun flushProgress(id: String) = throttle.flush(id) + + fun dropProgress(id: String) = throttle.drop(id) + + fun attempt(event: AttemptEvent) { + val module = UploaderModule.instance ?: return + module.emitAttempt(event.toWritableMap()) } fun notification() { val module = UploaderModule.instance ?: return - module.emitNotificationEvent(Arguments.createMap()) + module.emitNotification(Arguments.createMap()) } + + private fun isForeground(): Boolean = + runCatching { UploaderModule.instance?.isForeground() == true }.getOrDefault(false) } diff --git a/android/src/main/java/ai/openspace/backgroundupload/JsonBridge.kt b/android/src/main/java/ai/openspace/backgroundupload/JsonBridge.kt new file mode 100644 index 00000000..a84b8661 --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/JsonBridge.kt @@ -0,0 +1,176 @@ +package ai.openspace.backgroundupload + +import com.facebook.react.bridge.Arguments +import com.facebook.react.bridge.ReadableArray +import com.facebook.react.bridge.ReadableMap +import com.facebook.react.bridge.ReadableType +import com.facebook.react.bridge.WritableArray +import com.facebook.react.bridge.WritableMap +import com.google.gson.GsonBuilder +import com.google.gson.JsonArray +import com.google.gson.JsonElement +import com.google.gson.JsonNull +import com.google.gson.JsonObject +import com.google.gson.JsonParser +import com.google.gson.JsonPrimitive +import kotlin.math.abs +import kotlin.math.floor + +/** + * Moves values between three forms: the bridge (ReadableMap, WritableMap), + * plain Kotlin values (Map, List, String, Double, Boolean, null), and JSON + * text. `vars` and `data` are stored as JSON text. + * + * Numbers: RN gives every JS number to Kotlin as a Double. A Double with no + * fraction is written as an integer, so `{ n: 1 }` becomes `{"n":1}`, as + * JSON.stringify writes it, not `{"n":1.0}`. + * + * Key order: the bridge does not keep the JS key order. Object keys are + * written sorted, so the same object always gives the same text. The + * same-body check compares that text. + */ +object JsonBridge { + private val gson = GsonBuilder().serializeNulls().disableHtmlEscaping().create() + + // 2^53. Above this a Double can not hold every integer, so it keeps the + // Double form. + private const val MAX_SAFE_INTEGER = 9_007_199_254_740_992.0 + + fun fromReadable(map: ReadableMap): Map { + val out = LinkedHashMap() + val keys = map.keySetIterator() + while (keys.hasNextKey()) { + val key = keys.nextKey() + out[key] = valueOf(map, key) + } + return out + } + + fun fromReadableArray(array: ReadableArray): List = + (0 until array.size()).map { i -> + when (array.getType(i)) { + ReadableType.Null -> null + ReadableType.Boolean -> array.getBoolean(i) + ReadableType.Number -> array.getDouble(i) + ReadableType.String -> array.getString(i) + ReadableType.Map -> array.getMap(i)?.let { fromReadable(it) } + ReadableType.Array -> array.getArray(i)?.let { fromReadableArray(it) } + } + } + + /** The plain value at [key]. Null for an absent key. */ + fun valueOf(map: ReadableMap, key: String): Any? { + if (!map.hasKey(key)) return null + return when (map.getType(key)) { + ReadableType.Null -> null + ReadableType.Boolean -> map.getBoolean(key) + ReadableType.Number -> map.getDouble(key) + ReadableType.String -> map.getString(key) + ReadableType.Map -> map.getMap(key)?.let { fromReadable(it) } + ReadableType.Array -> map.getArray(key)?.let { fromReadableArray(it) } + } + } + + /** Plain values to JSON text. */ + fun toJson(value: Any?): String = gson.toJson(toElement(value)) + + /** JSON text to plain values. Throws on malformed text. Numbers come back as Double. */ + fun parse(json: String): Any? = fromElement(JsonParser.parseString(json)) + + /** A number as JSON would print it: an integer when it has no fraction. */ + fun numberText(d: Double): String = gson.toJson(number(d)) + + private fun number(d: Double): JsonPrimitive = + if (d.isFinite() && d == floor(d) && abs(d) < MAX_SAFE_INTEGER) JsonPrimitive(d.toLong()) + else JsonPrimitive(d) + + private fun toElement(value: Any?): JsonElement = when (value) { + null -> JsonNull.INSTANCE + is Boolean -> JsonPrimitive(value) + is Double -> number(value) + is Float -> number(value.toDouble()) + is Int, is Long, is Short, is Byte -> JsonPrimitive((value as Number).toLong()) + is Number -> JsonPrimitive(value) + is String -> JsonPrimitive(value) + is Map<*, *> -> JsonObject().apply { + value.entries + .sortedBy { it.key.toString() } + .forEach { (k, v) -> add(k.toString(), toElement(v)) } + } + is Iterable<*> -> JsonArray().apply { value.forEach { add(toElement(it)) } } + is Array<*> -> JsonArray().apply { value.forEach { add(toElement(it)) } } + else -> JsonPrimitive(value.toString()) + } + + private fun fromElement(element: JsonElement): Any? = when { + element.isJsonNull -> null + element.isJsonObject -> LinkedHashMap().apply { + element.asJsonObject.entrySet().forEach { (k, v) -> put(k, fromElement(v)) } + } + element.isJsonArray -> element.asJsonArray.map { fromElement(it) } + else -> { + val p = element.asJsonPrimitive + when { + p.isBoolean -> p.asBoolean + p.isNumber -> p.asDouble + else -> p.asString + } + } + } + + /** + * Plain values to the bridge. The factories default to the native ones. The + * JVM tests pass JavaOnlyMap and JavaOnlyArray, because the native ones need + * the React Native C++ library. + */ + fun toWritableMap( + map: Map, + newMap: () -> WritableMap = Arguments::createMap, + newArray: () -> WritableArray = Arguments::createArray, + ): WritableMap { + val out = newMap() + map.forEach { (k, v) -> putValue(out, k, v, newMap, newArray) } + return out + } + + fun toWritableArray( + list: List, + newMap: () -> WritableMap = Arguments::createMap, + newArray: () -> WritableArray = Arguments::createArray, + ): WritableArray { + val out = newArray() + list.forEach { v -> + when (v) { + null -> out.pushNull() + is Boolean -> out.pushBoolean(v) + is Number -> out.pushDouble(v.toDouble()) + is String -> out.pushString(v) + is Map<*, *> -> out.pushMap(toWritableMap(stringKeys(v), newMap, newArray)) + is List<*> -> out.pushArray(toWritableArray(v, newMap, newArray)) + else -> out.pushString(v.toString()) + } + } + return out + } + + fun putValue( + map: WritableMap, + key: String, + value: Any?, + newMap: () -> WritableMap = Arguments::createMap, + newArray: () -> WritableArray = Arguments::createArray, + ) { + when (value) { + null -> map.putNull(key) + is Boolean -> map.putBoolean(key, value) + is Number -> map.putDouble(key, value.toDouble()) + is String -> map.putString(key, value) + is Map<*, *> -> map.putMap(key, toWritableMap(stringKeys(value), newMap, newArray)) + is List<*> -> map.putArray(key, toWritableArray(value, newMap, newArray)) + else -> map.putString(key, value.toString()) + } + } + + private fun stringKeys(map: Map<*, *>): Map = + map.entries.associate { (k, v) -> k.toString() to v } +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/LegacyImport.kt b/android/src/main/java/ai/openspace/backgroundupload/LegacyImport.kt new file mode 100644 index 00000000..d00e97c5 --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/LegacyImport.kt @@ -0,0 +1,101 @@ +package ai.openspace.backgroundupload + +import android.content.Context +import com.google.gson.Gson +import java.io.File + +/** + * First v10 launch. Each v9 journal entry becomes a read-only settled row + * with key `legacy` and the v9 upload id. Nothing is delivered; the app + * reads the rows with getRequests() and cancels them. v9 chunked manifests + * are left in place: a same-id enqueue adopts them. + * + * The caller cancels the v9 WorkManager rows after this returns true. The + * order is safe: under v10 code a v9 row exits at once (it has no entry + * id), so nothing can write a v9 journal file after the import. + * + * Runs once, guarded by a marker file. A failed row save leaves the marker + * unwritten, so the next launch tries again. + */ +object LegacyImport { + const val MARKER = "v9-imported" + const val KEY = "legacy" + const val V9_JOURNAL_DIR = "rnbgupload-events" + + /** The v9 journal Entry, every field nullable: Gson reads whatever is there. */ + data class V9Entry( + val eventId: String?, + val uploadId: String?, + val type: String?, + val timestamp: Long?, + ) + + private val gson = Gson() + + /** Returns true when the import ran at this launch (no marker yet). */ + fun runOnce(context: Context, store: QueueStore): Boolean { + val marker = File(QueueStore.rootDir(context), MARKER) + if (marker.exists()) return false + val complete = import(File(context.filesDir, V9_JOURNAL_DIR), store) + if (complete) { + runCatching { AtomicFiles.writeText(marker, "1") } + .onFailure { Diag.error("could not write the v9 import marker", it) } + } + return true + } + + /** + * Imports every v9 journal entry in [v9Dir]. The newest entry per upload + * id wins. Returns false when a row could not be saved (its file is kept). + */ + internal fun import(v9Dir: File, store: QueueStore): Boolean { + val files = v9Dir.listFiles { f -> f.extension == "json" } ?: return true + val read = files.mapNotNull { f -> + val entry = runCatching { gson.fromJson(f.readText(), V9Entry::class.java) }.getOrNull() + if (entry == null) { + Diag.warn("v9 journal file unreadable, skipped: ${f.name}") + f.delete() + null + } else f to entry + } + var complete = true + read.groupBy { it.second.uploadId }.forEach { (uploadId, group) -> + val newest = group.maxByOrNull { it.second.timestamp ?: 0L }!!.second + val row = if (uploadId == null) null else legacyRow(newest) + val saved = when { + row == null -> true + store.load(row.id) != null -> true // a v10 entry already owns the id + else -> runCatching { store.save(row) }.isSuccess + } + if (saved) group.forEach { it.first.delete() } else complete = false + } + return complete + } + + internal fun legacyRow(entry: V9Entry): QueueEntry? { + val id = entry.uploadId ?: return null + val state = when (entry.type) { + "completed" -> EntryState.COMPLETED + "error" -> EntryState.ERROR + "cancelled" -> EntryState.CANCELLED + else -> return null + } + val at = entry.timestamp ?: 0L + return QueueEntry( + id = id, + key = KEY, + varsJson = "null", + descriptor = null, + body = null, + state = state, + attempts = 0, + bytesSent = 0, + totalBytes = 0, + expiresAt = at, + createdAt = at, + updatedAt = at, + generation = 1, + legacy = true, + ) + } +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/ProgressThrottle.kt b/android/src/main/java/ai/openspace/backgroundupload/ProgressThrottle.kt new file mode 100644 index 00000000..f4032d20 --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/ProgressThrottle.kt @@ -0,0 +1,55 @@ +package ai.openspace.backgroundupload + +/** + * Limits progress events per id: at most one per second while the app is in + * the foreground, one per 10 minutes in the background. A value held back is + * kept as pending; [flush] sends it (the trailing edge on settle, park, + * release, and stop). + */ +class ProgressThrottle( + private val clock: () -> Long = System::currentTimeMillis, + private val emit: (id: String, sent: Long, total: Long) -> Unit, +) { + companion object { + const val FOREGROUND_MS = 1_000L + const val BACKGROUND_MS = 600_000L + } + + private class Slot(var lastEmitAt: Long?, var pending: Pair?) + + private val slots = HashMap() + + fun offer(id: String, sent: Long, total: Long, foreground: Boolean) { + val interval = if (foreground) FOREGROUND_MS else BACKGROUND_MS + val now = clock() + val send = synchronized(slots) { + val slot = slots.getOrPut(id) { Slot(null, null) } + val last = slot.lastEmitAt + if (last == null || now - last >= interval) { + slot.lastEmitAt = now + slot.pending = null + true + } else { + slot.pending = sent to total + false + } + } + if (send) emit(id, sent, total) + } + + /** Sends the held-back value once, if there is one. */ + fun flush(id: String) { + val pending = synchronized(slots) { + val slot = slots[id] ?: return + val p = slot.pending ?: return + slot.pending = null + slot.lastEmitAt = clock() + p + } + emit(id, pending.first, pending.second) + } + + fun drop(id: String) { + synchronized(slots) { slots.remove(id) } + } +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/QueueController.kt b/android/src/main/java/ai/openspace/backgroundupload/QueueController.kt new file mode 100644 index 00000000..810895d7 --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/QueueController.kt @@ -0,0 +1,422 @@ +package ai.openspace.backgroundupload + +import java.io.IOException +import java.util.UUID + +/** + * Every transition that JS causes: enqueue, pause, resume, cancel, + * setWifiOnly, updateHeaders, ack, and the boot sweep. [UploaderModule] calls + * it from one single-thread executor, so these calls never overlap each other. + * Workers change entries at the same time; every change here is inside the + * store lock, so each one is atomic against them. + * + * Order of a change: journal (when there is an outcome), store, work + * schedule, then events. Events go out after the store lock is released. + */ +class QueueController( + private val store: QueueStore, + private val journal: EventJournal, + private val settings: QueueSettingsStore, + private val events: QueueEvents, + private val scheduler: WorkScheduler, + private val isWorkerRunning: (id: String) -> Boolean = WorkerGate::isRunning, + private val clock: () -> Long = System::currentTimeMillis, +) { + + // MARK: - enqueue + + private class Enqueued(val entry: QueueEntry?, val reEmit: EventJournal.SettledRecord?) + + /** A body staged before the store lock was taken, and the generation it was staged for. */ + private class PreStaged(val generation: Int, val staged: BodyStaging.Staged) + + /** Test seam: runs between the staging outside the lock and the commit under it. */ + internal var afterPreStage: () -> Unit = {} + + /** + * Persists the entry and every staged byte, then schedules it. Returns the + * id. Throws [QueueException] with E_RUNNING, E_FILE_MISSING, E_STORAGE, or + * E_INVALID. + * + * A copied body is staged before the store lock when the decision can not + * change meanwhile (see [EnqueueRules.preStageGeneration]), so a large + * copy does not block every worker transition. Under the lock the + * decision is made again; a staged body that no longer fits is deleted + * and the body is staged again under the lock. + */ + fun enqueue(p: EntryParsing.Parsed): String { + val pre = preStage(p) + var preUsed = false + fun preFor(generation: Int): BodyStaging.Staged? = + pre?.takeIf { it.generation == generation }?.staged?.also { preUsed = true } + val result = try { + store.locked { decideAndCommit(p, ::preFor) } + } finally { + if (pre != null && !preUsed) discard(p.id, pre) + } + result.reEmit?.let { events.settled(it) } + result.entry?.let { entry -> + scheduleRun(entry) + events.state(entry.toRow()) + } + return p.id + } + + private fun preStage(p: EntryParsing.Parsed): PreStaged? { + val generation = store.locked { + val existing = store.load(p.id) + val v9 = if (existing == null) store.legacyManifest(p.id) else null + val action = EnqueueRules.decide(existing, v9, p.descriptor) { journal.find(it) != null } + EnqueueRules.preStageGeneration(existing, action, p.descriptor) + } ?: return null + val staged = stageOrThrow(p.descriptor, store.entryDir(p.id), generation) + afterPreStage() + return PreStaged(generation, staged) + } + + /** Deletes a pre-staged body that the commit did not use, unless the stored entry points at it. */ + private fun discard(id: String, pre: PreStaged) { + val file = pre.staged.body.fileName?.let { java.io.File(store.entryDir(id), it) } ?: return + val current = store.load(id) + if (current == null || store.bodyFile(current) != file) file.delete() + } + + /** Runs under the store lock. [preStaged] returns the body staged outside the lock for a generation, if any. */ + private fun decideAndCommit(p: EntryParsing.Parsed, preStaged: (generation: Int) -> BodyStaging.Staged?): Enqueued { + val existing = store.load(p.id) + val v9 = if (existing == null) store.legacyManifest(p.id) else null + val s = settings.load() + val now = clock() + val dir = store.entryDir(p.id) + return when (val action = EnqueueRules.decide(existing, v9, p.descriptor) { journal.find(it) != null }) { + EnqueueRules.Action.RejectRunning -> throw QueueException( + QueueException.E_RUNNING, + "entry '${p.id}' is running; a different body is accepted once it stops", + ) + is EnqueueRules.Action.ReEmit -> Enqueued(null, journal.incrementDeliveries(action.eventId)) + EnqueueRules.Action.Resume -> { + val next = EnqueueRules.resumed(existing!!, p, s.paused, s.headerGeneration, now) + saveOrThrow(next) + Enqueued(next, null) + } + EnqueueRules.Action.Create -> { + val staged = preStaged(1) ?: stageOrThrow(p.descriptor, dir, 1) + commit(EnqueueRules.created(p, staged, null, s.paused, s.headerGeneration, now)) + } + is EnqueueRules.Action.AdoptV9 -> { + val incoming = p.descriptor.parts + val parts = incoming?.let { EnqueueRules.adoptedParts(action.manifest, it) } + // The same parts resume over the v9 blob, as a same-body enqueue does. + val keepOwned = incoming != null && ChunkedParts.sameParts(action.manifest.parts, incoming) + val staged = stageOrThrow(p.descriptor.copy(parts = parts), dir, 1, store.blobFile(p.id), keepOwned) + commit(EnqueueRules.created(p, staged, parts, s.paused, s.headerGeneration, now)) + } + EnqueueRules.Action.Replace -> { + val old = existing!! + // Different parts: a present caller file wins; the old blob is the fallback. + val ownedBlob = if (old.body?.kind == StagedBody.CHUNKED) store.bodyFile(old) else null + val staged = preStaged(old.generation + 1) + ?: stageOrThrow(p.descriptor, dir, old.generation + 1, ownedBlob) + commit(EnqueueRules.replaced(old, p, staged, s.paused, s.headerGeneration, now)) + } + } + } + + private fun stageOrThrow( + d: Descriptor, + dir: java.io.File, + generation: Int, + ownedBlob: java.io.File? = null, + keepOwned: Boolean = false, + ): BodyStaging.Staged = + try { + BodyStaging.stage(d, dir, generation, ownedBlob, keepOwned) + } catch (e: QueueException) { + throw e + } catch (e: IOException) { + throw QueueException(QueueException.E_STORAGE, "could not stage the body: ${e.message}") + } + + /** Saves a newly staged entry. On failure the new body file is removed; the old entry stays as it was. */ + private fun commit(next: QueueEntry): Enqueued { + try { + store.save(next) + } catch (e: IOException) { + if (next.body?.kind != StagedBody.CHUNKED) store.bodyFile(next)?.delete() + throw QueueException(QueueException.E_STORAGE, "could not save the entry: ${e.message}") + } + store.pruneUnreferenced(next) + return Enqueued(next, null) + } + + private fun saveOrThrow(next: QueueEntry) { + try { + store.save(next) + } catch (e: IOException) { + throw QueueException(QueueException.E_STORAGE, "could not save the entry: ${e.message}") + } + } + + // MARK: - queue control + + /** Whole-queue pause. Live rows move to paused and their work stops. No outcome. */ + fun pause() { + settings.update { it.copy(paused = true) } + val now = clock() + val paused = transformAll { e -> + if (e.isLive && e.state != EntryState.PAUSED) EntryTransitions.toPaused(e, now) else e + } + paused.forEach { scheduler.cancel(it.id) } + paused.forEach { events.state(it.toRow()) } + } + + fun resume() { + val s = settings.update { it.copy(paused = false) } + val now = clock() + val resumed = transformAll { e -> + if (e.state == EntryState.PAUSED) EntryTransitions.toResumed(e, s.headerGeneration, now) else e + } + resumed.forEach { events.state(it.toRow()) } + // Every queued entry, not only the resumed ones: a run is idempotent. + store.all().forEach { scheduleRun(it) } + } + + /** + * Live: journal a 'cancelled' (user) outcome, then forget after its ack. + * Settled: forget now, row and bytes. Unknown: no-op. Unacked records of a + * forgotten entry are kept, so a handler that has not run yet still runs. + */ + fun cancel(id: String) { + val settled = store.locked { + val e = store.load(id) ?: return@locked null + if (!e.isLive || e.legacy) { + store.remove(id) + return@locked null + } + val now = clock() + val record = cancelledRecord(e, now) + journal.append(record) + val next = EntryTransitions.toSettled(e, EntryState.CANCELLED, record.eventId, e.bytesSent, now) + saveOrThrow(next) + next to record + } + scheduler.cancel(id) + settled?.let { (entry, record) -> + if (record.deliveries > 0) events.settled(record) + events.state(entry.toRow()) + } + } + + private fun cancelledRecord(e: QueueEntry, now: Long) = EventJournal.SettledRecord( + eventId = UUID.randomUUID().toString(), + id = e.id, + key = e.key, + varsJson = e.varsJson, + at = now, + attempts = e.attempts, + requestId = e.lastRequestId, + deliveries = if (events.canDeliver()) 1 else 0, + state = EntryState.CANCELLED.wire, + bytesSent = e.bytesSent, + totalBytes = e.totalBytes, + url = e.descriptor?.reportUrl ?: "", + method = e.descriptor?.method ?: "POST", + partIndex = null, + kind = EventJournal.KIND_CANCELLED, + response = null, + errorKind = null, + message = null, + cancelReason = "user", + generation = e.generation, + ) + + fun setWifiOnly(enabled: Boolean) { + settings.update { it.copy(wifiOnly = enabled) } + } + + fun configureRetry(defaults: RetryDefaults) { + settings.update { it.copy(retry = defaults) } + } + + /** + * Bumps the header generation, merges [patch] into every entry not yet + * forgotten, and requeues the parked ones. Workers compare each entry's + * own headerGeneration, which changes here under the store lock together + * with its headers. So a worker that gets a 401 either sees the patched + * entry and re-issues, or parks first and is requeued here. + */ + fun updateHeaders(patch: Map) { + val s = settings.update { it.copy(headerGeneration = it.headerGeneration + 1) } + val now = clock() + val unparkedIds = mutableSetOf() + val changed = transformAll { e -> + val patched = e.withHeadersPatched(patch, s.headerGeneration) + if (patched.state != EntryState.AWAITING_AUTH) return@transformAll patched + unparkedIds += e.id + EntryTransitions.toUnparked(patched, s.paused, now) + } + changed.filter { it.id in unparkedIds }.forEach { entry -> + scheduleRun(entry) + events.state(entry.toRow()) + } + } + + // MARK: - journal + + /** Every unacknowledged outcome, each counted as one more delivery. */ + fun unacknowledged(): List = + journal.unacknowledged().mapNotNull { journal.incrementDeliveries(it.eventId) } + + /** + * Removes the records. An acked completed or cancelled outcome of the + * entry's current life forgets the entry: row and bytes. An error keeps + * the row until cancel() or a same-id enqueue. Unknown ids are ignored. + * + * A record of the entry's current life on a live entry means the settle + * journaled and emitted, but its store write failed. The record is applied + * first, as the boot sweep would; without that, the sweep finds no record + * after this ack and runs the finished request again. When that save fails + * too, the record stays unacked so the sweep can apply it later. + */ + fun ack(eventIds: List) { + val forgotten = mutableListOf() + val repaired = mutableListOf() + store.locked { + eventIds.forEach { eventId -> + val record = journal.find(eventId) ?: return@forEach + var e = store.load(record.id) + if (e != null && e.isLive && !e.legacy && e.generation == record.generation) { + val next = EntryTransitions.toSettled(e, stateOf(record), record.eventId, record.bytesSent, clock()) + if (!trySave(next)) return@forEach + repaired += next + e = next + } + journal.ack(listOf(eventId)) + if (record.kind == EventJournal.KIND_ERROR || e == null) return@forEach + if (e.generation == record.generation && e.settledEventId == record.eventId) { + store.remove(record.id) + forgotten += record.id + } + } + } + forgotten.forEach { scheduler.cancel(it) } + repaired.filter { it.id !in forgotten }.forEach { events.state(it.toRow()) } + } + + private fun stateOf(record: EventJournal.SettledRecord): EntryState = + EntryState.values().firstOrNull { it.wire == record.state } ?: EntryState.ERROR + + // MARK: - boot sweep + + /** + * Repairs what a process death can leave, then schedules queued work. An + * entry whose worker runs in this process is skipped: that worker finishes + * its own transition. Run at module init, after [LegacyImport]. + * + * 1. A live entry with a journal record of its own generation: the process + * died between the journal append and the store transition. Apply it. + * 2. A running entry with no worker: a process death mid-run. Queue it. + * A live row that disagrees with the queue's paused setting (a death + * partway through pause() or resume()): make it agree. + * 3. A settled entry with records of its generation other than its own: + * orphans from a cancel race. Ack them. + * 4. A completed or cancelled entry whose own record is gone: the ack + * landed but the forget did not. Forget it. + * 5. Schedule every queued entry, and the expiry wake of every parked one. + */ + fun sweep() { + val now = clock() + val changed = mutableListOf() + val toForget = mutableListOf() + val toSchedule = mutableListOf() + val paused = settings.load().paused + store.locked { + val records = journal.unacknowledged().groupBy { it.id } + for (e in store.all()) { + if (e.legacy || isWorkerRunning(e.id)) continue + val own = records[e.id].orEmpty().filter { it.generation == e.generation } + if (e.isLive) { + val latest = own.maxByOrNull { it.at } + if (latest != null) { + val next = EntryTransitions.toSettled(e, stateOf(latest), latest.eventId, latest.bytesSent, now) + if (trySave(next)) { + journal.ack(own.filter { it !== latest }.map { it.eventId }) + changed += next + } + continue + } + var cur = e + if (e.state == EntryState.RUNNING) { + cur = EntryTransitions.toStopped(e, now) + } + // A process death partway through pause() or resume() leaves rows + // that disagree with the queue setting. + if (paused && cur.state != EntryState.PAUSED) { + cur = EntryTransitions.toPaused(cur, now) + } else if (!paused && cur.state == EntryState.PAUSED) { + cur = EntryTransitions.toResumed(cur, settings.load().headerGeneration, now) + } + if (cur !== e) { + if (trySave(cur)) changed += cur else continue + } + if (!paused || cur.state == EntryState.AWAITING_AUTH) toSchedule += cur + } else { + val orphans = own.filter { it.eventId != e.settledEventId } + if (orphans.isNotEmpty()) journal.ack(orphans.map { it.eventId }) + val forgettable = e.state == EntryState.COMPLETED || e.state == EntryState.CANCELLED + if (forgettable && own.none { it.eventId == e.settledEventId }) { + store.remove(e.id) + toForget += e.id + } + } + } + } + toForget.forEach { scheduler.cancel(it) } + toSchedule.forEach { scheduleRun(it, keepWake = true) } + changed.forEach { events.state(it.toRow()) } + } + + private fun trySave(entry: QueueEntry): Boolean = try { + store.save(entry) + true + } catch (e: IOException) { + Diag.error("could not save '${entry.id}'", e) + false + } + + // MARK: - helpers + + /** + * Runs [entry] when it is queued. A backoff longer than a worker waits goes + * to the wake; a parked entry gets a wake at its expiry, so it settles + * 'expired' on time. + */ + private fun scheduleRun(entry: QueueEntry, keepWake: Boolean = false) { + val now = clock() + when (entry.state) { + EntryState.QUEUED -> { + val at = entry.nextAttemptAt + if (at != null && at - now > RetryClassifier.IN_WORKER_BACKOFF_MAX_MS) { + scheduler.scheduleWake(entry, at, replace = !keepWake) + } else { + scheduler.schedule(entry) + } + } + EntryState.AWAITING_AUTH -> scheduler.scheduleWake(entry, entry.expiresAt, replace = !keepWake) + else -> Unit + } + } + + /** Applies [transform] to every non-legacy entry under the store lock. Returns the ones it changed. */ + private fun transformAll(transform: (QueueEntry) -> QueueEntry): List { + val changed = mutableListOf() + store.locked { + store.all().filter { !it.legacy }.forEach { e -> + store.compute(e.id) { cur -> + if (cur == null) null else transform(cur).also { if (it !== cur) changed += it } + } + } + } + return changed + } +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/QueueEntry.kt b/android/src/main/java/ai/openspace/backgroundupload/QueueEntry.kt new file mode 100644 index 00000000..7a5e7f2f --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/QueueEntry.kt @@ -0,0 +1,228 @@ +package ai.openspace.backgroundupload + +import com.google.gson.annotations.SerializedName + +/** The row states. [wire] is the RequestState string JS sees. */ +enum class EntryState(val wire: String) { + @SerializedName("queued") QUEUED("queued"), + @SerializedName("running") RUNNING("running"), + @SerializedName("awaiting-auth") AWAITING_AUTH("awaiting-auth"), + @SerializedName("paused") PAUSED("paused"), + @SerializedName("completed") COMPLETED("completed"), + @SerializedName("error") ERROR("error"), + @SerializedName("cancelled") CANCELLED("cancelled"); + + val isLive get() = this == QUEUED || this == RUNNING || this == AWAITING_AUTH || this == PAUSED +} + +/** One multipart field. [path] is the caller's file, read only at staging. */ +data class FormPart( + val name: String, + val contentType: String, + val string: String?, + val path: String?, + val fileName: String?, +) + +/** A request's `retry`, as a partial override of the configure() defaults. */ +data class RetryOverride( + val baseMs: Long?, + val maxMs: Long?, + val jitter: Double?, + val exempt: List?, +) + +/** What request(vars) returned, as native needs it to run. */ +data class Descriptor( + /** Null only with parts. */ + val url: String?, + val method: String, + /** The merged headers, plus the Content-Type that staging sets. */ + val headers: Map, + /** JSON text of `data`. It is also the bytes of the staged body. */ + val dataJson: String?, + val form: List?, + /** The caller's path, with any file:// prefix removed. */ + val file: String?, + /** The accepted flags live here. */ + val parts: List?, + val accept: List, + val retry: RetryOverride?, + val noNotification: Boolean, +) { + val bodyKind: String + get() = when { + parts != null -> StagedBody.CHUNKED + file != null -> StagedBody.FILE + form != null -> StagedBody.MULTIPART + dataJson != null -> StagedBody.JSON + else -> StagedBody.NONE + } + + /** The url to report for this request: the descriptor's, or the last part's. */ + val reportUrl: String get() = url ?: parts?.lastOrNull()?.url ?: "" +} + +/** + * Where the request body is on disk. [fileName] is relative to the entry + * directory, so a moved app data directory does not break it. + */ +data class StagedBody( + val kind: String, + val fileName: String?, + val boundary: String?, + val totalBytes: Long, +) { + companion object { + const val NONE = "none" + const val JSON = "json" + const val MULTIPART = "multipart" + const val FILE = "file" + const val CHUNKED = "chunked" + } +} + +/** One queue entry. [QueueStore] persists it as `entry.json` with Gson. */ +data class QueueEntry( + val id: String, + val key: String, + /** "null" for null vars. */ + val varsJson: String, + /** Null only for a legacy row. */ + val descriptor: Descriptor?, + /** Null only for a legacy row. */ + val body: StagedBody?, + val state: EntryState, + /** Attempts in this life. Chunked: across every part. */ + val attempts: Int, + val bytesSent: Long, + val totalBytes: Long, + val expiresAt: Long, + val createdAt: Long, + val updatedAt: Long, + /** Set while the entry waits out a backoff: queued for a long one, running for a short one. */ + val nextAttemptAt: Long? = null, + /** The consecutive transient failures before the last release. The next backoff continues from it. */ + val backoffStreak: Int = 0, + /** The settings header generation the headers were last merged at. */ + val headerGeneration: Int = 0, + /** Set while awaiting-auth (and kept under pause): the header generation it parked under. */ + val parkedGeneration: Int? = null, + /** +1 each time a settled entry reopens, and on a different-body replace. Journal records carry it. */ + val generation: Int = 1, + /** The journal record of this life's outcome. */ + val settledEventId: String? = null, + /** The X-Request-Id of the last attempt. */ + val lastRequestId: String? = null, + val legacy: Boolean = false, +) { + val isLive get() = state.isLive + val isSettled get() = !state.isLive + + fun toRow() = RequestRow( + id = id, + key = key, + varsJson = varsJson, + state = state.wire, + bytesSent = bytesSent, + totalBytes = totalBytes, + attempts = attempts, + updatedAt = updatedAt, + nextAttemptAt = nextAttemptAt, + createdAt = createdAt, + ) + + /** + * Whether [incoming] carries the same body. A different body kind, a + * different url or method, or different content is a different body. + * Chunked compares the parts (the path is ignored: the owned blob is the + * truth, as in v9). + */ + fun sameBodyAs(incoming: Descriptor): Boolean { + val stored = descriptor ?: return false + if (stored.bodyKind != incoming.bodyKind) return false + if (stored.method != incoming.method) return false + return when (stored.bodyKind) { + StagedBody.CHUNKED -> ChunkedParts.sameParts(stored.parts!!, incoming.parts!!) + StagedBody.FILE -> stored.url == incoming.url && stored.file == incoming.file + StagedBody.MULTIPART -> stored.url == incoming.url && stored.form == incoming.form + StagedBody.JSON -> stored.url == incoming.url && stored.dataJson == incoming.dataJson + else -> stored.url == incoming.url + } + } + + /** + * updateHeaders(): the patch replaces same-named headers (any case) and adds + * the rest. A part that carries its own copy of a patched header gets the + * new value too, because a stale per-part Authorization would shadow the + * fresh one. + */ + fun withHeadersPatched(patch: Map, generation: Int): QueueEntry { + val d = descriptor ?: return copy(headerGeneration = generation) + return copy( + descriptor = d.copy( + headers = HeaderMap.merge(d.headers, patch), + parts = d.parts?.map { part -> + val shared = patch.filterKeys { name -> HeaderMap.contains(part.headers, name) } + if (shared.isEmpty()) part else part.copy(headers = HeaderMap.merge(part.headers, shared)) + }, + ), + headerGeneration = generation, + ) + } +} + +/** One row of getRequests() and of a state event. */ +class RequestRow( + val id: String, + val key: String, + val varsJson: String, + val state: String, + val bytesSent: Long, + val totalBytes: Long, + val attempts: Int, + val updatedAt: Long, + val nextAttemptAt: Long?, + /** Sort order only; not sent to JS. */ + val createdAt: Long, +) { + /** vars parsed back to an object, once. A malformed text reads as null. */ + val vars: Any? by lazy { runCatching { JsonBridge.parse(varsJson) }.getOrNull() } + + fun withBytes(sent: Long) = RequestRow( + id, key, varsJson, state, sent, totalBytes, attempts, updatedAt, nextAttemptAt, createdAt, + ) + + /** The RequestRow shape. nextAttemptAt only when set. */ + fun toMap(): Map = LinkedHashMap().apply { + put("id", id) + put("key", key) + put("vars", vars) + put("state", state) + put("bytesSent", bytesSent.toDouble()) + put("totalBytes", totalBytes.toDouble()) + put("attempts", attempts.toDouble()) + put("updatedAt", updatedAt.toDouble()) + nextAttemptAt?.let { put("nextAttemptAt", it.toDouble()) } + } +} + +/** Header maps whose names match without regard to case. */ +object HeaderMap { + fun contains(headers: Map, name: String) = + headers.keys.any { it.equals(name, ignoreCase = true) } + + fun get(headers: Map, name: String): String? = + headers.entries.firstOrNull { it.key.equals(name, ignoreCase = true) }?.value + + /** [over] replaces same-named entries of [base] (any case); its spelling is kept. */ + fun merge(base: Map, over: Map): Map { + val out = LinkedHashMap() + base.forEach { (k, v) -> if (!contains(over, k)) out[k] = v } + out.putAll(over) + return out + } + + fun without(headers: Map, name: String): Map = + headers.filterKeys { !it.equals(name, ignoreCase = true) } +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/QueueSettings.kt b/android/src/main/java/ai/openspace/backgroundupload/QueueSettings.kt new file mode 100644 index 00000000..1a364526 --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/QueueSettings.kt @@ -0,0 +1,97 @@ +package ai.openspace.backgroundupload + +import android.content.Context +import com.google.gson.Gson +import java.io.File + +/** The configure() retry defaults. A request's `retry` overrides them field by field. */ +data class RetryDefaults( + val baseMs: Long = 1_000, + val maxMs: Long = 7_200_000, + val jitter: Double = 0.2, + val exempt: List = listOf(404), +) + +/** Queue-wide settings. They live next to the entries, in `settings.json`. */ +data class QueueSettings( + val wifiOnly: Boolean = false, + val paused: Boolean = false, + /** +1 per updateHeaders(). A 401 from an attempt sent under an older value re-issues at once. */ + val headerGeneration: Int = 0, + val retry: RetryDefaults = RetryDefaults(), +) + +/** + * Reads and writes [QueueSettings]. The value is cached after the first read. + * Workers read it before every attempt, so the cache matters. A corrupt file + * reads as the defaults. + */ +class QueueSettingsStore(private val file: File) { + + companion object { + private val gson = Gson() + + @Volatile + private var instance: QueueSettingsStore? = null + + fun get(context: Context): QueueSettingsStore = + instance ?: synchronized(this) { + instance ?: QueueSettingsStore(File(QueueStore.rootDir(context), "settings.json")) + .also { instance = it } + } + + /** configure().retry → defaults. Absent fields keep the library defaults. */ + fun retryDefaults(retry: Map?): RetryDefaults { + val d = RetryDefaults() + if (retry == null) return d + val backoff = retry["backoff"] as? Map<*, *> + val terminal = retry["terminalHttp"] as? Map<*, *> + return RetryDefaults( + baseMs = (backoff?.get("baseMs") as? Number)?.toLong() ?: d.baseMs, + maxMs = (backoff?.get("maxMs") as? Number)?.toLong() ?: d.maxMs, + jitter = (backoff?.get("jitter") as? Number)?.toDouble() ?: d.jitter, + exempt = (terminal?.get("exempt") as? List<*>)?.mapNotNull { (it as? Number)?.toInt() } + ?: d.exempt, + ) + } + } + + private var cached: QueueSettings? = null + + @Synchronized + fun load(): QueueSettings { + cached?.let { return it } + val read = if (file.exists()) { + runCatching { gson.fromJson(file.readText(), QueueSettings::class.java) }.getOrNull() + } else null + return validated(read).also { cached = it } + } + + /** Throws IOException when the write fails. The cache then keeps the old value. */ + @Synchronized + fun update(transform: (QueueSettings) -> QueueSettings): QueueSettings { + val next = transform(load()) + AtomicFiles.writeText(file, gson.toJson(next)) + cached = next + return next + } + + // Gson does not run constructors, so absent fields read as null or 0. + @Suppress("SENSELESS_COMPARISON", "USELESS_ELVIS") + private fun validated(s: QueueSettings?): QueueSettings { + if (s == null) return QueueSettings() + val d = RetryDefaults() + val r = s.retry + return QueueSettings( + wifiOnly = s.wifiOnly, + paused = s.paused, + headerGeneration = s.headerGeneration, + retry = if (r == null) d else RetryDefaults( + baseMs = if (r.baseMs > 0) r.baseMs else d.baseMs, + maxMs = if (r.maxMs > 0) r.maxMs else d.maxMs, + jitter = r.jitter, + exempt = r.exempt ?: d.exempt, + ), + ) + } +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/QueueStore.kt b/android/src/main/java/ai/openspace/backgroundupload/QueueStore.kt new file mode 100644 index 00000000..1beb0870 --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/QueueStore.kt @@ -0,0 +1,215 @@ +package ai.openspace.backgroundupload + +import android.content.Context +import com.google.gson.Gson +import java.io.File +import java.io.IOException +import java.util.Base64 + +/** + * The durable queue: one directory per entry id. This is the v9 chunked + * manifest store, generalized in place. The root directory keeps its v9 name + * (`rnbgupload-chunked`) so v9 blobs and manifests are found without a move. + * + * A v10 directory holds `entry.json` and at most one staged body file (see + * [StagedBody.fileName]). A v9 directory holds `manifest.json` and `blob` + * until a same-id enqueue adopts it. + * + * Every write is tmp + fsync + rename ([AtomicFiles]). Bodies are staged + * before `entry.json` is saved, so an `entry.json` on disk means its body is + * durable. The store keeps [RequestIndex] current on every save and remove. + * It is reachable from a bare Context, because a worker can run in a process + * where React never started. + */ +class QueueStore(private val dir: File, private val index: RequestIndex = RequestIndex()) { + + companion object { + const val ENTRY_FILE = "entry.json" + const val V9_MANIFEST_FILE = "manifest.json" + const val BLOB_FILE = "blob" + + private val gson = Gson() + + @Volatile + private var instance: QueueStore? = null + + fun rootDir(context: Context) = File(context.filesDir, "rnbgupload-chunked") + + /** The process-wide store. The first call loads every row into [RequestIndex.shared]. */ + fun get(context: Context): QueueStore = + instance ?: synchronized(this) { + instance ?: QueueStore(rootDir(context), RequestIndex.shared) + .also { it.loadIndex() } + .also { instance = it } + } + } + + init { + dir.mkdirs() + } + + /** Rebuilds the index from disk. */ + @Synchronized + fun loadIndex() { + index.replaceAll(all().map { it.toRow() }) + } + + // Ids come from the caller and can hold path separators, so the directory + // name is an encoding of the id. The id is read back from the file. + fun entryDir(id: String) = + File(dir, Base64.getUrlEncoder().withoutPadding().encodeToString(id.toByteArray())) + + fun blobFile(id: String) = File(entryDir(id), BLOB_FILE) + + /** The staged body file of [entry], or null for a bodiless request. */ + fun bodyFile(entry: QueueEntry): File? = + entry.body?.fileName?.let { File(entryDir(entry.id), it) } + + private fun entryFile(id: String) = File(entryDir(id), ENTRY_FILE) + + /** Runs [block] under the store lock. For work that spans several store calls. */ + fun locked(block: () -> T): T = synchronized(this) { block() } + + @Synchronized + fun load(id: String): QueueEntry? = read(entryFile(id)) + + /** Throws IOException when the entry did not persist. */ + @Synchronized + fun save(entry: QueueEntry) { + AtomicFiles.writeText(entryFile(entry.id), gson.toJson(entry)) + index.put(entry.toRow()) + } + + /** + * An atomic read-modify-write. The lock spans load, [transform], and save, + * so nothing can write between them and be erased. A result that is the + * same object as the input writes nothing. A null result writes nothing: + * forgetting an entry is always an explicit [remove]. A throwing + * transform or a failed write propagates. + */ + @Synchronized + fun compute(id: String, transform: (QueueEntry?) -> QueueEntry?): QueueEntry? { + val current = load(id) + val next = transform(current) + if (next != null && next !== current) save(next) + return next + } + + /** Best effort, for a worker that can go on from memory: null when the entry is gone or the write failed. */ + @Synchronized + fun update(id: String, transform: (QueueEntry) -> QueueEntry): QueueEntry? = + runCatching { + val current = load(id) ?: return null + val next = transform(current) + if (next !== current) save(next) + next + }.getOrNull() + + /** + * Forgets the entry: row and bytes. `entry.json` goes first, so a partial + * delete never leaves a row that points at missing bytes. + */ + @Synchronized + fun remove(id: String) { + entryFile(id).delete() + entryDir(id).deleteRecursively() + index.remove(id) + } + + /** Every v10 entry. A directory with only a v9 manifest is not a row. */ + @Synchronized + fun all(): List = + (dir.listFiles { f -> f.isDirectory } ?: emptyArray()) + .mapNotNull { d -> File(d, ENTRY_FILE).takeIf { it.exists() }?.let { read(it) } } + + /** The v9 chunked manifest for [id], when the directory has no v10 entry. */ + @Synchronized + fun legacyManifest(id: String): LegacyManifest? { + if (entryFile(id).exists()) return null + val file = File(entryDir(id), V9_MANIFEST_FILE) + if (!file.exists()) return null + val parsed = runCatching { gson.fromJson(file.readText(), LegacyManifest::class.java) }.getOrNull() + return LegacyManifest.validated(parsed) + } + + /** + * Deletes every file in the entry directory that [entry] does not use: + * an older body, a v9 manifest it adopted, tmp files from a crash. + */ + @Synchronized + fun pruneUnreferenced(entry: QueueEntry) { + val keep = setOfNotNull(ENTRY_FILE, entry.body?.fileName) + entryDir(entry.id).listFiles()?.forEach { f -> + if (f.name !in keep) f.deleteRecursively() + } + } + + private fun read(file: File): QueueEntry? { + if (!file.exists()) return null + val parsed = try { + gson.fromJson(file.readText(), QueueEntry::class.java) + } catch (error: Throwable) { + Diag.warn("queue entry unreadable, skipped: ${file.parentFile?.name}", error) + return null + } + return validated(parsed).also { + if (it == null) Diag.warn("queue entry incomplete, skipped: ${file.parentFile?.name}") + } + } + + // Gson does not run constructors. A corrupt file, or one from an older + // build, can hold null in a non-null field. Reject what the engine relies + // on; normalize what has a safe default. + @Suppress("SENSELESS_COMPARISON", "USELESS_ELVIS") + private fun validated(e: QueueEntry?): QueueEntry? { + if (e == null || e.id == null || e.key == null || e.state == null) return null + if (!e.legacy && (e.descriptor == null || e.body == null || e.body.kind == null)) return null + val d = e.descriptor?.let { d -> + if (d.parts != null && d.parts.any { it == null || it.url == null }) return null + d.copy( + method = d.method ?: "POST", + headers = d.headers ?: emptyMap(), + accept = d.accept ?: emptyList(), + parts = d.parts?.map { if (it.headers == null) it.copy(headers = emptyMap()) else it }, + ) + } + return e.copy( + varsJson = e.varsJson ?: "null", + descriptor = d, + generation = if (e.generation <= 0) 1 else e.generation, + ) + } +} + +/** A v9 chunked manifest. The field names are the v9 ones. */ +data class LegacyManifest( + val id: String, + val sourcePath: String, + val parts: List, + val accept: List, + val expiresAt: Long, + val noNotification: Boolean, + val createdAt: Long, +) { + companion object { + @Suppress("SENSELESS_COMPARISON") + fun validated(m: LegacyManifest?): LegacyManifest? { + if (m == null || m.id == null || m.parts == null || m.parts.isEmpty()) return null + if (m.parts.any { it == null || it.url == null }) return null + return m.copy( + parts = m.parts.map { if (it.headers == null) it.copy(headers = emptyMap()) else it }, + accept = m.accept ?: emptyList(), + ) + } + } +} + +/** A rejection that reaches JS as `promise.reject(code, message)`. */ +class QueueException(val code: String, message: String) : IOException(message) { + companion object { + const val E_RUNNING = "E_RUNNING" + const val E_FILE_MISSING = "E_FILE_MISSING" + const val E_STORAGE = "E_STORAGE" + const val E_INVALID = "E_INVALID" + } +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/RequestIndex.kt b/android/src/main/java/ai/openspace/backgroundupload/RequestIndex.kt new file mode 100644 index 00000000..a59708be --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/RequestIndex.kt @@ -0,0 +1,49 @@ +package ai.openspace.backgroundupload + +import java.util.concurrent.ConcurrentHashMap + +/** + * The in-memory rows behind the synchronous getRequests(). [QueueStore] + * keeps it current: it calls [put] after every save and [remove] after every + * remove. Progress ticks move [RequestRow.bytesSent] here only. + */ +class RequestIndex { + companion object { + val shared = RequestIndex() + } + + private val rows = ConcurrentHashMap() + + fun replaceAll(all: Collection) { + rows.clear() + all.forEach { rows[it.id] = it } + } + + /** + * A save of a running entry keeps the larger bytesSent. The stored value + * lags the in-memory progress, and a save for an attempt must not move the + * row backwards. + */ + fun put(row: RequestRow) { + rows.compute(row.id) { _, old -> + if (old != null && old.state == row.state && row.state == EntryState.RUNNING.wire && + old.bytesSent > row.bytesSent && old.bytesSent <= row.totalBytes + ) row.withBytes(old.bytesSent) else row + } + } + + fun remove(id: String) { + rows.remove(id) + } + + fun get(id: String): RequestRow? = rows[id] + + /** Oldest first, then by id. */ + fun snapshot(): List = + rows.values.sortedWith(compareBy { it.createdAt }.thenBy { it.id }) + + /** A progress tick. A missing id is ignored. */ + fun setBytes(id: String, bytesSent: Long) { + rows.computeIfPresent(id) { _, row -> row.withBytes(bytesSent) } + } +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/RetryClassifier.kt b/android/src/main/java/ai/openspace/backgroundupload/RetryClassifier.kt new file mode 100644 index 00000000..cfd01781 --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/RetryClassifier.kt @@ -0,0 +1,83 @@ +package ai.openspace.backgroundupload + +import java.io.IOException +import kotlin.math.min +import kotlin.random.Random + +/** + * The retry table (plan section 6.1), as pure functions. + * + * | Response or failure | Verdict | + * | 2xx, or an accept rule matches | Accepted | + * | 401, 403 | Auth (park) | + * | 408, 429, 5xx | Transient | + * | other 4xx in `exempt` (default [404]) | Transient | + * | other 4xx | Terminal http | + * | any other status (1xx, a final 3xx) | Terminal http | + * | IOException, payload file missing | Terminal file | + * | IOException | Transient | + * | anything else | Terminal unknown | + */ +object RetryClassifier { + + sealed class Verdict { + object Accepted : Verdict() + object Transient : Verdict() + object Auth : Verdict() + data class Terminal(val errorKind: String, val message: String) : Verdict() + } + + data class Policy(val baseMs: Long, val maxMs: Long, val jitter: Double, val exempt: List) + + /** A wait up to this long happens inside the worker. A longer one releases the worker. */ + const val IN_WORKER_BACKOFF_MAX_MS = 30_000L + + fun policy(defaults: RetryDefaults, override: RetryOverride?): Policy = Policy( + baseMs = override?.baseMs ?: defaults.baseMs, + maxMs = override?.maxMs ?: defaults.maxMs, + jitter = override?.jitter ?: defaults.jitter, + exempt = override?.exempt ?: defaults.exempt, + ) + + fun classifyResponse( + code: Int, + body: String?, + accept: List, + exempt: List, + ): Verdict = when { + UploadOutcome.isAccepted(code, body, accept) -> Verdict.Accepted + code == 401 || code == 403 -> Verdict.Auth + code == 408 || code == 429 || code in 500..599 -> Verdict.Transient + code in 400..499 && code in exempt -> Verdict.Transient + else -> Verdict.Terminal("http", "HTTP $code") + } + + /** A CancellationException is never classified; the caller rethrows it first. */ + fun classifyFailure(error: Throwable, fileExists: Boolean): Verdict = when { + error is IOException && !fileExists -> + Verdict.Terminal("file", "request body file is missing: ${error.message ?: error.javaClass.simpleName}") + error is IOException -> Verdict.Transient + else -> Verdict.Terminal("unknown", error.message ?: error.javaClass.simpleName) + } + + /** The live attempt's errorKind for a failure: network, file, or unknown. */ + fun failureKind(error: Throwable, fileExists: Boolean): String = + UploadOutcome.errorKind(error, fileExists) + + fun isExpired(now: Long, expiresAt: Long) = now >= expiresAt + + /** + * base * 2^(streak-1), capped at maxMs, then spread by ± jitter and capped + * again. streak 1 is baseMs. + */ + fun backoffMs(policy: Policy, streak: Int, random: Random = Random.Default): Long { + val exponent = (streak.coerceAtLeast(1) - 1).coerceAtMost(40) + val raw = min(policy.baseMs.toDouble() * Math.pow(2.0, exponent.toDouble()), policy.maxMs.toDouble()) + val jitter = policy.jitter.coerceIn(0.0, 1.0) + val spread = raw * (1.0 + jitter * (2.0 * random.nextDouble() - 1.0)) + return min(spread, policy.maxMs.toDouble()).toLong().coerceAtLeast(0L) + } + + /** The wake time, never later than expiresAt, so an entry expires on time. */ + fun nextAttemptAt(now: Long, backoffMs: Long, expiresAt: Long): Long = min(now + backoffMs, expiresAt) +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/Scheduler.kt b/android/src/main/java/ai/openspace/backgroundupload/Scheduler.kt new file mode 100644 index 00000000..872dd2a3 --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/Scheduler.kt @@ -0,0 +1,111 @@ +package ai.openspace.backgroundupload + +import android.content.Context +import androidx.work.ExistingWorkPolicy +import androidx.work.OneTimeWorkRequest +import androidx.work.OneTimeWorkRequestBuilder +import androidx.work.WorkInfo +import androidx.work.WorkManager +import androidx.work.workDataOf +import java.util.concurrent.TimeUnit + +/** + * Starts and stops worker runs for entries. A run only carries the entry id; + * the worker reads everything else from the store. So an extra run is always + * harmless: it finds nothing to do and exits. + */ +interface WorkScheduler { + /** A run as soon as possible, on the entry's main chain. */ + fun schedule(entry: QueueEntry) + + /** + * A run at [at] under a second unique name. Used for long backoffs and for + * the expiry of a parked entry. [replace] false keeps a wake that already + * exists (the boot sweep). + */ + fun scheduleWake(entry: QueueEntry, at: Long, replace: Boolean) + + /** Cancels the main chain and the wake. */ + fun cancel(id: String) +} + +/** + * WorkManager unique work per entry id, APPEND_OR_REPLACE, as v9. + * + * Why APPEND_OR_REPLACE: a worker settles before doWork returns, so a same-id + * enqueue can arrive while the row is still RUNNING. KEEP would drop it and + * REPLACE would kill the running worker. APPEND runs it after; OR_REPLACE + * starts a fresh chain after a CANCELLED or FAILED one. Workers always return + * success, because WorkManager fails the dependents of a FAILED row without a + * run. + * + * Long waits do not go on the main chain. A delayed row there would hold + * back every later "run now" appended behind it. They use the wake name + * (`#wake`) with an initial delay instead. + * + * No WorkManager Constraints: connectivity and wifi-only are checked inside + * the worker, as v9, because the constraint path was unreliable. + */ +class WorkManagerScheduler(context: Context) : WorkScheduler { + companion object { + /** v9 rows carry the tag "RNFileUploader"; this one is new so the v9 cancel does not touch v10 work. */ + const val WORK_TAG = "RNFileUploader.v10" + const val ID_TAG_PREFIX = "RNFileUploaderId:" + /** A string literal, because WorkManager persists it across builds. */ + const val ENTRY_ID_KEY = "entryId" + const val V9_WORK_TAG = "RNFileUploader" + + fun wakeName(id: String) = "$id#wake" + + /** Milliseconds from [now] until [at]; never negative. */ + fun initialDelayMs(at: Long?, now: Long): Long = if (at == null) 0L else (at - now).coerceAtLeast(0L) + + /** An unfinished row that is not RUNNING: a queued run that did not start yet. */ + fun hasQueuedSuccessor(states: List): Boolean = + states.any { !it.isFinished && it != WorkInfo.State.RUNNING } + } + + private val workManager = WorkManager.getInstance(context) + + override fun schedule(entry: QueueEntry) { + // A queued successor already guarantees a run after the current one. + val states = workManager.getWorkInfosForUniqueWork(entry.id).get().map { it.state } + if (hasQueuedSuccessor(states)) return + workManager + .beginUniqueWork(entry.id, ExistingWorkPolicy.APPEND_OR_REPLACE, request(entry, 0L)) + .enqueue() + } + + override fun scheduleWake(entry: QueueEntry, at: Long, replace: Boolean) { + val delay = initialDelayMs(at, System.currentTimeMillis()) + workManager.enqueueUniqueWork( + wakeName(entry.id), + if (replace) ExistingWorkPolicy.REPLACE else ExistingWorkPolicy.KEEP, + request(entry, delay), + ) + } + + override fun cancel(id: String) { + workManager.cancelUniqueWork(id) + workManager.cancelUniqueWork(wakeName(id)) + } + + /** First v10 launch: the v9 rows. Their workers are gone. */ + fun cancelV9Work() { + workManager.cancelAllWorkByTag(V9_WORK_TAG) + } + + private fun request(entry: QueueEntry, delayMs: Long): OneTimeWorkRequest { + val builder = if (entry.body?.kind == StagedBody.CHUNKED) { + OneTimeWorkRequestBuilder() + } else { + OneTimeWorkRequestBuilder() + } + return builder + .addTag(WORK_TAG) + .addTag(ID_TAG_PREFIX + entry.id) + .setInputData(workDataOf(ENTRY_ID_KEY to entry.id)) + .setInitialDelay(delayMs, TimeUnit.MILLISECONDS) + .build() + } +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/Upload.kt b/android/src/main/java/ai/openspace/backgroundupload/Upload.kt deleted file mode 100644 index 3df469b0..00000000 --- a/android/src/main/java/ai/openspace/backgroundupload/Upload.kt +++ /dev/null @@ -1,107 +0,0 @@ -package ai.openspace.backgroundupload - -import com.facebook.react.bridge.ReadableArray -import com.facebook.react.bridge.ReadableMap -import java.util.UUID - -// Data model of a single upload -// Can be created from RN's ReadableMap -// Can be used for JSON deserialization -data class Upload( - val id: String, - val url: String, - val path: String, - val method: String, - val wifiOnly: Boolean, - // Non-2xx responses to treat as a successful completion (for example, a 409 - // whose body marks an expected duplicate). Every other non-2xx response is a - // terminal http error. The list is empty by default. - val accept: List, - val headers: Map, - /** - * Suppresses the progress notification for this upload. - * - * The notification is not decoration: posting one is what lets the worker run - * in foreground mode, which is how a long-running worker survives Doze and - * memory pressure. A suppressed upload is an ordinary background worker, so - * the OS may defer it or stop it mid-flight for WorkManager to re-run later. - * Suppress only payloads small enough that a restart costs nothing. - * - * An opt-out rather than an opt-in so that absence means "notify": this model - * is serialized into WorkManager's database, and a job enqueued by a build - * that predates the option can be replayed by a build that has it. - */ - val noNotification: Boolean, -) { - // v8 persisted `acceptStatus: List` where v9 persists `accept`. This is - // not a constructor parameter. It exists only so Gson can surface the legacy - // field to [normalized]. It is null, and thus never serialized, for every - // upload that this build creates. - private val acceptStatus: List? = null - - val showsNotification get() = !noNotification - - /** - * Gson does not use the constructor. Thus a WorkManager job that an older - * build enqueued can give this worker an object whose non-null fields are - * null. A v8 job carries `acceptStatus` and no `accept`. That NPEs the first - * time the worker touches [accept], after the file has fully transmitted, - * and the re-runs then re-send the whole file. This is the same - * normalize-after-fromJson pattern as ChunkedManifestStore.validated(): map - * the legacy statuses to rules, default what is absent, and give the worker - * an object that is safe to use. - */ - @Suppress("SENSELESS_COMPARISON", "USELESS_ELVIS") - fun normalized(): Upload = Upload( - id = id, - url = url, - path = path, - method = method ?: "POST", - wifiOnly = wifiOnly, - accept = accept - ?: acceptStatus?.map { UploadOutcome.AcceptRule(it) } - ?: emptyList(), - headers = headers ?: emptyMap(), - noNotification = noNotification, - ) - - class MissingOptionException(optionName: String) : - IllegalArgumentException("Missing '$optionName'") - - companion object { - fun fromReadableMap(map: ReadableMap) = Upload( - id = map.getString(Upload::id.name) ?: UUID.randomUUID().toString(), - url = map.getString(Upload::url.name) ?: throw MissingOptionException(Upload::url.name), - path = map.getString(Upload::path.name) ?: throw MissingOptionException(Upload::path.name), - method = map.getString(Upload::method.name) ?: "POST", - wifiOnly = if (map.hasKey(Upload::wifiOnly.name)) map.getBoolean(Upload::wifiOnly.name) else false, - accept = parseAcceptRules(map.getArray(Upload::accept.name)), - headers = parseHeaderMap(map.getMap(Upload::headers.name)), - // The notification text and identity are not per-upload options. The - // worker reads them from the NotificationConfig that configure() saved. - noNotification = if (map.hasKey(Upload::noNotification.name)) - map.getBoolean(Upload::noNotification.name) else false, - ) - } -} - -// Upload and ChunkedManifest share this: one accept-rules shape, one parser. -internal fun parseAcceptRules(arr: ReadableArray?): List { - if (arr == null) return listOf() - return (0 until arr.size()).mapNotNull { i -> - val rule = arr.getMap(i) ?: return@mapNotNull null - UploadOutcome.AcceptRule( - status = rule.getInt("status"), - bodyIncludes = if (rule.hasKey("bodyIncludes")) rule.getString("bodyIncludes") else null, - ) - } -} - -internal fun parseHeaderMap(headers: ReadableMap?): Map { - if (headers == null) return mapOf() - val map = mutableMapOf() - for (entry in headers.entryIterator) { - map[entry.key] = entry.value.toString() - } - return map -} diff --git a/android/src/main/java/ai/openspace/backgroundupload/UploadOutcome.kt b/android/src/main/java/ai/openspace/backgroundupload/UploadOutcome.kt index f4fb2763..9c22321a 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/UploadOutcome.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/UploadOutcome.kt @@ -11,8 +11,7 @@ object UploadOutcome { * A non-2xx response to treat as success. `bodyIncludes` narrows the rule by * a response-body substring. This is necessary when one status has several * meanings, and only the message shows the difference (our backend's 409). - * Gson persists it inside [Upload] and [ChunkedManifest]; see - * consumer-rules.pro. + * Gson persists it inside [Descriptor]; see consumer-rules.pro. */ data class AcceptRule( val status: Int, diff --git a/android/src/main/java/ai/openspace/backgroundupload/UploadUtils.kt b/android/src/main/java/ai/openspace/backgroundupload/UploadUtils.kt index 0e48da98..6d84b4f0 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/UploadUtils.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/UploadUtils.kt @@ -9,6 +9,7 @@ import okhttp3.OkHttpClient import okhttp3.Request import okhttp3.RequestBody import okhttp3.RequestBody.Companion.asRequestBody +import okhttp3.RequestBody.Companion.toRequestBody import okhttp3.Response import okio.Buffer import okio.BufferedSink @@ -19,7 +20,8 @@ import java.io.IOException import java.io.RandomAccessFile import kotlin.coroutines.resumeWithException -// Throttling interval of progress reports +// Throttling interval of the raw progress callback. ProgressThrottle limits +// the JS events above this. private const val PROGRESS_INTERVAL = 500 // milliseconds private const val RANGE_COPY_BUFFER = 64 * 1024 @@ -30,38 +32,65 @@ data class UploadResponse( val headers: Map ) -// make an upload request using okhttp -suspend fun okhttpUpload( +/** One request as the worker sends it. [body] is null only for GET and DELETE with no body. */ +data class TransferRequest( + val url: String, + val method: String, + val headers: Map, + val body: RequestBody?, +) + +/** Sends one request and reports bytes written. The headers are sent as they are. */ +suspend fun okhttpSend( client: OkHttpClient, - upload: Upload, - file: File, - onProgress: (Long) -> Unit + request: TransferRequest, + onProgress: (Long) -> Unit, ): UploadResponse { - val request = Request.Builder() - .url(upload.url) - .headers(upload.headers.toHeaders()) - .method(upload.method, withProgressListener(file.asRequestBody(), throttled(onProgress))) + val body = request.body?.let { withProgressListener(it, throttled(onProgress)) } + val built = Request.Builder() + .url(request.url) + .headers(request.headers.toHeaders()) + .method(request.method, body) .build() - return awaitResponse(client, request) + return awaitResponse(client, built) } +// Every body has a null content type, so OkHttp does not invent a +// Content-Type. The header on the request (the caller's, or the one staging +// set) is sent unchanged. + +/** A whole staged file. */ +fun fileBody(file: File): RequestBody = file.asRequestBody(null as MediaType?) + +/** A zero-length body for a POST, PUT, or PATCH with no body. OkHttp requires one. */ +fun emptyBody(): RequestBody = ByteArray(0).toRequestBody(null) + /** - * PUTs one byte range of the source file: a chunked part. It streams straight - * from disk, with no temporary chunk file. The headers are the consumer's, - * unchanged. The library adds nothing, per the design's protocol-as-data rule. + * The file bytes [start, end) as a request body: a chunked part. It streams + * from disk with no temporary chunk file. A RandomAccessFile is opened fresh + * on every writeTo, because OkHttp can replay a body (a connection-level + * retry), and a one-shot stream would then send truncated data. */ -suspend fun okhttpUploadPart( - client: OkHttpClient, - part: ChunkedManifest.Part, - file: File, - onProgress: (Long) -> Unit -): UploadResponse { - val request = Request.Builder() - .url(part.url) - .headers(part.headers.toHeaders()) - .put(withProgressListener(rangeRequestBody(file, part.start, part.end), throttled(onProgress))) - .build() - return awaitResponse(client, request) +fun rangeRequestBody(file: File, start: Long, end: Long): RequestBody = object : RequestBody() { + override fun contentType(): MediaType? = null + + override fun contentLength() = end - start + + override fun writeTo(sink: BufferedSink) { + RandomAccessFile(file, "r").use { raf -> + raf.seek(start) + val buffer = ByteArray(RANGE_COPY_BUFFER) + var remaining = end - start + while (remaining > 0L) { + val read = raf.read(buffer, 0, minOf(remaining, buffer.size.toLong()).toInt()) + if (read < 0) throw IOException( + "source file ended before part range [$start, $end): ${file.path}", + ) + sink.write(buffer, 0, read) + remaining -= read + } + } + } } private suspend fun awaitResponse(client: OkHttpClient, request: Request): UploadResponse = @@ -73,18 +102,21 @@ private suspend fun awaitResponse(client: OkHttpClient, request: Request): Uploa continuation.resumeWithException(e) override fun onResponse(call: Call, response: Response) { - val result = response.use { res -> // close the response asap - UploadResponse( - res.code, - // The body, unchanged: an empty body stays empty. A substituted - // HTTP reason phrase would make accept `bodyIncludes` rules match - // text that the server never sent. iOS also reports the body - // as-is. - res.body?.string().orEmpty(), - res.headers.toMultimap().mapValues { it.value.joinToString(", ") } - ) + val result = try { + response.use { res -> // close the response asap + UploadResponse( + res.code, + // The body unchanged: an empty body stays empty. A substituted + // reason phrase would make accept `bodyIncludes` rules match text + // the server never sent. + res.body?.string().orEmpty(), + res.headers.toMultimap().mapValues { it.value.joinToString(", ") } + ) + } + } catch (e: IOException) { + continuation.resumeWithException(e) + return } - continuation.resumeWith(Result.success(result)) } }) @@ -101,38 +133,7 @@ private fun throttled(onProgress: (Long) -> Unit): (Long) -> Unit { } } -/** - * Streams the file bytes [start, end) as a request body. A RandomAccessFile - * backs it, opened fresh on every writeTo call. OkHttp can replay a body (for - * example, after a connection-level retry), and a one-shot stream would then - * send truncated data silently. - */ -private fun rangeRequestBody(file: File, start: Long, end: Long) = object : RequestBody() { - // Null, so no Content-Type is invented. The consumer's header is already on - // the request, unchanged. - override fun contentType(): MediaType? = null - - override fun contentLength() = end - start - - override fun writeTo(sink: BufferedSink) { - RandomAccessFile(file, "r").use { raf -> - raf.seek(start) - val buffer = ByteArray(RANGE_COPY_BUFFER) - var remaining = end - start - while (remaining > 0L) { - val read = raf.read(buffer, 0, minOf(remaining, buffer.size.toLong()).toInt()) - if (read < 0) throw IOException( - "source file ended before part range [$start, $end): ${file.path}", - ) - sink.write(buffer, 0, read) - remaining -= read - } - } - } -} - -// create a request body that allows us to listen to progress. -// okhttp has no built-in way of reporting progress +// OkHttp has no built-in progress report, so the body counts bytes as it writes. private fun withProgressListener( body: RequestBody, onProgress: (Long) -> Unit diff --git a/android/src/main/java/ai/openspace/backgroundupload/UploadWorker.kt b/android/src/main/java/ai/openspace/backgroundupload/UploadWorker.kt index 1f0c63e8..ceb1ee1c 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/UploadWorker.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/UploadWorker.kt @@ -1,286 +1,108 @@ package ai.openspace.backgroundupload -import android.app.NotificationManager import android.content.Context -import androidx.work.CoroutineWorker -import androidx.work.ForegroundInfo import androidx.work.WorkerParameters -import com.google.gson.Gson -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.delay -import kotlinx.coroutines.withContext +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.sync.withPermit +import okhttp3.RequestBody import java.io.File -import java.io.IOException -import java.net.UnknownHostException import java.util.UUID -import java.util.concurrent.TimeUnit -// Retry delay -private val RETRY_DELAY = TimeUnit.SECONDS.toMillis(10L) - -// The retry budget for errors that count (see checkRetry). A connectivity gap -// or flaky-network IO resets the budget. The retry policy is internal to the -// library. It is not an option. -private const val MAX_RETRIES = 5 - -class UploadWorker(private val context: Context, params: WorkerParameters) : - CoroutineWorker(context, params) { - - companion object { - /** - * Key for the serialized [Upload] in the worker's input data. - * - * A string literal on purpose. This key is persisted in WorkManager's - * database, so the build that runs a job may not be the build that enqueued - * it — a key derived from a symbol name (an enum constant, a property) breaks - * the moment R8 renames it or someone refactors, and the failure looks like - * "No Params" on a job that was queued perfectly well by the previous version. - */ - const val PARAMS_KEY = "params" - } - - private lateinit var upload: Upload - // configure() saved this. The worker can read it when WorkManager relaunched - // the worker with no JS. It is lazy, so the SharedPreferences read occurs on - // the worker's IO dispatcher, not at construction. - private val config by lazy { NotificationConfig.load(context) } - private var retries = 0 - private var connectivity = Connectivity.Ok - private val notificationManager = - context.getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager - - override suspend fun doWork(): Result = withContext(Dispatchers.IO) { - // Retrieve the upload. If this throws errors, error reporting won't work. - // However, the only way it has errors is the implementation is incorrect, - // which can be caught in development - val paramsJson = inputData.getString(PARAMS_KEY) ?: throw Throwable("No Params") - // normalized(): an older build can have enqueued this job, and its JSON - // shape can make non-null fields null (Gson does not use the constructor). - // See Upload.normalized. - upload = Gson().fromJson(paramsJson, Upload::class.java).normalized() - - // initialization, errors thrown here won't be retried - try { - // An upload that suppresses its notification cannot enter foreground mode, - // since the notification is the foreground service's own notification. - if (upload.showsNotification) { - // The foreground notification needs a channel to exist first, or posting - // it silently fails and setForeground can crash on newer Android. - ensureNotificationChannel(notificationManager, config) - // `setForeground` is recommended for long-running workers. - // Foreground mode helps prioritize the worker, reducing the risk - // of it being killed during low memory or Doze/App Standby situations. - // ⚠️ This should be called in the foreground - setForeground(getForegroundInfo()) - } - } catch (error: Throwable) { - if (!isForegroundStartDenied(error)) { - if (!checkAndHandleCancellation()) handleError(error) - throw error - } - // The app is in the background on API 31+ (see isForegroundStartDenied). - // Continue the upload without foreground priority. Do not fail an upload - // that can run. +/** The WorkManager class for a single-body entry. The run is [EntryWorker]'s. */ +class UploadWorker(context: Context, params: WorkerParameters) : EntryWorker(context, params) + +/** + * One request with one body: none, JSON, multipart, or a copied file. One + * attempt at a time, until a verdict ends it: + * accepted → Completed; auth → re-issue (newer headers) or park; + * transient → back off (short: here; long: release); terminal → Failed. + */ +internal class SimpleTransfer(private val host: EntryWorker) { + + suspend fun run(start: QueueEntry): Settlement { + val d0 = start.descriptor!! + val file = host.bodyFile(start) + // The payload probe: a staged body that is gone can never be sent. + if (file != null && !file.exists()) { + return Settlement.Failed("file", "the staged request body is missing", null, null, d0.reportUrl, d0.method) } + val total = start.body?.totalBytes ?: 0L + UploadProgress.add(host.entryId, total) + var streak = start.backoffStreak - - // Complex work, errors thrown below here trigger retry. - // We don't let WorkManager manage retries and network constraints as it's very buggy. - // i.e. we'd occasionally get BackgroundServiceStartNotAllowedException, - // or ForegroundServiceStartNotAllowedException, or "isStopped" gets set to "true" - // for no reason - var isRetried = false while (true) { - try { - // - "delay" should be within the "try" block to account for worker cancellation, - // which cancels the delay immediately and throws CancellationException. - // - Linear backoff instead of exponential. One reason for this is we retry on - // invalid connections. Exponential will take too long. - // - We retry only transport failures here (no response). An HTTP - // response, 4xx and 5xx included, is terminal at this layer. - // handleResponse classifies it (a 2xx or an accept rule -> completed, - // else an http error), and the worker returns without a retry. A - // response-code retry policy is the JS queue's job. This matches the - // iOS behavior. - if (isRetried) delay(RETRY_DELAY) - isRetried = true - - val response = upload() ?: continue - handleResponse(response) - return@withContext Result.success() - } catch (error: Throwable) { - if (checkAndHandleCancellation()) throw error - if (checkRetry(error)) continue - handleError(error) + val latest = host.ops.latest(host.entryId, host.generation) + if (RetryClassifier.isExpired(host.now(), latest.expiresAt)) throw EntryWorker.ExpiredException() + host.waitForNetwork() + + val requestId = UUID.randomUUID().toString() + val entry = host.ops.recordAttempt(host.entryId, host.generation, requestId) + // The generation of the headers this attempt sends: both come from the same entry. + val headerGeneration = entry.headerGeneration + val d = entry.descriptor!! + val url = d.url!! + val policy = host.policy(entry) + + val response = try { + transferSemaphore.withPermit { + okhttpSend( + uploadHttpClient, + TransferRequest(url, d.method, host.headersFor(d, null, requestId), requestBody(file, d.method)), + ) { sent -> host.reportProgress(sent, total) } + } + } catch (error: CancellationException) { throw error + } catch (error: Throwable) { + host.reportProgress(0L, total) + val fileExists = file == null || runCatching { file.exists() }.getOrDefault(true) + val message = error.message ?: error.javaClass.simpleName + EventReporter.attempt( + AttemptEvent.ofFailure( + entry, requestId, url, null, RetryClassifier.failureKind(error, fileExists), message, host.now(), + ), + ) + when (val verdict = RetryClassifier.classifyFailure(error, fileExists)) { + is RetryClassifier.Verdict.Terminal -> + return Settlement.Failed(verdict.errorKind, verdict.message, null, null, url, d.method) + else -> { + streak++ + host.backoffOrRelease(policy, streak, entry.expiresAt) + continue + } + } } - } - - // This should never happen. Only here to satisfy the type check - return@withContext Result.failure() - } - - private suspend fun upload(): UploadResponse? { - val file = File(upload.path) - val size = file.length() - - // Register progress asap so the total progress is accurate - // This needs to happen before the semaphore wait - UploadProgress.add(upload.id, size) - - // Don't bother to run on an invalid network - if (!validateAndReportConnectivity()) return null - - // wait for its turn to run - transferSemaphore.acquire() - - try { - return okhttpUpload(uploadHttpClient, upload, file) { progress -> - handleProgress(progress, size) - } - } catch (error: Throwable) { - // reset progress on error - UploadProgress.set(upload.id, 0L) - // pass the error to upper layer for retry decision - throw error - } finally { - transferSemaphore.release() - } - } - - private fun handleProgress(bytesSentTotal: Long, fileSize: Long) { - UploadProgress.set(upload.id, bytesSentTotal) - EventReporter.progress(upload.id, bytesSentTotal, fileSize) - updateNotification() - } - - // Redraws the progress notification. A no-op for a suppressed upload — the - // worker never posted one, and `notify` would create it outside foreground mode. - private fun updateNotification() { - if (!upload.showsNotification) return - notificationManager.notify( - config.systemNotificationId, - buildUploadNotification(context, config, connectivity), - ) - } - - // An HTTP response came back. It is "completed" only for a 2xx or a matching - // accept rule (axios validateStatus semantics: a 400 is an error, not a - // completion). Every other response is a terminal http error that carries the - // full response. In both cases the request finished, so the worker does not - // retry. - private fun handleResponse(response: UploadResponse) { - UploadProgress.complete(upload.id) - val accepted = UploadOutcome.isAccepted(response.code, response.body, upload.accept) - val (body, truncated) = EventJournal.capBody(response.body) - journalAndEmit( - EventJournal.Entry( - eventId = UUID.randomUUID().toString(), - uploadId = upload.id, - type = if (accepted) "completed" else "error", - timestamp = System.currentTimeMillis(), - responseCode = response.code, - responseBody = body, - responseBodyTruncated = truncated, - responseHeaders = response.headers, - errorKind = if (accepted) null else "http", - error = if (accepted) null else "HTTP ${response.code}", - ) - ) - } - - private fun handleError(error: Throwable) { - UploadProgress.remove(upload.id) - // Default fileExists=true so a failed existence probe reads as network, not file. - val fileExists = runCatching { File(upload.path).exists() }.getOrDefault(true) - journalAndEmit( - EventJournal.Entry( - eventId = UUID.randomUUID().toString(), - uploadId = upload.id, - type = "error", - timestamp = System.currentTimeMillis(), - error = error.message ?: "Unknown exception", - errorKind = UploadOutcome.errorKind(error, fileExists), - ) - ) - } - - // Check if cancelled by user or new worker with same ID - // Worker won't rerun, perform teardown - private fun checkAndHandleCancellation(): Boolean { - if (!isStopped) return false - UploadProgress.remove(upload.id) - - // Only a user cancel is terminal, so only a user cancel is journaled. - // - // WorkManager decides whether to reschedule BEFORE it stops the worker, and - // it ignores the Result we return. cancelUniqueWork marks the row CANCELLED - // first, so a user cancel is genuinely the end. A system stop — a - // foreground-service timeout, quota, or memory pressure — leaves the row - // RUNNING and WorkManager re-runs this same upload. Journaling a terminal - // `cancelled` there would durably tell JS the upload was dead while it was in - // fact about to be retried, so the consumer would settle the transfer and the - // retry would land as a duplicate on the server. - // - // Emitting nothing is the honest answer for a system stop: the upload is - // still in flight as far as anyone should be concerned. If WorkManager ever - // declines to reschedule, `getAllUploads()` is how a consumer notices. - if (!UserCancellations.consume(upload.id)) return true - - journalAndEmit( - EventJournal.Entry( - eventId = UUID.randomUUID().toString(), - uploadId = upload.id, - type = "cancelled", - timestamp = System.currentTimeMillis(), - cancelReason = "user", + val verdict = RetryClassifier.classifyResponse(response.code, response.body, d.accept, policy.exempt) + EventReporter.attempt( + AttemptEvent.ofResponse( + entry, requestId, url, null, response, verdict == RetryClassifier.Verdict.Accepted, host.now(), + ), ) - ) - return true - } - - private fun journalAndEmit(entry: EventJournal.Entry) = - EventReporter.journalAndEmit(context, entry) - - /** @return whether to retry */ - private fun checkRetry(error: Throwable): Boolean { - var unlimitedRetry = false - - // Error was thrown due to unmet network preferences. - // Also happens every time you switch from one network to any other - if (!validateAndReportConnectivity()) unlimitedRetry = true - // Due to the flaky nature of networking, sometimes the network is - // valid but the URL is still inaccessible, so keep waiting until - // the URL is accessible - else if (error is UnknownHostException) unlimitedRetry = true - // There are many IOExceptions that only differ by messages, - // so we can't check using class, but theoretically, - // only the one caused by file not existing should stop the retry. - // The rest should be related to flaky network or flaky file I/O, - // where we can retry without limit. - else if (error is IOException) { - try { - if (!File(upload.path).exists()) return false - unlimitedRetry = true - } catch (_: Throwable) { - // read file error, can't do anything but retry - unlimitedRetry = false + when (verdict) { + RetryClassifier.Verdict.Accepted -> return Settlement.Completed(response, url, d.method) + RetryClassifier.Verdict.Auth -> { + // updateHeaders() landed while this attempt was in flight: re-issue now. + if (host.ops.hasNewerHeaders(host.entryId, host.generation, headerGeneration)) { + streak = 0 + continue + } + throw EntryWorker.ParkException(headerGeneration) + } + RetryClassifier.Verdict.Transient -> { + host.reportProgress(0L, total) + streak++ + host.backoffOrRelease(policy, streak, entry.expiresAt) + } + is RetryClassifier.Verdict.Terminal -> + return Settlement.Failed("http", verdict.message, response, null, url, d.method) } } - - retries = if (unlimitedRetry) 0 else retries + 1 - return retries <= MAX_RETRIES } - // Checks connection and alerts connection issues - private fun validateAndReportConnectivity(): Boolean { - this.connectivity = validateConnectivity(context, upload.wifiOnly) - // alert connectivity mode - updateNotification() - return this.connectivity == Connectivity.Ok + // OkHttp needs a body for POST, PUT, and PATCH, and forbids one for GET. + private fun requestBody(file: File?, method: String): RequestBody? = when { + file != null -> fileBody(file) + method == "GET" || method == "DELETE" -> null + else -> emptyBody() } - - override suspend fun getForegroundInfo(): ForegroundInfo = - uploadForegroundInfo(config, buildUploadNotification(context, config, connectivity)) } diff --git a/android/src/main/java/ai/openspace/backgroundupload/UploaderModule.kt b/android/src/main/java/ai/openspace/backgroundupload/UploaderModule.kt index cbd26a85..34f6fd2c 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/UploaderModule.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/UploaderModule.kt @@ -1,11 +1,5 @@ package ai.openspace.backgroundupload -import android.util.Log -import androidx.work.ExistingWorkPolicy -import androidx.work.OneTimeWorkRequestBuilder -import androidx.work.WorkInfo -import androidx.work.WorkManager -import androidx.work.workDataOf import com.facebook.react.bridge.Arguments import com.facebook.react.bridge.Promise import com.facebook.react.bridge.ReactApplicationContext @@ -13,16 +7,21 @@ import com.facebook.react.bridge.ReadableArray import com.facebook.react.bridge.ReadableMap import com.facebook.react.bridge.WritableArray import com.facebook.react.bridge.WritableMap -import com.google.gson.Gson -import java.io.File -import java.nio.file.Files -import java.nio.file.StandardCopyOption - +import com.facebook.react.common.LifecycleState +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.Future +import java.util.concurrent.TimeUnit /** * TurboModule (New Architecture). [NativeRNFileUploaderSpec] is generated by * codegen from `src/NativeRNFileUploader.ts` into this same package (see - * `codegenConfig.android.javaPackageName` in package.json), so it needs no import. + * `codegenConfig.android.javaPackageName` in package.json). + * + * A thin shell over [QueueController]. Every promise method runs on one + * single-thread executor ([queueExecutor]), because enqueue copies files and + * every method touches the disk. The promise resolves from that executor. + * getRequests() is synchronous and reads the in-memory index only. */ class UploaderModule(context: ReactApplicationContext) : NativeRNFileUploaderSpec(context) { @@ -30,359 +29,189 @@ class UploaderModule(context: ReactApplicationContext) : companion object { const val NAME = "RNFileUploader" const val TAG = "RNFileUploader.UploaderModule" - const val WORKER_TAG = "RNFileUploader" - // WorkInfo exposes tags but not the unique-work name, so the upload id is - // also stored as a prefixed tag to recover it from a WorkInfo row. - const val ID_TAG_PREFIX = "RNFileUploaderId:" - // v10 slice 1 ships the JS layer alone. Every queue method rejects with - // this code until slice 2 builds the Android queue and executor. - const val E_NOT_IMPLEMENTED = "E_NOT_IMPLEMENTED" - - // The live module, so EventReporter can reach the codegen emitters — they are - // protected on the generated spec, so only this class may call them. Null - // whenever JS is absent (headless worker, mid-reload); terminal outcomes are - // journaled before being emitted, so a dropped live event is never lost. - // - // Volatile: written on the module-creation thread and read from the - // WorkManager worker, OkHttp callbacks and main, with no other barrier. + + /** The longest getRequests() waits for the first-launch import. */ + private const val IMPORT_WAIT_MS = 2_000L + + // The live module, so EventReporter can reach the codegen emitters: they + // are protected on the generated spec. Null whenever JS is absent + // (headless worker, mid-reload); outcomes are journaled before they are + // emitted, so a dropped live event is never lost. @Volatile var instance: UploaderModule? = null private set + + /** One thread for every module-side disk operation. It outlives a JS reload. */ + val queueExecutor: ExecutorService = Executors.newSingleThreadExecutor { r -> + Thread(r, "RNFileUploader.queue") + } } - private val workManager = WorkManager.getInstance(context) + private val store = QueueStore.get(context) + private val journal = EventJournal.get(context) + private val settings = QueueSettingsStore.get(context) + private val scheduler = WorkManagerScheduler(context) + private val controller = QueueController(store, journal, settings, EventReporter, scheduler) + + /** + * True once JS subscribed to onSettled. JS subscribes, then calls + * getUnacknowledgedEvents() at once, so that first call is the signal. + * Until then a settled emit reaches no listener, so it must not count as + * a delivery. Cleared on teardown. + */ + @Volatile + var listening = false + private set + + // The v9 import, then the v9 work cancel, then the boot sweep. + // getRequests() waits for the import only (file reads and row saves), so + // the first call after an upgrade already shows the legacy rows. The + // cancel opens the WorkManager database, so it runs after that wait. + private val importDone: Future init { instance = this + importDone = queueExecutor.submit { + runCatching { LegacyImport.runOnce(context, store) } + .onFailure { Diag.error("v9 import failed", it) } + .getOrDefault(true) // a failed import still cancels the v9 work + } + queueExecutor.execute { + if (runCatching { importDone.get() }.getOrDefault(false)) { + runCatching { scheduler.cancelV9Work() }.onFailure { Diag.error("v9 work cancel failed", it) } + } + runCatching { controller.sweep() }.onFailure { Diag.error("boot sweep failed", it) } + } } override fun invalidate() { - // A reload constructs the replacement before tearing this one down, so only - // clear the pointer when it still refers to us. + // A reload constructs the replacement before tearing this one down, so + // only clear the pointer when it still refers to us. + listening = false if (instance === this) instance = null super.invalidate() } override fun getName(): String = NAME + /** Picks the progress throttle interval: 1 s in the foreground, 10 min otherwise. */ + fun isForeground(): Boolean = reactApplicationContext.lifecycleState == LifecycleState.RESUMED // MARK: - Event emission (called by EventReporter) - // The v9 workers still report through these. The v10 spec has no per-outcome - // emitters and a different progress shape ({ id, bytesSent, totalBytes }), so - // until slice 2 rewires the workers to onState/onProgress/onSettled, the live - // v9 payloads are dropped here. Terminal outcomes are journaled first, so - // nothing durable is lost. - fun emitProgressEvent(@Suppress("UNUSED_PARAMETER") params: WritableMap) = Unit + fun emitState(params: WritableMap) = safeEmit { emitOnState(params) } - fun emitCompletedEvent(@Suppress("UNUSED_PARAMETER") params: WritableMap) = Unit + fun emitProgress(params: WritableMap) = safeEmit { emitOnProgress(params) } - fun emitErrorEvent(@Suppress("UNUSED_PARAMETER") params: WritableMap) = Unit + fun emitAttempt(params: WritableMap) = safeEmit { emitOnAttempt(params) } - fun emitCancelledEvent(@Suppress("UNUSED_PARAMETER") params: WritableMap) = Unit + fun emitSettled(params: WritableMap) = safeEmit { emitOnSettled(params) } - fun emitNotificationEvent(params: WritableMap) = safeEmit { emitOnNotification(params) } + fun emitNotification(params: WritableMap) = safeEmit { emitOnNotification(params) } private inline fun safeEmit(emit: () -> Unit) { try { emit() } catch (exc: NullPointerException) { - // The generated spec's emitter callback is only installed when the C++ - // TurboModule is constructed, and is gone once the runtime tears down, so a - // null callback is expected in both gaps. It is ALSO null for the whole - // process on the old architecture, where this module still registers and its - // methods work but no event can ever be delivered — hence warn, not debug, - // so that case is diagnosable instead of silent. - Log.w(TAG, "live event dropped (no event emitter — New Architecture required)") - } catch (exc: Throwable) { - // Anything else is a real bridging or payload failure worth seeing. - Log.e(TAG, "failed to emit live event", exc) - } - } - - - /** - * Returns terminal events (completed/error/cancelled) that JS has not yet - * acknowledged, including ones that fired while JS was dead. Read these on - * startup, process them, then call ackEvents to remove them. - */ - override fun getUnacknowledgedEvents(promise: Promise) { - try { - val events = EventJournal.get(reactApplicationContext).unacknowledged() - val arr = Arguments.createArray() - events.forEach { arr.pushMap(it.toWritableMap()) } - promise.resolve(arr) + // The emitter callback exists only while the C++ TurboModule does, so a + // null callback is expected before setup and after teardown. It is also + // null for the whole process on the old architecture, so warn. + Diag.warn("live event dropped (no event emitter; New Architecture required)") } catch (exc: Throwable) { - Log.e(TAG, exc.message, exc) - promise.reject(exc) + Diag.error("failed to emit live event", exc) } } - - /** - * Removes journaled events by eventId once JS has processed them. Resolves - * void. Idempotent: an unknown id is ignored. - */ - override fun ackEvents(ids: ReadableArray, promise: Promise) { - try { - val eventIds = (0 until ids.size()).mapNotNull { ids.getString(it) } - val journal = EventJournal.get(reactApplicationContext) - // An acknowledged 'completed' is the ONE moment when a chunked upload's - // manifest and moved bytes may be deleted. Every other terminal keeps - // them for a resume. Resolve which uploads those are before the entries - // are removed. - val completedUploadIds = journal.unacknowledged() - .filter { it.type == "completed" && eventIds.contains(it.eventId) } - .map { it.uploadId } - journal.ack(eventIds) - releaseAckedCompletions( - completedUploadIds, - ChunkedManifestStore.get(reactApplicationContext), - ) { id -> workManager.cancelUniqueWork(id) } - promise.resolve(null) - } catch (exc: Throwable) { - Log.e(TAG, exc.message, exc) - promise.reject(exc) + // MARK: - Promise methods + + /** Runs [block] on the queue executor and settles [promise] with its value or a coded rejection. */ + private fun onQueue(promise: Promise, block: () -> Any?) { + queueExecutor.execute { + try { + promise.resolve(block()) + } catch (e: QueueException) { + promise.reject(e.code, e.message, e) + } catch (e: EntryParsing.InvalidEntryException) { + promise.reject(QueueException.E_INVALID, e.message, e) + } catch (e: Throwable) { + Diag.error("queue operation failed", e) + promise.reject(QueueException.E_STORAGE, e.message ?: e.javaClass.simpleName, e) + } } } - - /** - * Synchronous. The live rows of the v10 queue. Slice 2 serializes them from - * the in-memory index; until then the queue is empty. - */ - override fun getRequests(): WritableArray = Arguments.createArray() - - /** - * Saves the notification configuration (see [NotificationConfig]). Thus a - * worker that WorkManager relaunches with no JS can read it. Each call - * replaces the full configuration. An omitted field goes back to the library - * default. The v10 `lifetimeMs` and `retry` fields ride along in the same - * map; slice 2 persists them next to the queue. + * Saves the notification configuration (read by headless workers) and the + * retry defaults. Each call replaces the full configuration. lifetimeMs is + * not stored: JS already applied it to expiresAt. */ override fun configure(options: ReadableMap) { NotificationConfig.save(reactApplicationContext, NotificationConfig.fromReadableMap(options)) + @Suppress("UNCHECKED_CAST") + val retry = JsonBridge.valueOf(options, "retry") as? Map + val defaults = QueueSettingsStore.retryDefaults(retry) + queueExecutor.execute { + runCatching { controller.configureRetry(defaults) } + .onFailure { Diag.error("could not save the retry defaults", it) } + } } + override fun enqueue(entry: ReadableMap, promise: Promise) { + // Parse on the calling thread: the ReadableMap belongs to the bridge call. + val parsed = try { + EntryParsing.parse(entry) + } catch (e: EntryParsing.InvalidEntryException) { + promise.reject(QueueException.E_INVALID, e.message, e) + return + } + onQueue(promise) { controller.enqueue(parsed) } + } - // MARK: - v10 queue (stubs until slice 2) - - private fun notImplemented(promise: Promise, method: String) = - promise.reject(E_NOT_IMPLEMENTED, "RNFileUploader.$method: the Android queue is not built yet") - - /** Persists { id, key, vars, descriptor } and schedules it. Slice 2. */ - override fun enqueue(entry: ReadableMap, promise: Promise) = notImplemented(promise, "enqueue") - - override fun pause(promise: Promise) = notImplemented(promise, "pause") - - override fun resume(promise: Promise) = notImplemented(promise, "resume") - - override fun cancel(id: String, promise: Promise) = notImplemented(promise, "cancel") - - override fun setWifiOnly(enabled: Boolean, promise: Promise) = notImplemented(promise, "setWifiOnly") + override fun pause(promise: Promise) = onQueue(promise) { controller.pause(); null } - override fun updateHeaders(patch: ReadableMap, promise: Promise) = notImplemented(promise, "updateHeaders") + override fun resume(promise: Promise) = onQueue(promise) { controller.resume(); null } + override fun cancel(id: String, promise: Promise) = onQueue(promise) { controller.cancel(id); null } - // MARK: - v9 enqueue paths, kept for slice 2 to wire behind enqueue() + override fun setWifiOnly(enabled: Boolean, promise: Promise) = + onQueue(promise) { controller.setWifiOnly(enabled); null } - /** - * @return the id of the enqueued upload - */ - @Suppress("unused") - private fun enqueueUpload(options: ReadableMap): String { - val upload = Upload.fromReadableMap(options) - val data = Gson().toJson(upload) - - // Clear any stale user-cancel mark for this (possibly reused) id - // from a prior life, so a later system stop of this fresh upload isn't - // misreported as a user cancel. Done here (before enqueue), never in the - // worker, so a real cancel arriving as the worker starts can't be erased. - UserCancellations.consume(upload.id) - - val request = OneTimeWorkRequestBuilder() - .addTag(WORKER_TAG) - .addTag(ID_TAG_PREFIX + upload.id) - .setInputData(workDataOf(UploadWorker.PARAMS_KEY to data)) - .build() - - workManager - // Using KEEP policy to prevent it from cancelling the work if it's already running. - // Otherwise, it will emit "cancelled" and then go on to emit "progress" events, - // which is confusing and quite difficult to manage. "cancelled" should be reserved for - // when the user explicitly cancels the upload. - .beginUniqueWork(upload.id, ExistingWorkPolicy.KEEP, request) - .enqueue() - - return upload.id + override fun updateHeaders(patch: ReadableMap, promise: Promise) { + val headers = try { + EntryParsing.headerPatch(patch) + } catch (e: EntryParsing.InvalidEntryException) { + promise.reject(QueueException.E_INVALID, e.message, e) + return + } + onQueue(promise) { controller.updateHeaders(headers); null } } + /** Synchronous. From the in-memory index, never from disk. */ + override fun getRequests(): WritableArray { + runCatching { importDone.get(IMPORT_WAIT_MS, TimeUnit.MILLISECONDS) } + val out = Arguments.createArray() + RequestIndex.shared.snapshot().forEach { out.pushMap(JsonBridge.toWritableMap(it.toMap())) } + return out + } /** - * Starts, or resumes, a chunked upload. It is idempotent against the durable - * [ChunkedManifest]. A first call takes ownership of the source file (an - * O(1) rename into the library's directory) and persists the manifest. A - * re-call with the same id reconciles instead: identical parts are required, - * the stored headers are replaced, and the accepted parts are skipped. Crash - * recovery, a resume after a stop, and a resume with fresh auth are all this - * same call. + * Every unacknowledged outcome, each counted as one more delivery. Sets + * [listening] first: an outcome settled from here on is emitted live with + * deliveries 1; one settled before it was journaled with 0, and this + * drain makes it 1. */ - @Suppress("unused") - private fun enqueueChunkedUpload(options: ReadableMap): String { - val store = ChunkedManifestStore.get(reactApplicationContext) - val id = options.getString("id") - ?: throw Upload.MissingOptionException("id") - val blob = store.blobFile(id) - val incoming = ChunkedManifest.fromReadableMap( - options, - sourcePath = blob.absolutePath, - createdAt = System.currentTimeMillis(), - ) - - // One atomic store operation, persisted BEFORE the work is enqueued. The - // manifest is what a worker relaunched with no JS runs from. The store - // lock spans load, reconcile, and save. Thus a running worker's - // markAccepted can never land between them and be erased. The running flag - // inside the lock is race-free too. A worker acquires ChunkedWorkerGate - // before its first manifest read. Thus it either registers first (and the - // recreate is rejected), or it reads the manifest that this call saved. - store.compute(id) { existing -> - if (existing == null) { - val path = options.getString("path") ?: throw Upload.MissingOptionException("path") - takeOwnership(File(path), blob) - incoming - } else { - // `path` is deliberately ignored here. When a manifest exists, the - // owned bytes are the source of truth. - existing.reconcile( - incoming, - running = ChunkedWorkerGate.isRunning(id), - blobSize = File(existing.sourcePath).length(), - ) - } - } - - // The stale-mark reasoning is the same as in enqueueUpload. - UserCancellations.consume(id) - - // A queued successor (an unfinished row that is not RUNNING) already - // guarantees a run after the current one finishes. An appended second run - // would only stack duplicate no-op runs. The manifest reconcile above - // still landed. That is how this call's fresh headers reach the queued - // run. - val states = workManager.getWorkInfosForUniqueWork(id).get().map { it.state } - if (hasQueuedSuccessor(states)) return id - - val request = OneTimeWorkRequestBuilder() - .addTag(WORKER_TAG) - .addTag(ID_TAG_PREFIX + id) - .setInputData(workDataOf(ChunkedUploadWorker.ID_KEY to id)) - .build() - - // APPEND_OR_REPLACE, not KEEP. A worker journals its terminal error before - // doWork returns. Thus a consumer that resumes from the error handler can - // arrive while that run's row is still RUNNING. KEEP would silently drop - // the resume, and nothing would ever run it. An append keeps the runs - // strictly sequential, and a trailing run over an already-settled manifest - // is a clean no-op (see ChunkedEngine.startAction). Appended work is a - // chain DEPENDENT: WorkManager marks the dependents of a failed - // prerequisite FAILED without a run. That is why ChunkedUploadWorker - // always returns Result.success(), even after it journals a terminal error - // (see terminalErrorResult). The OR_REPLACE half only rescues enqueues - // that arrive AFTER the chain already settled failed or cancelled: it - // starts a fresh sequence. A re-call while the worker runs still never - // restarts it. The running worker re-reads the stored manifest before - // every part attempt, so a resume's fresh headers reach it. - workManager - .beginUniqueWork(id, ExistingWorkPolicy.APPEND_OR_REPLACE, request) - .enqueue() - - return id - } - - @Suppress("unused") - private fun takeOwnership(source: File, blob: File) { - if (!source.exists()) { - // A crash between the rename and the manifest save leaves the bytes at - // the blob path with no manifest. Adopt them. Do not fail the retry. - if (blob.exists()) return - throw IllegalArgumentException("chunked source file does not exist: ${source.path}") + override fun getUnacknowledgedEvents(promise: Promise) { + listening = true + onQueue(promise) { + val out = Arguments.createArray() + controller.unacknowledged().forEach { out.pushMap(it.toWritableMap()) } + out } - blob.parentFile?.mkdirs() - if (blob.exists()) blob.delete() - if (source.renameTo(blob)) return - // renameTo cannot cross filesystems. Files.move falls back to copy+delete. - Files.move(source.toPath(), blob.toPath(), StandardCopyOption.REPLACE_EXISTING) } -} -/** - * Releases the uploads whose 'completed' events were just acknowledged. That - * is the ONE moment when a chunked upload's manifest and moved bytes may be - * deleted. The allAccepted guard protects a recreate: the id may have been - * RECREATED (a different parts array under the same id) and run again over - * these bytes. An ack of the old life's completion must not cancel that work, - * and it must not delete the blob under it. Simple uploads have no manifest - * and fall through untouched. A cancel of their unique work could kill an - * unrelated new upload that reuses the id. - */ -internal fun releaseAckedCompletions( - uploadIds: List, - store: ChunkedManifestStore, - cancelWork: (String) -> Unit, -) { - uploadIds.forEach { id -> - val manifest = store.load(id) ?: return@forEach - if (!manifest.allAccepted) return@forEach - // Cancel a still-enqueued trailing run BEFORE the delete. A worker that - // starts after the delete finds nothing. It exits silently, but there is - // no reason to run it at all. - cancelWork(id) - store.remove(id) + /** Resolves void. Idempotent; unknown ids are ignored. */ + override fun ackEvents(ids: ReadableArray, promise: Promise) { + val eventIds = (0 until ids.size()).mapNotNull { runCatching { ids.getString(it) }.getOrNull() } + onQueue(promise) { controller.ack(eventIds); null } } } - -/** - * Whether cancelUpload must journal and emit the 'cancelled' event itself. - * That is the case only when NO row is RUNNING. A never-started row (ENQUEUED, - * or BLOCKED as an appended chain's dependent) has no worker to run a stop - * handler. A RUNNING worker's stop handler owns the report, including a worker - * that still waits on the ChunkedWorkerGate. - */ -internal fun cancelReportsFromModule(unfinishedStates: List): Boolean = - unfinishedStates.isNotEmpty() && unfinishedStates.none { it == WorkInfo.State.RUNNING } - -/** An unfinished row that is not RUNNING: a queued run that did not start yet. */ -internal fun hasQueuedSuccessor(states: List): Boolean = - states.any { !it.isFinished && it != WorkInfo.State.RUNNING } - -/** - * One state for a chunked upload id, from all its WorkInfo rows plus the - * durable manifest. A live row wins. With no live row, the manifest speaks. - * The state is never "cancelled". iOS getAllUploads has no lingering cancelled - * rows (a cancelled task leaves the session). And on Android, a cancelled - * chunked upload keeps its manifest. Its truthful state is - * stalled-awaiting-resume, that is, "error". - */ -internal fun chunkedUploadState(states: List, allAccepted: Boolean): String = - when { - WorkInfo.State.RUNNING in states -> "running" - states.any { !it.isFinished } -> "pending" - allAccepted -> "completed" - else -> "error" - } - -/** - * One state for a simple upload id, from all its WorkInfo rows. An id can have - * a lingering finished chain next to a live one. A live row wins. Otherwise - * the most conclusive finished state wins. - */ -internal fun simpleUploadState(states: List): String = - when { - WorkInfo.State.RUNNING in states -> "running" - states.any { !it.isFinished } -> "pending" - WorkInfo.State.SUCCEEDED in states -> "completed" - WorkInfo.State.FAILED in states -> "error" - else -> "cancelled" - } diff --git a/android/src/main/java/ai/openspace/backgroundupload/UserCancellations.kt b/android/src/main/java/ai/openspace/backgroundupload/UserCancellations.kt deleted file mode 100644 index 5b19cea6..00000000 --- a/android/src/main/java/ai/openspace/backgroundupload/UserCancellations.kt +++ /dev/null @@ -1,17 +0,0 @@ -package ai.openspace.backgroundupload - -// Upload ids the JS side explicitly cancelled. Consulted by the worker to -// distinguish user cancels from system kills (WorkManager 2.8.1 has no -// getStopReason). Same-process only: a user cancel always originates from live -// JS, so the set never needs to persist across process death. -object UserCancellations { - private val ids = mutableSetOf() - - @Synchronized - fun mark(id: String) { - ids.add(id) - } - - @Synchronized - fun consume(id: String): Boolean = ids.remove(id) -} diff --git a/android/src/main/java/ai/openspace/backgroundupload/WorkerGate.kt b/android/src/main/java/ai/openspace/backgroundupload/WorkerGate.kt new file mode 100644 index 00000000..d1313aeb --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/WorkerGate.kt @@ -0,0 +1,32 @@ +package ai.openspace.backgroundupload + +import java.util.concurrent.ConcurrentHashMap + +/** + * At most one worker EXECUTES per entry id, process-wide (renamed from the + * v9 ChunkedWorkerGate; both workers use it now). + * + * The unique-work chain almost guarantees this, but not across a cancel: + * cancelUniqueWork marks the row CANCELLED at once while the old worker's + * coroutine still winds down, and the wake-up work runs under a second + * unique name. Two concurrent requests for one chunked part are unsafe on + * the server. A starting worker acquires its id here and a second one waits. + * + * Same-process only. A worker in a dead process holds nothing. + */ +object WorkerGate { + private val holders = ConcurrentHashMap() + + /** True when [token] now holds the id, or already held it. False while another token holds it. */ + fun tryAcquire(id: String, token: Any): Boolean { + val current = holders.putIfAbsent(id, token) + return current == null || current === token + } + + /** Releases only when [token] is the holder, so a late release can not evict a successor. */ + fun release(id: String, token: Any) { + holders.remove(id, token) + } + + fun isRunning(id: String): Boolean = holders.containsKey(id) +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/WorkerOps.kt b/android/src/main/java/ai/openspace/backgroundupload/WorkerOps.kt new file mode 100644 index 00000000..4f1fe2ef --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/WorkerOps.kt @@ -0,0 +1,271 @@ +package ai.openspace.backgroundupload + +import java.io.IOException +import java.util.UUID + +/** How a run ended. */ +sealed class Settlement { + abstract val url: String + abstract val method: String + + /** [response] is null for a chunked completion. */ + data class Completed( + val response: UploadResponse?, + override val url: String, + override val method: String, + ) : Settlement() + + data class Failed( + val errorKind: String, + val message: String, + val response: UploadResponse?, + val partIndex: Int?, + override val url: String, + override val method: String, + ) : Settlement() +} + +/** + * The entry was paused, cancelled, replaced, or forgotten under a running + * worker. The worker stops without a transition; the module owns what + * happened. Not a CancellationException: it must fail a chunked part's + * scope so the sibling parts stop too. + */ +class NotOwnedException(id: String) : Exception("entry '$id' is no longer owned by this run") + +/** + * Every transition the network causes: running, one attempt, part accepted, + * awaiting-auth, queued-with-backoff, a system stop, and the settle. Each is + * a `compute` guarded by the run's [generation], so a cancel, pause, or + * replace that landed first always wins. + */ +class WorkerOps( + private val store: QueueStore, + private val journal: EventJournal, + private val settings: QueueSettingsStore, + private val events: QueueEvents, + private val scheduler: WorkScheduler, + private val clock: () -> Long = System::currentTimeMillis, +) { + enum class ParkResult { PARKED, REISSUE, NOT_OWNED } + + /** Takes a queued entry. Null when there is nothing to run. */ + fun begin(id: String): QueueEntry? { + val now = clock() + var changed = false + val entry = store.compute(id) { e -> + if (e != null && !e.legacy && (e.state == EntryState.QUEUED || e.state == EntryState.RUNNING)) { + changed = true + EntryTransitions.toRunning(e, now) + } else e + } + if (entry == null || entry.state != EntryState.RUNNING) return null + if (changed) events.state(entry.toRow()) + return entry + } + + /** The stored entry, while this run still owns it. */ + fun latest(id: String, generation: Int): QueueEntry { + val e = store.load(id) + if (!EntryTransitions.isOwnedRun(e, generation)) throw NotOwnedException(id) + return e!! + } + + /** + * Write-ahead for one attempt: attempts + 1 and the X-Request-Id, persisted + * before the request is sent. Clears a short backoff's nextAttemptAt, and + * emits the row when it did. Returns the fresh entry, whose headers the + * attempt uses. + */ + fun recordAttempt(id: String, generation: Int, requestId: String): QueueEntry { + val now = clock() + var clearedBackoff = false + val next = store.compute(id) { e -> + if (EntryTransitions.isOwnedRun(e, generation)) { + clearedBackoff = e!!.nextAttemptAt != null + EntryTransitions.toAttempt(e, requestId, now) + } else e + } + if (next == null || next.lastRequestId != requestId || !EntryTransitions.isOwnedRun(next, generation)) { + throw NotOwnedException(id) + } + if (clearedBackoff) events.state(next.toRow()) + return next + } + + /** + * A short backoff the worker waits out in place: the row stays running + * and carries [nextAttemptAt]. Best effort; a lost write only hides the + * time. For a chunked entry, a sibling part's next attempt clears it. + */ + fun backingOff(id: String, generation: Int, nextAttemptAt: Long) { + val now = clock() + var applied = false + val next = store.update(id) { e -> + if (EntryTransitions.isOwnedRun(e, generation)) { + applied = true + EntryTransitions.toBackingOff(e, nextAttemptAt, now) + } else e + } + if (applied && next != null) events.state(next.toRow()) + } + + /** A chunked part the server accepted. Best effort, as v9: a lost flag only re-sends that part later. */ + fun markAccepted(id: String, generation: Int, index: Int): QueueEntry? = + store.update(id) { e -> + val parts = e.descriptor?.parts + if (e.generation != generation || parts == null || index !in parts.indices) e + else { + val next = ChunkedParts.withAccepted(parts, index) + e.copy( + descriptor = e.descriptor.copy(parts = next), + bytesSent = ChunkedParts.acceptedBytes(next), + backoffStreak = 0, + ) + } + } + + /** + * Journal, then transition, then emit. When a cancel or a replace landed + * first, the record is an orphan: it is acked at once and nothing is + * emitted. Returns whether this run's outcome stands. + */ + fun settle(id: String, generation: Int, s: Settlement): Boolean { + val e = store.load(id) + if (!EntryTransitions.canSettle(e, generation)) return false + e!! + val now = clock() + val completed = s is Settlement.Completed + val state = if (completed) EntryState.COMPLETED else EntryState.ERROR + val bytesSent = if (completed) e.totalBytes else e.bytesSent + val failed = s as? Settlement.Failed + val record = EventJournal.SettledRecord( + eventId = UUID.randomUUID().toString(), + id = e.id, + key = e.key, + varsJson = e.varsJson, + at = now, + attempts = e.attempts, + requestId = e.lastRequestId, + deliveries = if (events.canDeliver()) 1 else 0, + state = state.wire, + bytesSent = bytesSent, + totalBytes = e.totalBytes, + url = s.url, + method = s.method, + partIndex = failed?.partIndex, + kind = if (completed) EventJournal.KIND_COMPLETED else EventJournal.KIND_ERROR, + response = when (s) { + is Settlement.Completed -> s.response?.let { EventJournal.Response.of(it) } ?: EventJournal.Response.NONE + is Settlement.Failed -> s.response?.let { EventJournal.Response.of(it) } + }, + errorKind = failed?.errorKind, + message = failed?.message, + cancelReason = null, + generation = generation, + ) + // 1. The durable outcome. It never throws. + journal.append(record) + // JS can subscribe between the check above and the append, and its first + // drain can miss this record. Count it as a live delivery then. + val live = if (record.deliveries == 0 && events.canDeliver()) { + journal.incrementDeliveries(record.eventId) ?: record + } else record + // 2. The transition, atomic against cancel(). + var applied = false + val next = try { + store.compute(id) { cur -> + if (EntryTransitions.canSettle(cur, generation)) { + applied = true + EntryTransitions.toSettled(cur!!, state, record.eventId, bytesSent, now) + } else cur + } + } catch (error: IOException) { + // The record is durable; the boot sweep applies it to the entry. + Diag.error("settle could not save '$id'; its ack or the boot sweep repairs it", error) + if (live.deliveries > 0) events.settled(live) + return true + } + if (!applied || next == null) { + journal.ack(listOf(record.eventId)) + return false + } + // 3 and 4. Best effort. + if (live.deliveries > 0) events.settled(live) + events.state(next.toRow()) + return true + } + + /** + * Whether the stored entry holds newer headers than the ones an attempt + * sent. [headerGeneration] is the entry's own value from [recordAttempt], + * so it always belongs to the headers that went out. The settings value + * is not used: updateHeaders() bumps it before it patches the entries. + */ + fun hasNewerHeaders(id: String, generation: Int, headerGeneration: Int): Boolean = + latest(id, generation).headerGeneration > headerGeneration + + /** + * A 401/403. [headerGeneration] is the entry's value from [recordAttempt]. + * When the entry got newer headers since, the attempt re-issues at once + * instead of parking. The check is inside the store lock, and + * updateHeaders() patches entries inside it too, so it either patched + * this entry first (REISSUE) or finds it parked and requeues it. + */ + fun park(id: String, generation: Int, headerGeneration: Int): ParkResult { + val now = clock() + var result = ParkResult.NOT_OWNED + val next = store.compute(id) { e -> + when { + !EntryTransitions.isOwnedRun(e, generation) -> e + e!!.headerGeneration > headerGeneration -> { + result = ParkResult.REISSUE + e + } + else -> { + result = ParkResult.PARKED + EntryTransitions.toParked(e, headerGeneration, now) + } + } + } + if (result == ParkResult.PARKED && next != null) { + events.state(next.toRow()) + // A parked entry still expires on time. + scheduler.scheduleWake(next, next.expiresAt, replace = true) + } + return result + } + + /** A backoff longer than a worker waits: back to queued, woken at [nextAttemptAt]. */ + fun release(id: String, generation: Int, nextAttemptAt: Long, streak: Int): Boolean { + val now = clock() + var applied = false + val next = store.compute(id) { e -> + if (EntryTransitions.isOwnedRun(e, generation)) { + applied = true + EntryTransitions.toReleased(e!!, nextAttemptAt, streak, now) + } else e + } + if (!applied || next == null) return false + events.state(next.toRow()) + scheduler.scheduleWake(next, nextAttemptAt, replace = true) + return true + } + + /** A system stop. A paused or cancelled entry is the module's, so only a running one moves. Never journals. */ + fun stopped(id: String, generation: Int) { + val now = clock() + var applied = false + val next = runCatching { + store.compute(id) { e -> + if (EntryTransitions.isOwnedRun(e, generation)) { + applied = true + EntryTransitions.toStopped(e!!, now) + } else e + } + }.getOrNull() + if (applied && next != null) events.state(next.toRow()) + } + + fun settings(): QueueSettings = settings.load() +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/AckReleaseTest.kt b/android/src/test/java/ai/openspace/backgroundupload/AckReleaseTest.kt deleted file mode 100644 index 0afe1e88..00000000 --- a/android/src/test/java/ai/openspace/backgroundupload/AckReleaseTest.kt +++ /dev/null @@ -1,69 +0,0 @@ -package ai.openspace.backgroundupload - -import org.junit.Assert.assertEquals -import org.junit.Assert.assertNotNull -import org.junit.Assert.assertNull -import org.junit.Assert.assertTrue -import org.junit.Rule -import org.junit.Test -import org.junit.rules.TemporaryFolder - -// An acknowledged 'completed' is the one moment when a chunked upload's stored -// state may be released. But only the completed life's state may go. A recreate -// under the same id can run over the same bytes, and it must survive the old -// life's ack. -class AckReleaseTest { - @get:Rule - val tmp = TemporaryFolder() - - private fun manifest(id: String, accepted: Boolean) = ChunkedManifest( - id = id, - sourcePath = "/data/blob", - parts = listOf( - ChunkedManifest.Part( - url = "https://example.com/1", - headers = emptyMap(), - start = 0, - end = 100, - accepted = accepted, - ), - ), - accept = emptyList(), - expiresAt = 5_000, - wifiOnly = false, - noNotification = false, - createdAt = 1_000, - ) - - @Test - fun `releases a completed upload's manifest and cancels its trailing runs`() { - val store = ChunkedManifestStore(tmp.newFolder()) - store.save(manifest("u1", accepted = true)) - val cancelled = mutableListOf() - releaseAckedCompletions(listOf("u1"), store) { cancelled.add(it) } - assertNull(store.load("u1")) - assertEquals(listOf("u1"), cancelled) - } - - @Test - fun `spares a recreate running under the same id`() { - // The acknowledged completion belongs to the id's PREVIOUS life. The - // manifest now holds a recreate's unaccepted parts, and a worker can be - // mid-transfer. A work cancel or a blob delete here would destroy its - // bytes. - val store = ChunkedManifestStore(tmp.newFolder()) - store.save(manifest("u1", accepted = false)) - val cancelled = mutableListOf() - releaseAckedCompletions(listOf("u1"), store) { cancelled.add(it) } - assertNotNull(store.load("u1")) - assertTrue(cancelled.isEmpty()) - } - - @Test - fun `ignores ids with no manifest (simple uploads)`() { - val store = ChunkedManifestStore(tmp.newFolder()) - val cancelled = mutableListOf() - releaseAckedCompletions(listOf("raw-upload"), store) { cancelled.add(it) } - assertTrue(cancelled.isEmpty()) - } -} diff --git a/android/src/test/java/ai/openspace/backgroundupload/BodyStagingTest.kt b/android/src/test/java/ai/openspace/backgroundupload/BodyStagingTest.kt new file mode 100644 index 00000000..216c69a6 --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/BodyStagingTest.kt @@ -0,0 +1,207 @@ +package ai.openspace.backgroundupload + +import org.junit.Assert.assertArrayEquals +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertThrows +import org.junit.Assert.assertTrue +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import java.io.File + +class BodyStagingTest { + @get:Rule + val tmp = TemporaryFolder() + + private fun source(name: String, bytes: ByteArray) = File(tmp.newFolder(), name).apply { writeBytes(bytes) } + + @Test + fun `json bytes are the data text, with a default content type`() { + val dir = tmp.newFolder() + val staged = BodyStaging.stage(desc(dataJson = """{"n":1,"t":"é"}""", headers = mapOf()), dir, 1) + assertEquals(StagedBody.JSON, staged.body.kind) + assertEquals("body-1.json", staged.body.fileName) + assertEquals("""{"n":1,"t":"é"}""", File(dir, "body-1.json").readText()) + assertEquals(File(dir, "body-1.json").length(), staged.body.totalBytes) + assertEquals(mapOf("Content-Type" to "application/json"), staged.headers) + } + + @Test + fun `a caller content type wins for json, in any case`() { + val staged = BodyStaging.stage(desc(dataJson = "{}", headers = mapOf("content-type" to "application/vnd+json")), tmp.newFolder(), 1) + assertEquals(mapOf("content-type" to "application/vnd+json"), staged.headers) + } + + @Test + fun `multipart bytes follow RFC 7578`() { + val photo = source("photo.jpg", byteArrayOf(1, 2, 3)) + val form = listOf( + FormPart("meta", "application/json", "{\"a\":\"b\"}", null, null), + FormPart("pho\"to", "image/jpeg", null, photo.path, null), + FormPart("named", "image/jpeg", null, photo.path, "new\nname.jpg"), + ) + val target = File(tmp.newFolder(), "body.multipart") + BodyStaging.writeMultipart(form, "BOUND", target) + val expected = ("--BOUND\r\n" + + "Content-Disposition: form-data; name=\"meta\"\r\n" + + "Content-Type: application/json\r\n\r\n" + + "{\"a\":\"b\"}\r\n" + + "--BOUND\r\n" + + "Content-Disposition: form-data; name=\"pho%22to\"; filename=\"photo.jpg\"\r\n" + + "Content-Type: image/jpeg\r\n\r\n").toByteArray() + byteArrayOf(1, 2, 3) + ("\r\n" + + "--BOUND\r\n" + + "Content-Disposition: form-data; name=\"named\"; filename=\"new%0Aname.jpg\"\r\n" + + "Content-Type: image/jpeg\r\n\r\n").toByteArray() + byteArrayOf(1, 2, 3) + ("\r\n" + + "--BOUND--\r\n").toByteArray() + assertArrayEquals(expected, target.readBytes()) + } + + @Test + fun `form staging always sets the library content type with its boundary`() { + val dir = tmp.newFolder() + val staged = BodyStaging.stage( + desc(form = listOf(FormPart("a", "text/plain", "x", null, null)), headers = mapOf("CONTENT-TYPE" to "text/plain")), + dir, 2, + ) + val boundary = staged.body.boundary!! + assertTrue(boundary.startsWith("----RNBGU")) + assertEquals(mapOf("Content-Type" to "multipart/form-data; boundary=$boundary"), staged.headers) + assertTrue(File(dir, "body-2.multipart").readText().startsWith("--$boundary\r\n")) + } + + @Test + fun `a file body is copied and the source stays`() { + val src = source("a.bin", ByteArray(1000) { it.toByte() }) + val dir = tmp.newFolder() + val staged = BodyStaging.stage(desc(file = src.path), dir, 3) + assertTrue(src.exists()) + assertArrayEquals(src.readBytes(), File(dir, "file-3").readBytes()) + assertEquals(1000, staged.body.totalBytes) + assertEquals(mapOf("Authorization" to "Bearer old"), staged.headers) // no content type added + src.delete() + assertTrue(File(dir, "file-3").exists()) + } + + @Test + fun `a chunked file is moved and must tile the parts`() { + val src = source("video.bin", ByteArray(20)) + val dir = tmp.newFolder() + val staged = BodyStaging.stage(desc(url = null, file = src.path, parts = listOf(part(0, 10), part(10, 20))), dir, 1) + assertFalse(src.exists()) + assertEquals(20, File(dir, "blob").length()) + assertEquals(StagedBody.CHUNKED, staged.body.kind) + assertEquals(20, staged.body.totalBytes) + } + + @Test + fun `an orphan blob is adopted when the source is gone`() { + val dir = tmp.newFolder() + File(dir, "blob").writeBytes(ByteArray(20)) + val staged = BodyStaging.stage(desc(url = null, file = "/gone.bin", parts = listOf(part(0, 20))), dir, 1) + assertEquals(StagedBody.CHUNKED, staged.body.kind) + } + + @Test + fun `a tiling mismatch rejects E_INVALID before the move, so the source stays`() { + val src = source("video.bin", ByteArray(20)) + val dir = tmp.newFolder() + val e = assertThrows(QueueException::class.java) { + BodyStaging.stage(desc(url = null, file = src.path, parts = listOf(part(0, 15))), dir, 1) + } + assertEquals(QueueException.E_INVALID, e.code) + assertTrue(src.exists()) + assertFalse(File(dir, "blob").exists()) + } + + @Test + fun `keepOwned runs over the owned blob and ignores the path`() { + val dir = tmp.newFolder() + val owned = File(dir, "blob").apply { writeBytes(ByteArray(20)) } + val other = source("other.bin", ByteArray(5)) + val staged = BodyStaging.stage( + desc(url = null, file = other.path, parts = listOf(part(0, 20))), dir, 2, owned, keepOwned = true, + ) + assertTrue(other.exists()) + assertEquals("blob", staged.body.fileName) + assertEquals(20, owned.length()) + } + + @Test + fun `a present source wins over the owned blob and moves to this generation's name`() { + val dir = tmp.newFolder() + val owned = File(dir, "blob").apply { writeBytes(ByteArray(20)) } + val bytes = ByteArray(20) { (it + 1).toByte() } + val src = source("new.bin", bytes) + val staged = BodyStaging.stage(desc(url = null, file = src.path, parts = listOf(part(0, 5), part(5, 20))), dir, 3, owned) + assertEquals("blob-3", staged.body.fileName) + assertArrayEquals(bytes, File(dir, "blob-3").readBytes()) + assertFalse(src.exists()) + // The old entry's blob is untouched until the new entry is saved and prunes it. + assertArrayEquals(ByteArray(20), owned.readBytes()) + } + + @Test + fun `a present source of another size is checked against its own length`() { + val dir = tmp.newFolder() + val owned = File(dir, "blob").apply { writeBytes(ByteArray(20)) } + val src = source("new.bin", ByteArray(30)) + val staged = BodyStaging.stage(desc(url = null, file = src.path, parts = listOf(part(0, 30))), dir, 2, owned) + assertEquals("blob-2", staged.body.fileName) + assertEquals(30, staged.body.totalBytes) + assertEquals(20, owned.length()) + } + + @Test + fun `with the source gone, the owned blob is the fallback`() { + val dir = tmp.newFolder() + File(dir, "blob").writeBytes(ByteArray(20)) + val staged = BodyStaging.stage( + desc(url = null, file = "/gone.bin", parts = listOf(part(0, 5), part(5, 20))), dir, 2, File(dir, "blob"), + ) + assertEquals("blob", staged.body.fileName) + } + + @Test + fun `with the source gone, this generation's crash leftover wins over the owned blob`() { + val dir = tmp.newFolder() + File(dir, "blob").writeBytes(ByteArray(20)) + File(dir, "blob-2").writeBytes(ByteArray(30)) + val staged = BodyStaging.stage(desc(url = null, file = "/gone.bin", parts = listOf(part(0, 30))), dir, 2, File(dir, "blob")) + assertEquals("blob-2", staged.body.fileName) + } + + @Test + fun `a chunked body with no source and no blob rejects E_FILE_MISSING`() { + val e = assertThrows(QueueException::class.java) { + BodyStaging.stage(desc(url = null, file = "/gone.bin", parts = listOf(part(0, 20))), tmp.newFolder(), 2, null) + } + assertEquals(QueueException.E_FILE_MISSING, e.code) + } + + @Test + fun `a missing source rejects E_FILE_MISSING and writes nothing`() { + val dir = tmp.newFolder() + val form = listOf( + FormPart("a", "text/plain", "x", null, null), + FormPart("b", "image/jpeg", null, "/missing.jpg", null), + ) + val e = assertThrows(QueueException::class.java) { BodyStaging.stage(desc(form = form), dir, 1) } + assertEquals(QueueException.E_FILE_MISSING, e.code) + assertEquals(0, dir.list()!!.size) + val f = assertThrows(QueueException::class.java) { BodyStaging.stage(desc(file = "/missing.bin"), dir, 1) } + assertEquals(QueueException.E_FILE_MISSING, f.code) + val c = assertThrows(QueueException::class.java) { + BodyStaging.stage(desc(url = null, file = "/missing.bin", parts = listOf(part(0, 1))), dir, 1) + } + assertEquals(QueueException.E_FILE_MISSING, c.code) + } + + @Test + fun `no body stages nothing`() { + val dir = tmp.newFolder() + val staged = BodyStaging.stage(desc(method = "DELETE"), dir, 1) + assertEquals(StagedBody(StagedBody.NONE, null, null, 0), staged.body) + assertEquals(0, dir.list()!!.size) + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/ChunkedEngineTest.kt b/android/src/test/java/ai/openspace/backgroundupload/ChunkedEngineTest.kt index 8c909d62..453bab91 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/ChunkedEngineTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/ChunkedEngineTest.kt @@ -11,8 +11,8 @@ import java.util.concurrent.ConcurrentHashMap class ChunkedEngineTest { // runBlocking is single-threaded, so the overlap is deterministic. Every - // executor suspends at yield(). Thus all launchable siblings start before any - // executor finishes. + // executor suspends at yield(), so all launchable siblings start before + // any executor finishes. private class Tracker { var inFlight = 0 var maxInFlight = 0 @@ -39,6 +39,7 @@ class ChunkedEngineTest { fun `never more than WINDOW parts in flight`() = runBlocking { val tracker = Tracker() ChunkedEngine.run((0 until 10).toList()) { tracker.execute(it) } + assertEquals(3, ChunkedEngine.WINDOW) assertEquals(ChunkedEngine.WINDOW, tracker.maxInFlight) } @@ -55,7 +56,7 @@ class ChunkedEngineTest { } @Test - fun `a terminal part failure propagates and cancels the remaining parts`() { + fun `a part failure propagates and cancels the remaining parts`() { val tracker = Tracker() val thrown = assertThrows(IllegalStateException::class.java) { runBlocking { @@ -70,112 +71,17 @@ class ChunkedEngineTest { } @Test - fun `backoff grows exponentially and caps`() { - assertEquals(1_000, ChunkedEngine.backoffMs(1)) - assertEquals(2_000, ChunkedEngine.backoffMs(2)) - assertEquals(4_000, ChunkedEngine.backoffMs(3)) - assertEquals(60_000, ChunkedEngine.backoffMs(7)) - assertEquals(60_000, ChunkedEngine.backoffMs(100)) - // Defensive: a nonsense attempt number must not shift into a huge delay. - assertEquals(1_000, ChunkedEngine.backoffMs(0)) - } - - // MARK: - startAction - - private fun manifest(vararg accepted: Boolean) = ChunkedManifest( - id = "u1", - sourcePath = "/data/blob", - parts = accepted.mapIndexed { i, a -> - ChunkedManifest.Part( - url = "https://example.com/part?n=$i", - headers = emptyMap(), - start = i * 100L, - end = (i + 1) * 100L, - accepted = a, - ) - }, - accept = emptyList(), - expiresAt = 5_000, - wifiOnly = false, - noNotification = false, - createdAt = 1_000, - ) - - @Test - fun `no manifest at start is a silent success, never a journaled error`() { - // A completed ack or removeUpload deleted the manifest while this run sat - // in the queue. That is a legitimate end, already settled. - assertEquals(ChunkedEngine.StartAction.NO_MANIFEST, ChunkedEngine.startAction(null)) - } - - @Test - fun `an all-accepted manifest re-reports completion instead of running`() { - assertEquals( - ChunkedEngine.StartAction.ALREADY_COMPLETE, - ChunkedEngine.startAction(manifest(true, true)), - ) - } - - @Test - fun `pending parts run the engine`() { - assertEquals(ChunkedEngine.StartAction.RUN, ChunkedEngine.startAction(manifest(true, false))) - } - - // MARK: - completionReport - - private fun completedEntry(uploadId: String) = EventJournal.Entry( - eventId = "e-$uploadId", - uploadId = uploadId, - type = "completed", - timestamp = 1, - ) - - @Test - fun `an unacked completed entry is re-emitted, never minted twice`() { - assertEquals( - ChunkedEngine.CompletionReport.ReEmit(completedEntry("u1")), - ChunkedEngine.completionReport(listOf(completedEntry("u1")), "u1", freshCompletion = false), - ) - assertEquals( - ChunkedEngine.CompletionReport.ReEmit(completedEntry("u1")), - ChunkedEngine.completionReport(listOf(completedEntry("u1")), "u1", freshCompletion = true), - ) - } - - @Test - fun `a fresh completion with nothing journaled mints a new entry`() { - assertEquals( - ChunkedEngine.CompletionReport.Mint, - ChunkedEngine.completionReport(emptyList(), "u1", freshCompletion = true), - ) - } - - @Test - fun `a trailing run over an acked completion reports nothing`() { - // The trailing run raced ackEvents. The journal entry is already gone, but - // the manifest still exists for a moment. An acknowledged completion means - // that nobody is owed an event. A minted event would be a duplicate - // 'completed' for an upload that the consumer already settled. - assertEquals( - ChunkedEngine.CompletionReport.None, - ChunkedEngine.completionReport(emptyList(), "u1", freshCompletion = false), - ) - } - - @Test - fun `another upload's completed entry does not satisfy the lookup`() { - assertEquals( - ChunkedEngine.CompletionReport.None, - ChunkedEngine.completionReport(listOf(completedEntry("other")), "u1", freshCompletion = false), - ) - } - - @Test - fun `only 5xx responses are transient`() { - assertTrue(ChunkedEngine.isTransientHttp(500)) - assertTrue(ChunkedEngine.isTransientHttp(599)) - for (code in listOf(400, 401, 403, 404, 409, 429, 499, 600)) { - assertEquals("code $code", false, ChunkedEngine.isTransientHttp(code)) + fun `a park from one part stops the siblings with the park itself`() { + // The worker needs the ParkException back, not a CancellationException. + val thrown = assertThrows(EntryWorker.ParkException::class.java) { + runBlocking { + ChunkedEngine.run((0 until 6).toList()) { index -> + yield() + if (index == 1) throw EntryWorker.ParkException(4) + yield() + } + } } + assertEquals(4, thrown.headerGeneration) } } diff --git a/android/src/test/java/ai/openspace/backgroundupload/ChunkedManifestTest.kt b/android/src/test/java/ai/openspace/backgroundupload/ChunkedManifestTest.kt deleted file mode 100644 index 798852ef..00000000 --- a/android/src/test/java/ai/openspace/backgroundupload/ChunkedManifestTest.kt +++ /dev/null @@ -1,384 +0,0 @@ -package ai.openspace.backgroundupload - -import ai.openspace.backgroundupload.UploadOutcome.AcceptRule -import org.junit.Assert.assertEquals -import org.junit.Assert.assertFalse -import org.junit.Assert.assertNull -import org.junit.Assert.assertThrows -import org.junit.Assert.assertTrue -import org.junit.Rule -import org.junit.Test -import org.junit.rules.TemporaryFolder -import java.io.File -import java.util.concurrent.CountDownLatch -import java.util.concurrent.TimeUnit - -class ChunkedManifestTest { - @get:Rule - val tmp = TemporaryFolder() - - private fun manifest( - id: String = "u1", - parts: List = listOf( - part(0, 100), - part(100, 250), - ), - expiresAt: Long = 5_000, - ) = ChunkedManifest( - id = id, - sourcePath = "/data/blob", - parts = parts, - accept = listOf(AcceptRule(409, "already completed")), - expiresAt = expiresAt, - wifiOnly = false, - noNotification = false, - createdAt = 1_000, - ) - - private fun part(start: Long, end: Long, accepted: Boolean = false) = - ChunkedManifest.Part( - url = "https://example.com/part?start=$start", - headers = mapOf("Authorization" to "Bearer old"), - start = start, - end = end, - accepted = accepted, - ) - - // MARK: - Model - - @Test - fun `byte math is range-based`() { - val m = manifest(parts = listOf(part(0, 100, accepted = true), part(100, 250))) - assertEquals(250, m.totalBytes) - assertEquals(100, m.acceptedBytes) - } - - @Test - fun `completed only when every part is accepted`() { - val none = manifest() - assertFalse(none.allAccepted) - val partial = none.withPartAccepted(0) - assertFalse(partial.allAccepted) - val all = partial.withPartAccepted(1) - assertTrue(all.allAccepted) - assertEquals(emptyList(), all.pendingIndexes()) - assertEquals(listOf(1), partial.pendingIndexes()) - } - - @Test - fun `expiry is inclusive of the deadline`() { - val m = manifest(expiresAt = 5_000) - assertFalse(m.isExpired(4_999)) - assertTrue(m.isExpired(5_000)) - assertTrue(m.isExpired(5_001)) - } - - // MARK: - Reconcile: resume (same parts array) - - private val blobSize = 250L - - @Test - fun `resume replaces headers and deadline, keeps accepted parts and the moved source`() { - val stored = manifest().withPartAccepted(0) - val fresh = manifest(expiresAt = 99_000).copy( - sourcePath = "/ignored/by/reconcile", - createdAt = 42, - accept = listOf(AcceptRule(208)), - wifiOnly = true, - parts = stored.parts.map { it.copy(headers = mapOf("Authorization" to "Bearer new"), accepted = false) }, - ) - - val merged = stored.reconcile(fresh, running = false, blobSize = blobSize) - - assertEquals("Bearer new", merged.parts[0].headers["Authorization"]) - assertEquals(99_000, merged.expiresAt) - assertEquals(listOf(AcceptRule(208)), merged.accept) - assertTrue(merged.wifiOnly) - // Accepted statuses, ownership, and identity survive from the stored copy. - assertTrue(merged.parts[0].accepted) - assertFalse(merged.parts[1].accepted) - assertEquals("/data/blob", merged.sourcePath) - assertEquals(1_000, merged.createdAt) - } - - @Test - fun `resume is allowed while the upload is running`() { - // Fresh auth must reach a running worker's stalled parts. - val stored = manifest().withPartAccepted(0) - val fresh = manifest().copy( - parts = stored.parts.map { it.copy(headers = mapOf("Authorization" to "Bearer new"), accepted = false) }, - ) - val merged = stored.reconcile(fresh, running = true, blobSize = blobSize) - assertTrue(merged.parts[0].accepted) - assertEquals("Bearer new", merged.parts[1].headers["Authorization"]) - } - - @Test - fun `resume matches the same parts authored in a different order`() { - // Identical tiles, reordered, are the SAME upload: a resume, never a - // recreate (running = true would reject a recreate). Accepted flags follow - // the range, not the array index. - val stored = manifest().withPartAccepted(0) - val fresh = manifest().copy( - parts = listOf(stored.parts[1], stored.parts[0]).map { - it.copy(headers = mapOf("Authorization" to "Bearer new"), accepted = false) - }, - ) - val merged = stored.reconcile(fresh, running = true, blobSize = blobSize) - assertTrue(merged.parts.first { it.start == 0L }.accepted) - assertFalse(merged.parts.first { it.start == 100L }.accepted) - assertEquals("Bearer new", merged.parts[0].headers["Authorization"]) - } - - // MARK: - Reconcile: recreate (different parts array) - - @Test - fun `recreate from a stalled upload replaces parts and resets every status`() { - // The consumer re-authored under a fresh server uploadId: new urls, a new - // split, and fresh headers, accept, and expiresAt. The owned bytes stay. - val stored = manifest().withPartAccepted(0) - val fresh = manifest( - parts = listOf( - part(0, 120).copy(url = "https://example.com/v2?part=1"), - part(120, 250).copy(url = "https://example.com/v2?part=2"), - ), - expiresAt = 99_000, - ).copy(sourcePath = "/ignored/by/reconcile", createdAt = 42, accept = listOf(AcceptRule(208))) - - val recreated = stored.reconcile(fresh, running = false, blobSize = blobSize) - - assertTrue(recreated.parts.none { it.accepted }) - assertEquals(listOf("https://example.com/v2?part=1", "https://example.com/v2?part=2"), recreated.parts.map { it.url }) - assertEquals(99_000, recreated.expiresAt) - assertEquals(listOf(AcceptRule(208)), recreated.accept) - // Ownership survives. The blob is reused for the full re-upload. - assertEquals("/data/blob", recreated.sourcePath) - assertEquals(1_000, recreated.createdAt) - } - - @Test - fun `recreate with the same ranges but new urls also resets statuses`() { - // New part urls embed a new server uploadId, even when the split is - // identical. Nothing sent under the old id counts for the new one. - val stored = manifest().withPartAccepted(0) - val fresh = manifest( - parts = listOf( - part(0, 100).copy(url = "https://example.com/v2?part=1"), - part(100, 250).copy(url = "https://example.com/v2?part=2"), - ), - ) - val recreated = stored.reconcile(fresh, running = false, blobSize = blobSize) - assertTrue(recreated.parts.none { it.accepted }) - } - - @Test - fun `recreate is rejected while the upload is running`() { - val stored = manifest() - val fresh = manifest(parts = listOf(part(0, 250).copy(url = "https://example.com/v2"))) - assertThrows(ChunkedManifest.ReconcileException::class.java) { - stored.reconcile(fresh, running = true, blobSize = blobSize) - } - } - - @Test - fun `recreate rejects parts that do not tile the blob exactly`() { - val stored = manifest() - for ( - bad in listOf( - listOf(part(0, 100), part(150, 250)), // gap - listOf(part(0, 150), part(100, 250)), // overlap - listOf(part(50, 250)), // does not start at 0 - listOf(part(0, 200)), // short of the blob size - listOf(part(0, 100), part(100, 251)), // past the blob size - ) - ) { - assertThrows(ChunkedManifest.ReconcileException::class.java) { - stored.reconcile(manifest(parts = bad), running = false, blobSize = blobSize) - } - } - } - - @Test - fun `recreate accepts parts authored in any order`() { - val stored = manifest() - val fresh = manifest(parts = listOf(part(100, 250), part(0, 100)).map { it.copy(url = it.url + "&v=2") }) - val recreated = stored.reconcile(fresh, running = false, blobSize = blobSize) - assertEquals(2, recreated.parts.size) - } - - @Test - fun `tilesExactly covers the edge shapes`() { - assertTrue(ChunkedManifest.tilesExactly(listOf(part(0, 250)), 250)) - assertFalse(ChunkedManifest.tilesExactly(emptyList(), 0)) - assertFalse(ChunkedManifest.tilesExactly(listOf(part(0, 0)), 0)) // empty range - assertFalse(ChunkedManifest.tilesExactly(listOf(part(0, 250)), 300)) - } - - // MARK: - Create validation - - @Test - fun `create accepts parts that tile the blob exactly`() { - val m = manifest() - assertEquals(m, ChunkedManifest.validatedForCreate(m, blobSize)) - } - - @Test - fun `create rejects parts that do not tile the blob`() { - for ( - bad in listOf( - listOf(part(0, 100), part(150, 250)), // gap - listOf(part(0, 150), part(100, 250)), // overlap - listOf(part(50, 250)), // does not start at 0 - listOf(part(0, 200)), // short of the blob size - listOf(part(0, 100), part(100, 251)), // past the blob size - ) - ) { - assertThrows(ChunkedManifest.ReconcileException::class.java) { - ChunkedManifest.validatedForCreate(manifest(parts = bad), blobSize) - } - } - } - - @Test - fun `a rejected create writes no manifest, leaving the blob adoptable`() { - // startUpload validates AFTER takeOwnership moved the bytes. The throw - // propagates out of compute before a save. Thus the blob sits ownerless at - // its path. That is exactly what takeOwnership's orphan branch adopts on - // the corrected retry. - val store = ChunkedManifestStore(tmp.newFolder()) - store.blobFile("u1").apply { parentFile!!.mkdirs() }.writeText("owned bytes") - assertThrows(ChunkedManifest.ReconcileException::class.java) { - store.compute("u1") { ChunkedManifest.validatedForCreate(manifest(), 999L) } - } - assertNull(store.load("u1")) - assertTrue(store.blobFile("u1").exists()) - } - - // MARK: - Store - - @Test - fun `save then load round-trips, across store instances`() { - val dir = tmp.newFolder() - val m = manifest().withPartAccepted(1) - ChunkedManifestStore(dir).save(m) - // A new instance over the same dir is what a process relaunch looks like. - assertEquals(m, ChunkedManifestStore(dir).load("u1")) - } - - @Test - fun `load returns null for an unknown id`() { - assertNull(ChunkedManifestStore(tmp.newFolder()).load("nope")) - } - - @Test - fun `update persists the transformed manifest`() { - val store = ChunkedManifestStore(tmp.newFolder()) - store.save(manifest()) - val updated = store.update("u1") { it.withPartAccepted(0) } - assertTrue(updated!!.parts[0].accepted) - assertTrue(store.load("u1")!!.parts[0].accepted) - } - - @Test - fun `update of a missing manifest returns null`() { - assertNull(ChunkedManifestStore(tmp.newFolder()).update("nope") { it }) - } - - @Test - fun `compute creates when no manifest exists`() { - val store = ChunkedManifestStore(tmp.newFolder()) - val created = store.compute("u1") { existing -> - assertNull(existing) - manifest() - } - assertEquals(created, store.load("u1")) - } - - @Test - fun `compute holds the store lock across load, transform, and save`() { - // The startUpload reconcile and a running worker's markAccepted race. If - // the lock did not span all three steps, the update below could land - // between compute's load and save, and it would be erased from disk. When - // they are serialized, both effects must survive. - val store = ChunkedManifestStore(tmp.newFolder()) - store.save(manifest()) - val inTransform = CountDownLatch(1) - val computing = Thread { - store.compute("u1") { existing -> - inTransform.countDown() - Thread.sleep(300) // hold the lock with load done and save not yet run - existing!!.copy(expiresAt = 99_000) - } - }.apply { start() } - assertTrue(inTransform.await(5, TimeUnit.SECONDS)) - val updating = Thread { store.update("u1") { it.withPartAccepted(0) } }.apply { start() } - computing.join() - updating.join() - val final = store.load("u1")!! - assertEquals(99_000, final.expiresAt) - assertTrue(final.parts[0].accepted) - } - - @Test - fun `a throwing compute transform propagates and writes nothing`() { - val store = ChunkedManifestStore(tmp.newFolder()) - store.save(manifest()) - assertThrows(ChunkedManifest.ReconcileException::class.java) { - store.compute("u1") { throw ChunkedManifest.ReconcileException("rejected") } - } - assertEquals(manifest(), store.load("u1")) - } - - @Test - fun `contains tracks save and remove`() { - val store = ChunkedManifestStore(tmp.newFolder()) - assertFalse(store.contains("u1")) - store.save(manifest()) - assertTrue(store.contains("u1")) - store.remove("u1") - assertFalse(store.contains("u1")) - } - - @Test - fun `remove deletes the manifest and the blob`() { - val store = ChunkedManifestStore(tmp.newFolder()) - store.save(manifest()) - store.blobFile("u1").writeText("bytes") - store.remove("u1") - assertNull(store.load("u1")) - assertFalse(store.blobFile("u1").exists()) - } - - @Test - fun `remove of an unknown id is a no-op`() { - ChunkedManifestStore(tmp.newFolder()).remove("simple-upload-id") - } - - @Test - fun `ids with filesystem-hostile characters round-trip`() { - val store = ChunkedManifestStore(tmp.newFolder()) - val id = "a/b:c dü..\\e" - store.save(manifest(id = id)) - assertEquals(id, store.load(id)!!.id) - store.remove(id) - assertNull(store.load(id)) - } - - @Test - fun `a corrupt manifest reads as absent, not fatal`() { - val dir = tmp.newFolder() - val store = ChunkedManifestStore(dir) - store.save(manifest()) - File(File(dir, dir.list()!!.first()), "manifest.json").writeText("{not json") - assertNull(store.load("u1")) - assertEquals(emptyList(), store.all()) - } - - @Test - fun `all lists every stored manifest`() { - val store = ChunkedManifestStore(tmp.newFolder()) - store.save(manifest(id = "u1")) - store.save(manifest(id = "u2").withPartAccepted(0)) - assertEquals(setOf("u1", "u2"), store.all().map { it.id }.toSet()) - } -} diff --git a/android/src/test/java/ai/openspace/backgroundupload/ChunkedPartsTest.kt b/android/src/test/java/ai/openspace/backgroundupload/ChunkedPartsTest.kt new file mode 100644 index 00000000..e7ba858e --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/ChunkedPartsTest.kt @@ -0,0 +1,56 @@ +package ai.openspace.backgroundupload + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test + +class ChunkedPartsTest { + + @Test + fun `tilesExactly covers the edge shapes`() { + assertTrue(ChunkedParts.tilesExactly(listOf(part(0, 100), part(100, 250)), 250)) + assertTrue(ChunkedParts.tilesExactly(listOf(part(100, 250), part(0, 100)), 250)) // any order + assertFalse(ChunkedParts.tilesExactly(emptyList(), 0)) + assertFalse(ChunkedParts.tilesExactly(listOf(part(0, 0)), 0)) // empty range + assertFalse(ChunkedParts.tilesExactly(listOf(part(0, 100), part(150, 250)), 250)) // gap + assertFalse(ChunkedParts.tilesExactly(listOf(part(0, 150), part(100, 250)), 250)) // overlap + assertFalse(ChunkedParts.tilesExactly(listOf(part(50, 250)), 250)) // not from 0 + assertFalse(ChunkedParts.tilesExactly(listOf(part(0, 200)), 250)) // short + assertFalse(ChunkedParts.tilesExactly(listOf(part(0, 251)), 250)) // past the end + } + + @Test + fun `the same parts in another order are the same upload`() { + val a = listOf(part(0, 100), part(100, 250)) + assertTrue(ChunkedParts.sameParts(a, a.reversed())) + // Headers are not compared: a resume sends fresh ones. + assertTrue(ChunkedParts.sameParts(a, a.map { it.copy(headers = mapOf("X" to "new")) })) + } + + @Test + fun `new urls or a new split are different parts`() { + val a = listOf(part(0, 100), part(100, 250)) + assertFalse(ChunkedParts.sameParts(a, listOf(part(0, 100, url = "https://v2/1"), part(100, 250)))) + assertFalse(ChunkedParts.sameParts(a, listOf(part(0, 120), part(120, 250)))) + assertFalse(ChunkedParts.sameParts(a, listOf(part(0, 250)))) + } + + @Test + fun `accepted flags follow the range, not the index`() { + val stored = listOf(part(0, 100, accepted = true), part(100, 250)) + val incoming = listOf(part(100, 250), part(0, 100)) + val carried = ChunkedParts.carryAccepted(stored, incoming) + assertTrue(carried.first { it.start == 0L }.accepted) + assertFalse(carried.first { it.start == 100L }.accepted) + } + + @Test + fun `byte math and pending indexes`() { + val parts = listOf(part(0, 100, accepted = true), part(100, 250)) + assertEquals(250, ChunkedParts.totalBytes(parts)) + assertEquals(100, ChunkedParts.acceptedBytes(parts)) + assertEquals(listOf(1), ChunkedParts.pendingIndexes(parts)) + assertEquals(emptyList(), ChunkedParts.pendingIndexes(ChunkedParts.withAccepted(parts, 1))) + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/ChunkedWorkerGateTest.kt b/android/src/test/java/ai/openspace/backgroundupload/ChunkedWorkerGateTest.kt deleted file mode 100644 index 0b65d54b..00000000 --- a/android/src/test/java/ai/openspace/backgroundupload/ChunkedWorkerGateTest.kt +++ /dev/null @@ -1,57 +0,0 @@ -package ai.openspace.backgroundupload - -import org.junit.After -import org.junit.Assert.assertFalse -import org.junit.Assert.assertTrue -import org.junit.Test - -class ChunkedWorkerGateTest { - private val a = Any() - private val b = Any() - - @After - fun tearDown() { - // The gate is a process-wide singleton. Leave nothing for other tests. - ChunkedWorkerGate.release("u1", a) - ChunkedWorkerGate.release("u1", b) - ChunkedWorkerGate.release("u2", a) - } - - @Test - fun `a second worker for the same id must wait`() { - assertTrue(ChunkedWorkerGate.tryAcquire("u1", a)) - // The replacement worker after a cancel-then-start: it must not run a part - // PUT while the cancelled worker still holds the id. - assertFalse(ChunkedWorkerGate.tryAcquire("u1", b)) - ChunkedWorkerGate.release("u1", a) - assertTrue(ChunkedWorkerGate.tryAcquire("u1", b)) - } - - @Test - fun `reacquiring with the same token is idempotent`() { - assertTrue(ChunkedWorkerGate.tryAcquire("u1", a)) - assertTrue(ChunkedWorkerGate.tryAcquire("u1", a)) - } - - @Test - fun `a stale release cannot evict a successor`() { - assertTrue(ChunkedWorkerGate.tryAcquire("u1", a)) - ChunkedWorkerGate.release("u1", a) - assertTrue(ChunkedWorkerGate.tryAcquire("u1", b)) - ChunkedWorkerGate.release("u1", a) // the old worker's finally, arriving late - assertTrue(ChunkedWorkerGate.isRunning("u1")) - assertFalse(ChunkedWorkerGate.tryAcquire("u1", a)) - } - - @Test - fun `ids are independent and isRunning tracks the holder`() { - assertFalse(ChunkedWorkerGate.isRunning("u1")) - assertTrue(ChunkedWorkerGate.tryAcquire("u1", a)) - assertTrue(ChunkedWorkerGate.isRunning("u1")) - assertFalse(ChunkedWorkerGate.isRunning("u2")) - assertTrue(ChunkedWorkerGate.tryAcquire("u2", a)) - ChunkedWorkerGate.release("u1", a) - assertFalse(ChunkedWorkerGate.isRunning("u1")) - assertTrue(ChunkedWorkerGate.isRunning("u2")) - } -} diff --git a/android/src/test/java/ai/openspace/backgroundupload/EntryParsingTest.kt b/android/src/test/java/ai/openspace/backgroundupload/EntryParsingTest.kt new file mode 100644 index 00000000..1d0fc10c --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/EntryParsingTest.kt @@ -0,0 +1,124 @@ +package ai.openspace.backgroundupload + +import com.facebook.react.bridge.JavaOnlyArray +import com.facebook.react.bridge.JavaOnlyMap +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Assert.assertThrows +import org.junit.Test + +class EntryParsingTest { + + private fun entryMap(descriptor: JavaOnlyMap, vars: Any? = JavaOnlyMap.of("n", 1.0)) = + JavaOnlyMap.of("id", "e1", "key", "note", "vars", vars, "descriptor", descriptor) + + private fun base(vararg extra: Any?) = + JavaOnlyMap.of("url", "https://example.com/items", "expiresAt", 9_000.0, *extra) + + @Test + fun `a JSON POST parses with defaults`() { + val p = EntryParsing.parse(entryMap(base("data", JavaOnlyMap.of("n", 1.0, "text", "hi")))) + assertEquals("e1", p.id) + assertEquals("note", p.key) + assertEquals("""{"n":1}""", p.varsJson) + assertEquals(9_000L, p.expiresAt) + assertEquals("POST", p.descriptor.method) + assertEquals("""{"n":1,"text":"hi"}""", p.descriptor.dataJson) + assertEquals(StagedBody.JSON, p.descriptor.bodyKind) + } + + @Test + fun `null vars store as the text null`() { + assertEquals("null", EntryParsing.parse(entryMap(base(), vars = null)).varsJson) + } + + @Test + fun `a null data is no body, because the bridge turns undefined into null`() { + assertNull(EntryParsing.parse(entryMap(base("data", null))).descriptor.dataJson) + } + + @Test + fun `chunked parts, headers, accept, retry, and android parse`() { + val d = JavaOnlyMap.of( + "method", "PUT", + "file", "file:///data/a%20b.bin", + "expiresAt", 9_000.0, + "headers", JavaOnlyMap.of("Content-Type", "video/mp4", "X-N", 5.0), + "parts", JavaOnlyArray.of( + JavaOnlyMap.of("url", "https://s3/1", "range", JavaOnlyMap.of("start", 0.0, "end", 10.0)), + JavaOnlyMap.of( + "url", "https://s3/2", "headers", JavaOnlyMap.of("Content-Range", "10-19"), + "range", JavaOnlyMap.of("start", 10.0, "end", 20.0), + ), + ), + "accept", JavaOnlyArray.of(JavaOnlyMap.of("status", 409.0, "bodyIncludes", "already completed")), + "retry", JavaOnlyMap.of( + "backoff", JavaOnlyMap.of("baseMs", 50.0), + "terminalHttp", JavaOnlyMap.of("exempt", JavaOnlyArray()), + ), + "android", JavaOnlyMap.of("noNotification", true), + ) + val parsed = EntryParsing.parse(entryMap(d)).descriptor + assertEquals("/data/a b.bin", parsed.file) + assertEquals(StagedBody.CHUNKED, parsed.bodyKind) + assertEquals(listOf(Part("https://s3/1", mapOf(), 0, 10), Part("https://s3/2", mapOf("Content-Range" to "10-19"), 10, 20)), parsed.parts) + assertEquals(mapOf("Content-Type" to "video/mp4", "X-N" to "5"), parsed.headers) + assertEquals(listOf(UploadOutcome.AcceptRule(409, "already completed")), parsed.accept) + assertEquals(RetryOverride(50, null, null, emptyList()), parsed.retry) + assertEquals(true, parsed.noNotification) + assertEquals("https://s3/2", parsed.reportUrl) + } + + @Test + fun `form parts parse with exactly one of string or path`() { + val d = base( + "form", JavaOnlyArray.of( + JavaOnlyMap.of("name", "meta", "contentType", "application/json", "string", "{}"), + JavaOnlyMap.of("name", "photo", "contentType", "image/jpeg", "path", "/p.jpg", "fileName", "p.jpg"), + ), + ) + assertEquals( + listOf( + FormPart("meta", "application/json", "{}", null, null), + FormPart("photo", "image/jpeg", null, "/p.jpg", "p.jpg"), + ), + EntryParsing.parse(entryMap(d)).descriptor.form, + ) + val both = base("form", JavaOnlyArray.of(JavaOnlyMap.of("name", "x", "contentType", "t", "string", "s", "path", "/p"))) + assertThrows(EntryParsing.InvalidEntryException::class.java) { EntryParsing.parse(entryMap(both)) } + } + + @Test + fun `what native can not run is rejected`() { + val cases = listOf( + JavaOnlyMap.of("url", "https://example.com"), // no expiresAt + JavaOnlyMap.of("expiresAt", 1.0), // no url and no parts + base("data", 1.0, "file", "/a"), // two body kinds + base("method", "GET", "data", 1.0), // GET with a body + base("method", "TRACE"), + JavaOnlyMap.of("url", "not a url", "expiresAt", 1.0), + base("headers", JavaOnlyMap.of("Bad\nName", "v")), + base("parts", JavaOnlyArray.of(JavaOnlyMap.of("url", "https://s3/1", "range", JavaOnlyMap.of("start", 0.0, "end", 1.0)))), // parts without file + ) + cases.forEach { d -> + assertThrows("$d", EntryParsing.InvalidEntryException::class.java) { EntryParsing.parse(entryMap(d)) } + } + assertThrows(EntryParsing.InvalidEntryException::class.java) { + EntryParsing.parse(JavaOnlyMap.of("key", "k", "descriptor", base())) + } + } + + @Test + fun `an updateHeaders patch is checked like descriptor headers`() { + assertEquals(mapOf("Authorization" to "Bearer new"), EntryParsing.headerPatch(JavaOnlyMap.of("Authorization", "Bearer new"))) + assertThrows(EntryParsing.InvalidEntryException::class.java) { + EntryParsing.headerPatch(JavaOnlyMap.of("Authorization", "Bearer\nnew")) + } + } + + @Test + fun `file scheme stripping`() { + assertEquals("/a/b c.jpg", EntryParsing.stripFileScheme("file:///a/b%20c.jpg")) + assertEquals("/a/b.jpg", EntryParsing.stripFileScheme("/a/b.jpg")) + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/EntryTransitionsTest.kt b/android/src/test/java/ai/openspace/backgroundupload/EntryTransitionsTest.kt new file mode 100644 index 00000000..e5a37f73 --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/EntryTransitionsTest.kt @@ -0,0 +1,176 @@ +package ai.openspace.backgroundupload + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test + +class EntryTransitionsTest { + + @Test + fun `a settle on a cancelled entry or an older generation is not allowed`() { + assertTrue(EntryTransitions.canSettle(entry(state = EntryState.RUNNING), 1)) + assertTrue(EntryTransitions.canSettle(entry(state = EntryState.PAUSED), 1)) // an in-flight response under pause + assertFalse(EntryTransitions.canSettle(entry(state = EntryState.CANCELLED), 1)) + assertFalse(EntryTransitions.canSettle(entry(state = EntryState.COMPLETED), 1)) + assertFalse(EntryTransitions.canSettle(entry(state = EntryState.RUNNING, generation = 2), 1)) + assertFalse(EntryTransitions.canSettle(null, 1)) + } + + @Test + fun `resume returns to awaiting-auth only when the parked generation is current`() { + val paused = entry(state = EntryState.PAUSED, parkedGeneration = 3) + assertEquals(EntryState.AWAITING_AUTH, EntryTransitions.toResumed(paused, headerGeneration = 3, now = 9).state) + val stale = EntryTransitions.toResumed(paused, headerGeneration = 4, now = 9) + assertEquals(EntryState.QUEUED, stale.state) + assertNull(stale.parkedGeneration) + assertEquals(EntryState.QUEUED, EntryTransitions.toResumed(entry(state = EntryState.PAUSED), 0, 9).state) + } + + @Test + fun `pause keeps the parked generation and clears the wake time`() { + val p = EntryTransitions.toPaused(entry(state = EntryState.AWAITING_AUTH, parkedGeneration = 2, nextAttemptAt = 50), 9) + assertEquals(EntryState.PAUSED, p.state) + assertEquals(2, p.parkedGeneration) + assertNull(p.nextAttemptAt) + } + + @Test + fun `park, release, run, stop, settle`() { + val running = EntryTransitions.toRunning(entry(nextAttemptAt = 5), 9) + assertEquals(EntryState.RUNNING, running.state) + assertNull(running.nextAttemptAt) + + val parked = EntryTransitions.toParked(running.copy(backoffStreak = 4), 7, 10) + assertEquals(EntryState.AWAITING_AUTH, parked.state) + assertEquals(7, parked.parkedGeneration) + assertEquals(0, parked.backoffStreak) + + val released = EntryTransitions.toReleased(running, 99, 6, 11) + assertEquals(EntryState.QUEUED, released.state) + assertEquals(99L, released.nextAttemptAt) + assertEquals(6, released.backoffStreak) + + assertEquals(EntryState.QUEUED, EntryTransitions.toStopped(running, 12).state) + + val settled = EntryTransitions.toSettled(parked, EntryState.ERROR, "ev", 5, 13) + assertEquals("ev", settled.settledEventId) + assertNull(settled.parkedGeneration) + assertEquals(13, settled.updatedAt) + } + + @Test + fun `a short backoff keeps the row running and shows the time, and the next attempt clears it`() { + val waiting = EntryTransitions.toBackingOff(entry(state = EntryState.RUNNING), 9_000, 5) + assertEquals(EntryState.RUNNING, waiting.state) + assertEquals(9_000L, waiting.nextAttemptAt) + assertEquals(9_000.0, waiting.toRow().toMap()["nextAttemptAt"]) + val attempt = EntryTransitions.toAttempt(waiting, "req-2", 6) + assertNull(attempt.nextAttemptAt) + assertEquals(1, attempt.attempts) + assertEquals("req-2", attempt.lastRequestId) + assertFalse(attempt.toRow().toMap().containsKey("nextAttemptAt")) + } +} + +class EnqueueRulesTest { + private val body = desc(dataJson = """{"a":1}""") + private val other = desc(dataJson = """{"a":2}""") + + private fun decide(existing: QueueEntry?, incoming: Descriptor = body, hasRecord: Boolean = true, v9: LegacyManifest? = null) = + EnqueueRules.decide(existing, v9, incoming) { hasRecord } + + @Test + fun `the same-id table`() { + assertEquals(EnqueueRules.Action.Create, decide(null)) + val v9 = LegacyManifest("e1", "/b", listOf(part(0, 10)), emptyList(), 1, false, 1) + assertEquals(EnqueueRules.Action.AdoptV9(v9), decide(null, v9 = v9)) + assertEquals(EnqueueRules.Action.Replace, decide(entry(legacy = true, descriptor = null, body = null))) + assertEquals(EnqueueRules.Action.ReEmit("ev"), decide(entry(state = EntryState.COMPLETED, settledEventId = "ev"))) + assertEquals(EnqueueRules.Action.Replace, decide(entry(state = EntryState.COMPLETED, settledEventId = "ev"), hasRecord = false)) + EntryState.values().filter { it != EntryState.COMPLETED }.forEach { state -> + assertEquals("$state", EnqueueRules.Action.Resume, decide(entry(state = state))) + } + assertEquals(EnqueueRules.Action.RejectRunning, decide(entry(state = EntryState.RUNNING), other)) + EntryState.values().filter { it != EntryState.RUNNING }.forEach { state -> + assertEquals("$state", EnqueueRules.Action.Replace, decide(entry(state = state), other)) + } + } + + @Test + fun `resume replaces the metadata, keeps the body and accepted parts`() { + val parts = listOf(part(0, 10, accepted = true), part(10, 20)) + val stored = entry( + state = EntryState.AWAITING_AUTH, + descriptor = desc(url = null, method = "PUT", file = "/f", parts = parts), + body = StagedBody(StagedBody.CHUNKED, "blob", null, 20), + attempts = 4, + parkedGeneration = 1, + ) + val incoming = parsed( + descriptor = desc(url = null, method = "PUT", file = "/f", headers = mapOf("Authorization" to "Bearer new"), + parts = parts.map { it.copy(accepted = false) }), + varsJson = """{"n":2}""", + expiresAt = 77, + ) + val next = EnqueueRules.resumed(stored, incoming, paused = false, headerGeneration = 5, now = 9) + assertEquals(EntryState.QUEUED, next.state) + assertEquals(mapOf("Authorization" to "Bearer new"), next.descriptor!!.headers) + assertTrue(next.descriptor!!.parts!![0].accepted) + assertEquals(10, next.bytesSent) + assertEquals(4, next.attempts) + assertEquals(77, next.expiresAt) + assertEquals("""{"n":2}""", next.varsJson) + assertEquals(5, next.headerGeneration) + assertNull(next.parkedGeneration) + assertEquals(1, next.generation) // a live entry keeps its life + } + + @Test + fun `resume of a settled entry reopens it with a fresh generation`() { + val next = EnqueueRules.resumed(entry(state = EntryState.ERROR, settledEventId = "ev", generation = 2), parsed(), false, 0, 9) + assertEquals(EntryState.QUEUED, next.state) + assertEquals(3, next.generation) + assertNull(next.settledEventId) + } + + @Test + fun `resume of a running entry stays running, and under pause becomes paused`() { + assertEquals(EntryState.RUNNING, EnqueueRules.resumed(entry(state = EntryState.RUNNING), parsed(), true, 0, 9).state) + assertEquals(EntryState.PAUSED, EnqueueRules.resumed(entry(state = EntryState.QUEUED), parsed(), true, 0, 9).state) + } + + @Test + fun `resume re-applies the json content type`() { + val next = EnqueueRules.resumed(entry(), parsed(descriptor = desc(dataJson = """{"a":1}""", headers = mapOf())), false, 0, 9) + assertEquals(mapOf("Content-Type" to "application/json"), next.descriptor!!.headers) + } + + @Test + fun `adopting v9 parts carries the flags only for the same parts`() { + val v9 = LegacyManifest("e1", "/b", listOf(part(0, 10, accepted = true), part(10, 20)), emptyList(), 1, false, 1) + assertTrue(EnqueueRules.adoptedParts(v9, listOf(part(0, 10), part(10, 20)))[0].accepted) + assertFalse(EnqueueRules.adoptedParts(v9, listOf(part(0, 20)))[0].accepted) + } + + @Test + fun `a copied body is staged outside the lock only when no worker can change the decision`() { + val json = desc(dataJson = """{"a":2}""") + val create = EnqueueRules.Action.Create + val replace = EnqueueRules.Action.Replace + assertEquals(1, EnqueueRules.preStageGeneration(null, create, json)) + assertEquals(1, EnqueueRules.preStageGeneration(null, create, desc(file = "/f"))) + assertEquals(3, EnqueueRules.preStageGeneration(entry(state = EntryState.ERROR, generation = 2), replace, json)) + assertEquals(2, EnqueueRules.preStageGeneration(entry(state = EntryState.PAUSED), replace, json)) + // A worker can take a queued entry meanwhile, and a running one is the worker's. + assertNull(EnqueueRules.preStageGeneration(entry(state = EntryState.QUEUED), replace, json)) + assertNull(EnqueueRules.preStageGeneration(entry(state = EntryState.RUNNING), replace, json)) + // A chunked move and a bodiless request stage under the lock. + assertNull(EnqueueRules.preStageGeneration(null, create, desc(url = null, file = "/f", parts = listOf(part(0, 10))))) + assertNull(EnqueueRules.preStageGeneration(null, create, desc())) + // Nothing to stage. + assertNull(EnqueueRules.preStageGeneration(entry(), EnqueueRules.Action.Resume, json)) + assertNull(EnqueueRules.preStageGeneration(entry(), EnqueueRules.Action.RejectRunning, json)) + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/EventJournalTest.kt b/android/src/test/java/ai/openspace/backgroundupload/EventJournalTest.kt index 811f1bd2..b095c4c7 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/EventJournalTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/EventJournalTest.kt @@ -2,6 +2,7 @@ package ai.openspace.backgroundupload import org.junit.Assert.assertEquals import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull import org.junit.Assert.assertTrue import org.junit.Rule import org.junit.Test @@ -12,93 +13,153 @@ class EventJournalTest { @get:Rule val tmp = TemporaryFolder() - private fun entry(id: String, uploadId: String = "u1") = EventJournal.Entry( - eventId = id, - uploadId = uploadId, - type = "completed", - timestamp = System.currentTimeMillis(), - responseCode = 200, - responseBody = "ok", - responseHeaders = mapOf("x-a" to "b"), - ) + private val id1 = "00000000-0000-0000-0000-000000000001" + private val id2 = "00000000-0000-0000-0000-000000000002" @Test - fun `append then read returns the entry`() { + fun `append then read returns the record`() { val journal = EventJournal(tmp.newFolder()) - journal.append(entry("e1")) + assertTrue(journal.append(record(id1))) val events = journal.unacknowledged() - assertEquals(1, events.size) - assertEquals("e1", events[0].eventId) - assertEquals(200, events[0].responseCode) - assertEquals("ok", events[0].responseBody) + assertEquals(listOf(record(id1)), events) } @Test - fun `ack removes only the acked entry`() { + fun `ack removes only the acked record and is idempotent`() { val journal = EventJournal(tmp.newFolder()) - journal.append(entry("e1")) - journal.append(entry("e2")) - journal.ack(listOf("e1")) - assertEquals(listOf("e2"), journal.unacknowledged().map { it.eventId }) + journal.append(record(id1)) + journal.append(record(id2)) + journal.ack(listOf(id1, id1, "unknown")) + assertEquals(listOf(id2), journal.unacknowledged().map { it.eventId }) } @Test - fun `entries survive a new journal instance over the same dir`() { + fun `ack ignores ids that are not event ids`() { val dir = tmp.newFolder() - EventJournal(dir).append(entry("e1")) + val outside = File(dir.parentFile, "precious.json").apply { writeText("x") } + EventJournal(dir).ack(listOf("../precious")) + assertTrue(outside.exists()) + } + + @Test + fun `records survive a new journal instance`() { + val dir = tmp.newFolder() + EventJournal(dir).append(record(id1)) assertEquals(1, EventJournal(dir).unacknowledged().size) } @Test - fun `oversized body is truncated and flagged`() { + fun `a body over 1 MB is cut and flagged`() { val journal = EventJournal(tmp.newFolder()) val big = "x".repeat(EventJournal.MAX_BODY_CHARS + 100) - journal.append(entry("e1").copy(responseBody = big)) - val read = journal.unacknowledged()[0] - assertTrue(read.responseBodyTruncated) - assertTrue(read.responseBody!!.length <= EventJournal.MAX_BODY_CHARS) + journal.append(record(id1).copy(response = EventJournal.Response(200, null, big, false))) + val read = journal.unacknowledged()[0].response!! + assertTrue(read.bodyTruncated) + assertEquals(EventJournal.MAX_BODY_CHARS, read.body!!.length) } @Test - fun `corrupt file is skipped, not fatal`() { + fun `a corrupt file is skipped`() { val dir = tmp.newFolder() val journal = EventJournal(dir) - journal.append(entry("e1")) - java.io.File(dir, "garbage.json").writeText("{not json") - assertEquals(1, journal.unacknowledged().size) + journal.append(record(id1)) + File(dir, "garbage.json").writeText("{not json") + File(dir, "partial.json").writeText("""{"eventId":"x"}""") + assertEquals(listOf(id1), journal.unacknowledged().map { it.eventId }) } @Test - fun `entries are ordered by timestamp`() { + fun `records are ordered by time`() { val journal = EventJournal(tmp.newFolder()) - journal.append(entry("late").copy(timestamp = 2000)) - journal.append(entry("early").copy(timestamp = 1000)) - assertEquals(listOf("early", "late"), journal.unacknowledged().map { it.eventId }) + journal.append(record(id1, at = 2_000)) + journal.append(record(id2, at = 1_000)) + assertEquals(listOf(id2, id1), journal.unacknowledged().map { it.eventId }) } @Test - fun `append does not throw when the directory is unwritable`() { - // A regular file where a directory is expected: mkdirs() and every write fail. - val notADir = tmp.newFile() - val journal = EventJournal(notADir) - journal.append(entry("e1")) // must not throw - assertEquals(emptyList(), journal.unacknowledged().map { it.eventId }) + fun `append never throws, and says whether it wrote`() { + val journal = EventJournal(tmp.newFile()) // a file where the directory should be + assertFalse(journal.append(record(id1))) + assertEquals(emptyList(), journal.unacknowledged()) } @Test - fun `prunes the oldest entries beyond the cap`() { + fun `prunes the oldest records beyond the cap`() { val dir = tmp.newFolder() val journal = EventJournal(dir, maxEntries = 3) - // Stamp increasing mtimes so pruning order is deterministic. Each mtime is - // set before the next append, which is when pruning reads it. - journal.append(entry("e1")); File(dir, "e1.json").setLastModified(1000) - journal.append(entry("e2")); File(dir, "e2.json").setLastModified(2000) - journal.append(entry("e3")); File(dir, "e3.json").setLastModified(3000) - journal.append(entry("e4")) // 4th write trips the cap; oldest (e1) is dropped - - val ids = journal.unacknowledged().map { it.eventId } - assertEquals(3, ids.size) - assertFalse(ids.contains("e1")) - assertTrue(ids.contains("e4")) + val ids = (1..4).map { "00000000-0000-0000-0000-00000000000$it" } + ids.take(3).forEachIndexed { i, id -> + journal.append(record(id)) + File(dir, "$id.json").setLastModified(1_000L * (i + 1)) + } + journal.append(record(ids[3])) + val left = journal.unacknowledged().map { it.eventId } + assertEquals(3, left.size) + assertFalse(left.contains(ids[0])) + } + + @Test + fun `incrementDeliveries persists`() { + val dir = tmp.newFolder() + val journal = EventJournal(dir) + journal.append(record(id1)) + assertEquals(2, journal.incrementDeliveries(id1)!!.deliveries) + assertEquals(3, journal.incrementDeliveries(id1)!!.deliveries) + assertEquals(3, EventJournal(dir).find(id1)!!.deliveries) + assertNull(journal.incrementDeliveries(id2)) + } + + @Test + fun `forEntry filters by entry id`() { + val journal = EventJournal(tmp.newFolder()) + journal.append(record(id1, id = "a")) + journal.append(record(id2, id = "b")) + assertEquals(listOf(id2), journal.forEntry("b").map { it.eventId }) + } + + @Test + fun `the completed shape is SettledEvent`() { + val map = record(id1).toMap() + assertEquals( + listOf( + "eventId", "id", "key", "vars", "at", "attempts", "requestId", "deliveries", "state", + "bytesSent", "totalBytes", "url", "method", "kind", "response", + ), + map.keys.toList(), + ) + assertEquals(mapOf("n" to 1.0), map["vars"]) + assertEquals(mapOf("status" to 200.0, "headers" to mapOf(), "body" to "ok", "bodyTruncated" to false), map["response"]) + } + + @Test + fun `a chunked completion has a response with no status`() { + val map = record(id1).copy(response = null).toMap() + assertEquals(mapOf("bodyTruncated" to false), map["response"]) + } + + @Test + fun `the error shape nests errorKind, message, response, and partIndex`() { + val map = record(id1, kind = EventJournal.KIND_ERROR).copy( + partIndex = 2, + response = EventJournal.Response(404, null, "gone", false), + ).toMap() + assertEquals(2.0, map["partIndex"]) + assertEquals( + mapOf( + "errorKind" to "http", "message" to "HTTP 400", + "response" to mapOf("status" to 404.0, "body" to "gone", "bodyTruncated" to false), + "partIndex" to 2.0, + ), + map["error"], + ) + assertFalse(map.containsKey("response")) + } + + @Test + fun `the cancelled shape carries the reason`() { + val map = record(id1, kind = EventJournal.KIND_CANCELLED).toMap() + assertEquals("user", map["cancelReason"]) + assertFalse(map.containsKey("error")) + assertFalse(map.containsKey("response")) } } diff --git a/android/src/test/java/ai/openspace/backgroundupload/JsonBridgeTest.kt b/android/src/test/java/ai/openspace/backgroundupload/JsonBridgeTest.kt new file mode 100644 index 00000000..bb420821 --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/JsonBridgeTest.kt @@ -0,0 +1,81 @@ +package ai.openspace.backgroundupload + +import com.facebook.react.bridge.JavaOnlyArray +import com.facebook.react.bridge.JavaOnlyMap +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Assert.assertThrows +import org.junit.Test + +class JsonBridgeTest { + + @Test + fun `integral doubles print as integers, as JSON stringify does`() { + assertEquals("""{"n":1,"neg":-3,"zero":0}""", JsonBridge.toJson(mapOf("n" to 1.0, "neg" to -3.0, "zero" to -0.0))) + assertEquals("12345678901", JsonBridge.toJson(12_345_678_901.0)) + } + + @Test + fun `fractions and very large magnitudes keep a decimal form`() { + assertEquals("1.5", JsonBridge.toJson(1.5)) + assertEquals("0.1", JsonBridge.toJson(0.1)) + // Above 2^53 a double can not hold every integer, so it stays a double. + assertEquals(1e20, (JsonBridge.parse(JsonBridge.toJson(1e20)) as Double), 0.0) + } + + @Test + fun `nested maps and lists round trip`() { + val value = mapOf("a" to listOf(1.0, "x", true, null, mapOf("b" to 2.5)), "c" to mapOf()) + val text = JsonBridge.toJson(value) + assertEquals("""{"a":[1,"x",true,null,{"b":2.5}],"c":{}}""", text) + assertEquals(value, JsonBridge.parse(text)) + } + + @Test + fun `keys are sorted so the same object always gives the same text`() { + assertEquals(JsonBridge.toJson(mapOf("b" to 1.0, "a" to 2.0)), JsonBridge.toJson(mapOf("a" to 2.0, "b" to 1.0))) + } + + @Test + fun `null is the text null, and HTML characters are not escaped`() { + assertEquals("null", JsonBridge.toJson(null)) + assertNull(JsonBridge.parse("null")) + assertEquals("\"\"", JsonBridge.toJson("")) + } + + @Test + fun `malformed text throws`() { + assertThrows(Exception::class.java) { JsonBridge.parse("{\"a\":") } + assertThrows(Exception::class.java) { JsonBridge.parse("[1,") } + } + + @Test + fun `bridge maps read into plain values`() { + val map = JavaOnlyMap.of( + "n", 2.0, + "s", "x", + "b", false, + "z", null, + "m", JavaOnlyMap.of("k", 1.0), + "a", JavaOnlyArray.of(1.0, "y"), + ) + assertEquals( + mapOf("n" to 2.0, "s" to "x", "b" to false, "z" to null, "m" to mapOf("k" to 1.0), "a" to listOf(1.0, "y")), + JsonBridge.fromReadable(map), + ) + assertNull(JsonBridge.valueOf(map, "absent")) + } + + @Test + fun `plain values write to the bridge`() { + val out = JsonBridge.toWritableMap( + mapOf("n" to 1.0, "list" to listOf("a", 2.0), "nested" to mapOf("k" to true), "none" to null), + ::JavaOnlyMap, + ::JavaOnlyArray, + ) as JavaOnlyMap + assertEquals(1.0, out.getDouble("n"), 0.0) + assertEquals("a", out.getArray("list")!!.getString(0)) + assertEquals(true, out.getMap("nested")!!.getBoolean("k")) + assertEquals(true, out.isNull("none")) + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/LegacyImportTest.kt b/android/src/test/java/ai/openspace/backgroundupload/LegacyImportTest.kt new file mode 100644 index 00000000..9f7be8ef --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/LegacyImportTest.kt @@ -0,0 +1,82 @@ +package ai.openspace.backgroundupload + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import java.io.File + +class LegacyImportTest { + @get:Rule + val tmp = TemporaryFolder() + + private fun v9(dir: File, eventId: String, uploadId: String, type: String, timestamp: Long) = + File(dir, "$eventId.json").writeText( + """{"eventId":"$eventId","uploadId":"$uploadId","type":"$type","timestamp":$timestamp,"responseCode":200}""", + ) + + @Test + fun `each v9 id becomes one legacy row with its newest outcome`() { + val v9Dir = tmp.newFolder() + v9(v9Dir, "a", "up-1", "completed", 100) + v9(v9Dir, "b", "up-2", "error", 200) + v9(v9Dir, "c", "up-3", "cancelled", 300) + v9(v9Dir, "d", "up-2", "completed", 250) // newer for up-2 + File(v9Dir, "bad.json").writeText("{not json") + val store = QueueStore(tmp.newFolder(), RequestIndex()) + + assertTrue(LegacyImport.import(v9Dir, store)) + + val rows = store.all().associateBy { it.id } + assertEquals(setOf("up-1", "up-2", "up-3"), rows.keys) + assertEquals(EntryState.COMPLETED, rows["up-1"]!!.state) + assertEquals(EntryState.COMPLETED, rows["up-2"]!!.state) + assertEquals(EntryState.CANCELLED, rows["up-3"]!!.state) + val row = rows["up-2"]!! + assertEquals("legacy", row.key) + assertEquals("null", row.varsJson) + assertTrue(row.legacy) + assertEquals(0, row.attempts) + assertEquals(250, row.updatedAt) + assertNull(row.descriptor) + assertEquals(0, v9Dir.list()!!.size) // every v9 file is gone + } + + @Test + fun `a second run imports nothing`() { + val v9Dir = tmp.newFolder() + val store = QueueStore(tmp.newFolder(), RequestIndex()) + v9(v9Dir, "a", "up-1", "completed", 100) + LegacyImport.import(v9Dir, store) + store.remove("up-1") + assertTrue(LegacyImport.import(v9Dir, store)) + assertEquals(emptyList(), store.all()) + } + + @Test + fun `an id that a v10 entry owns is left alone`() { + val v9Dir = tmp.newFolder() + val store = QueueStore(tmp.newFolder(), RequestIndex()) + store.save(entry(id = "up-1")) + v9(v9Dir, "a", "up-1", "error", 100) + LegacyImport.import(v9Dir, store) + assertEquals("note", store.load("up-1")!!.key) + } + + @Test + fun `a failed save keeps the v9 file and reports incomplete`() { + val v9Dir = tmp.newFolder() + v9(v9Dir, "a", "up-1", "error", 100) + val broken = QueueStore(tmp.newFile(), RequestIndex()) // a file where the directory should be + assertEquals(false, LegacyImport.import(v9Dir, broken)) + assertTrue(File(v9Dir, "a.json").exists()) + } + + @Test + fun `an unknown type makes no row`() { + assertNull(LegacyImport.legacyRow(LegacyImport.V9Entry("a", "up", "progress", 1))) + assertNull(LegacyImport.legacyRow(LegacyImport.V9Entry("a", null, "completed", 1))) + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/QueueControllerTest.kt b/android/src/test/java/ai/openspace/backgroundupload/QueueControllerTest.kt new file mode 100644 index 00000000..92663412 --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/QueueControllerTest.kt @@ -0,0 +1,617 @@ +package ai.openspace.backgroundupload + +import org.junit.Assert.assertArrayEquals +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNotNull +import org.junit.Assert.assertNull +import org.junit.Assert.assertThrows +import org.junit.Assert.assertTrue +import org.junit.Before +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import java.io.File + +class QueueControllerTest { + @get:Rule + val tmp = TemporaryFolder() + + private lateinit var root: File + private lateinit var store: QueueStore + private lateinit var journal: EventJournal + private lateinit var settings: QueueSettingsStore + private val events = RecordingEvents() + private val scheduler = FakeScheduler() + private val running = mutableSetOf() + private var now = 10_000L + private lateinit var controller: QueueController + + @Before + fun setUp() { + root = tmp.newFolder("queue") + store = QueueStore(root, RequestIndex()) + journal = EventJournal(tmp.newFolder("journal")) + settings = QueueSettingsStore(File(root, "settings.json")) + controller = QueueController(store, journal, settings, events, scheduler, { it in running }, { now }) + } + + private fun source(name: String, size: Int) = File(tmp.newFolder(), name).apply { writeBytes(ByteArray(size) { it.toByte() }) } + + private fun dirFiles(id: String = "e1") = store.entryDir(id).list()!!.toSet() + + // MARK: - enqueue + + @Test + fun `create stages the json body, persists, schedules, then emits`() { + assertEquals("e1", controller.enqueue(parsed())) + val e = store.load("e1")!! + assertEquals(EntryState.QUEUED, e.state) + assertEquals(1, e.generation) + assertEquals("""{"a":1}""", File(store.entryDir("e1"), e.body!!.fileName!!).readText()) + assertEquals("application/json", e.descriptor!!.headers["Content-Type"]) + assertEquals(setOf("entry.json", "body-1.json"), dirFiles()) + assertEquals(listOf("e1"), scheduler.scheduled) + assertEquals(listOf("state:e1:queued"), events.log) + } + + @Test + fun `create while paused is paused and not scheduled`() { + controller.pause() + controller.enqueue(parsed()) + assertEquals(EntryState.PAUSED, store.load("e1")!!.state) + assertEquals(emptyList(), scheduler.scheduled) + } + + @Test + fun `a file body is copied, so the caller may delete its source`() { + val src = source("photo.jpg", 100) + controller.enqueue(parsed(descriptor = desc(file = src.path))) + src.delete() + val e = store.load("e1")!! + assertEquals(100, File(store.entryDir("e1"), e.body!!.fileName!!).length()) + assertEquals(100, e.totalBytes) + } + + @Test + fun `a missing file rejects E_FILE_MISSING and persists nothing`() { + val e = assertThrows(QueueException::class.java) { controller.enqueue(parsed(descriptor = desc(file = "/nope.bin"))) } + assertEquals(QueueException.E_FILE_MISSING, e.code) + assertNull(store.load("e1")) + assertEquals(emptyList(), events.log) + } + + @Test + fun `same body on a queued entry resumes with the new headers, vars, and expiry`() { + controller.enqueue(parsed()) + val bodyFile = store.load("e1")!!.body!!.fileName + controller.enqueue(parsed(descriptor = desc(dataJson = """{"a":1}""", headers = mapOf("Authorization" to "Bearer new")), varsJson = """{"n":2}""", expiresAt = 5)) + val e = store.load("e1")!! + assertEquals("Bearer new", e.descriptor!!.headers["Authorization"]) + assertEquals("""{"n":2}""", e.varsJson) + assertEquals(5, e.expiresAt) + assertEquals(1, e.generation) + assertEquals(bodyFile, e.body!!.fileName) + } + + @Test + fun `same body on a running entry stays running and is not scheduled again`() { + store.save(entry(state = EntryState.RUNNING)) + controller.enqueue(parsed(descriptor = desc(dataJson = """{"a":1}""", headers = mapOf("Authorization" to "Bearer new")))) + val e = store.load("e1")!! + assertEquals(EntryState.RUNNING, e.state) + assertEquals("Bearer new", e.descriptor!!.headers["Authorization"]) + assertEquals(emptyList(), scheduler.scheduled) + } + + @Test + fun `a different body on a running entry rejects E_RUNNING and changes nothing`() { + store.save(entry(state = EntryState.RUNNING)) + val e = assertThrows(QueueException::class.java) { controller.enqueue(parsed(descriptor = desc(dataJson = """{"a":2}"""))) } + assertEquals(QueueException.E_RUNNING, e.code) + assertEquals(entry(state = EntryState.RUNNING), store.load("e1")) + } + + @Test + fun `a different body on a queued entry with a worker sleeping out a short backoff is accepted`() { + // The gate is held, but the entry is queued. The contract: queued is not running. + controller.enqueue(parsed()) + running += "e1" + controller.enqueue(parsed(descriptor = desc(dataJson = """{"a":2}"""))) + assertEquals("""{"a":2}""", store.load("e1")!!.descriptor!!.dataJson) + } + + @Test + fun `a different body on an error entry replaces it and reopens it`() { + controller.enqueue(parsed()) + store.save(store.load("e1")!!.copy(state = EntryState.ERROR, attempts = 3, settledEventId = "ev")) + controller.enqueue(parsed(descriptor = desc(dataJson = """{"a":2}"""))) + val e = store.load("e1")!! + assertEquals(EntryState.QUEUED, e.state) + assertEquals(2, e.generation) + assertEquals(0, e.attempts) + assertNull(e.settledEventId) + assertEquals("""{"a":2}""", File(store.entryDir("e1"), "body-2.json").readText()) + assertEquals(setOf("entry.json", "body-2.json"), dirFiles()) // the old body is pruned + } + + @Test + fun `same body on a completed unacked entry re-emits with one more delivery and does not re-run`() { + controller.enqueue(parsed()) + val eventId = "00000000-0000-0000-0000-00000000000a" + journal.append(record(eventId)) + store.save(store.load("e1")!!.copy(state = EntryState.COMPLETED, settledEventId = eventId)) + scheduler.scheduled.clear() + events.log.clear() + controller.enqueue(parsed()) + assertEquals(listOf("settled:e1:completed"), events.log) + assertEquals(2, events.records.single().deliveries) + assertEquals(2, journal.find(eventId)!!.deliveries) + assertEquals(EntryState.COMPLETED, store.load("e1")!!.state) + assertEquals(emptyList(), scheduler.scheduled) + } + + @Test + fun `same body on a cancelled unacked entry gets a fresh generation, and the old ack forgets nothing`() { + controller.enqueue(parsed()) + controller.cancel("e1") + val cancelled = journal.unacknowledged().single() + controller.enqueue(parsed()) + val e = store.load("e1")!! + assertEquals(EntryState.QUEUED, e.state) + assertEquals(2, e.generation) + controller.ack(listOf(cancelled.eventId)) + assertNotNull(store.load("e1")) + } + + @Test + fun `same body on an error entry reopens it and keeps attempts`() { + controller.enqueue(parsed()) + store.save(store.load("e1")!!.copy(state = EntryState.ERROR, attempts = 3, settledEventId = "ev")) + controller.enqueue(parsed(expiresAt = FAR_FUTURE + 1)) + val e = store.load("e1")!! + assertEquals(EntryState.QUEUED, e.state) + assertEquals(2, e.generation) + assertEquals(3, e.attempts) + } + + @Test + fun `enqueue over a legacy row replaces it`() { + store.save(LegacyImport.legacyRow(LegacyImport.V9Entry("x", "e1", "completed", 5))!!) + controller.enqueue(parsed()) + val e = store.load("e1")!! + assertFalse(e.legacy) + assertEquals(2, e.generation) + assertEquals(EntryState.QUEUED, e.state) + } + + // MARK: - chunked replace (a present file wins over the old blob) + + private fun chunkedErrorEntry(): File { + val first = source("first.bin", 20) + controller.enqueue(parsed(descriptor = desc(url = null, method = "PUT", file = first.path, parts = listOf(part(0, 10), part(10, 20))))) + store.save(store.load("e1")!!.copy(state = EntryState.ERROR, settledEventId = "ev")) + return File(store.entryDir("e1"), "blob") + } + + @Test + fun `a different-parts replace with a present file uploads that file, not the old blob`() { + chunkedErrorEntry() + val bytes = ByteArray(20) { (it * 3).toByte() } + val second = File(tmp.newFolder(), "second.bin").apply { writeBytes(bytes) } + controller.enqueue(parsed(descriptor = desc(url = null, method = "PUT", file = second.path, parts = listOf(part(0, 20))))) + val e = store.load("e1")!! + assertEquals(2, e.generation) + assertEquals("blob-2", e.body!!.fileName) + assertArrayEquals(bytes, store.bodyFile(e)!!.readBytes()) + assertEquals(setOf("entry.json", "blob-2"), dirFiles()) // the old blob is pruned + } + + @Test + fun `a different-parts replace with a present file of another size is accepted`() { + chunkedErrorEntry() + val second = source("second.bin", 30) + controller.enqueue(parsed(descriptor = desc(url = null, method = "PUT", file = second.path, parts = listOf(part(0, 30))))) + assertEquals(30, store.load("e1")!!.totalBytes) + } + + @Test + fun `a different-parts replace whose file was moved away runs over the old blob`() { + chunkedErrorEntry() + controller.enqueue(parsed(descriptor = desc(url = null, method = "PUT", file = "/moved.bin", parts = listOf(part(0, 20))))) + val e = store.load("e1")!! + assertEquals("blob", e.body!!.fileName) + assertEquals(setOf("entry.json", "blob"), dirFiles()) + } + + @Test + fun `a replace whose plan does not tile the present file changes nothing`() { + val oldBlob = chunkedErrorEntry() + val second = source("second.bin", 30) + val e = assertThrows(QueueException::class.java) { + controller.enqueue(parsed(descriptor = desc(url = null, method = "PUT", file = second.path, parts = listOf(part(0, 20))))) + } + assertEquals(QueueException.E_INVALID, e.code) + assertTrue(second.exists()) + assertEquals(20, oldBlob.length()) + assertEquals(1, store.load("e1")!!.generation) + } + + // MARK: - staging outside the lock + + @Test + fun `a new file body is copied outside the store lock`() { + val src = source("big.bin", 1000) + var hookRan = false + controller.afterPreStage = { + hookRan = true + assertFalse(Thread.holdsLock(store)) + assertTrue(File(store.entryDir("e1"), "file-1").exists()) + } + controller.enqueue(parsed(descriptor = desc(file = src.path))) + assertTrue(hookRan) + assertEquals("file-1", store.load("e1")!!.body!!.fileName) + assertEquals(setOf("entry.json", "file-1"), dirFiles()) + } + + @Test + fun `a replace over a queued entry stages under the lock`() { + controller.enqueue(parsed()) + var hookRan = false + controller.afterPreStage = { hookRan = true } + controller.enqueue(parsed(descriptor = desc(dataJson = """{"a":2}"""))) + assertFalse(hookRan) + assertEquals(setOf("entry.json", "body-2.json"), dirFiles()) + } + + @Test + fun `a pre-staged body that no longer fits is deleted and staged again under the lock`() { + controller.enqueue(parsed()) + store.save(store.load("e1")!!.copy(state = EntryState.ERROR)) + // Between the staging and the lock, the entry moves on to another generation. + controller.afterPreStage = { store.save(store.load("e1")!!.copy(generation = 5)) } + controller.enqueue(parsed(descriptor = desc(dataJson = """{"a":2}"""))) + val e = store.load("e1")!! + assertEquals(6, e.generation) + assertEquals("body-6.json", e.body!!.fileName) + assertEquals(setOf("entry.json", "body-6.json"), dirFiles()) // body-2.json is gone + } + + @Test + fun `a pre-staged body is deleted when the enqueue rejects`() { + controller.enqueue(parsed()) + store.save(store.load("e1")!!.copy(state = EntryState.ERROR)) + controller.afterPreStage = { store.save(store.load("e1")!!.copy(state = EntryState.RUNNING)) } + val e = assertThrows(QueueException::class.java) { controller.enqueue(parsed(descriptor = desc(dataJson = """{"a":2}"""))) } + assertEquals(QueueException.E_RUNNING, e.code) + assertEquals(setOf("entry.json", "body-1.json"), dirFiles()) + } + + // MARK: - v9 adoption + + private fun v9Dir(id: String, parts: String, blobSize: Int): File { + val dir = store.entryDir(id).apply { mkdirs() } + File(dir, "blob").writeBytes(ByteArray(blobSize)) + File(dir, "manifest.json").writeText( + """{"id":"$id","sourcePath":"${File(dir, "blob").path}","parts":$parts,"accept":[],"expiresAt":1,"wifiOnly":false,"noNotification":false,"createdAt":1}""", + ) + return dir + } + + private val v9Parts = """[{"url":"https://example.com/part?start=0","headers":{},"start":0,"end":10,"accepted":true},""" + + """{"url":"https://example.com/part?start=10","headers":{},"start":10,"end":20,"accepted":false}]""" + + @Test + fun `a same-id enqueue with the same parts adopts the v9 blob and accepted parts`() { + v9Dir("e1", v9Parts, 20) + controller.enqueue(parsed(descriptor = desc(url = null, method = "PUT", file = "/gone.bin", parts = listOf(part(0, 10), part(10, 20))))) + val e = store.load("e1")!! + assertTrue(e.descriptor!!.parts!![0].accepted) + assertFalse(e.descriptor!!.parts!![1].accepted) + assertEquals(10, e.bytesSent) + assertEquals(setOf("entry.json", "blob"), dirFiles()) + } + + @Test + fun `a v9 adoption with the same parts keeps the v9 blob even when the caller's file is present`() { + v9Dir("e1", v9Parts, 20) + val present = source("again.bin", 20) + controller.enqueue(parsed(descriptor = desc(url = null, method = "PUT", file = present.path, parts = listOf(part(0, 10), part(10, 20))))) + assertTrue(present.exists()) + assertTrue(store.load("e1")!!.descriptor!!.parts!![0].accepted) + assertArrayEquals(ByteArray(20), File(store.entryDir("e1"), "blob").readBytes()) + } + + @Test + fun `a v9 adoption with different parts and a present file uploads that file`() { + v9Dir("e1", v9Parts, 20) + val present = source("new.bin", 30) + controller.enqueue(parsed(descriptor = desc(url = null, method = "PUT", file = present.path, parts = listOf(part(0, 30))))) + assertFalse(present.exists()) + val e = store.load("e1")!! + assertFalse(e.descriptor!!.parts!![0].accepted) + assertEquals(30, store.bodyFile(e)!!.length()) + assertEquals(setOf("entry.json", "blob"), dirFiles()) + } + + @Test + fun `a v9 adoption with parts that do not tile rejects E_INVALID and keeps the v9 files`() { + v9Dir("e1", v9Parts, 20) + val e = assertThrows(QueueException::class.java) { + controller.enqueue(parsed(descriptor = desc(url = null, method = "PUT", file = "/gone.bin", parts = listOf(part(0, 30))))) + } + assertEquals(QueueException.E_INVALID, e.code) + assertEquals(setOf("manifest.json", "blob"), dirFiles()) + } + + // MARK: - cancel + + @Test + fun `cancel of a live entry journals, settles cancelled, stops work, emits, and forgets after ack`() { + controller.enqueue(parsed()) + events.log.clear() + controller.cancel("e1") + val record = journal.unacknowledged().single() + assertEquals(EventJournal.KIND_CANCELLED, record.kind) + assertEquals("user", record.cancelReason) + assertEquals("https://example.com/items", record.url) + val e = store.load("e1")!! + assertEquals(EntryState.CANCELLED, e.state) + assertEquals(record.eventId, e.settledEventId) + assertEquals(listOf("e1"), scheduler.cancelled) + assertEquals(listOf("settled:e1:cancelled", "state:e1:cancelled"), events.log) + controller.ack(listOf(record.eventId)) + assertNull(store.load("e1")) + assertFalse(store.entryDir("e1").exists()) + } + + @Test + fun `cancel of a settled entry forgets it now and keeps its unacked record`() { + controller.enqueue(parsed()) + val eventId = "00000000-0000-0000-0000-00000000000b" + journal.append(record(eventId, kind = EventJournal.KIND_ERROR)) + store.save(store.load("e1")!!.copy(state = EntryState.ERROR, settledEventId = eventId)) + events.log.clear() + controller.cancel("e1") + assertNull(store.load("e1")) + assertFalse(store.entryDir("e1").exists()) + assertEquals(emptyList(), events.log) + assertNotNull(journal.find(eventId)) + } + + @Test + fun `cancel of an unknown id is a no-op`() { + controller.cancel("nope") + assertEquals(emptyList(), events.log) + } + + // MARK: - pause, resume, wifi, headers + + @Test + fun `pause moves live rows to paused with no outcome, and resume brings them back`() { + store.save(entry(id = "q", state = EntryState.QUEUED)) + store.save(entry(id = "r", state = EntryState.RUNNING)) + store.save(entry(id = "a", state = EntryState.AWAITING_AUTH, parkedGeneration = 0)) + store.save(entry(id = "s", state = EntryState.AWAITING_AUTH, parkedGeneration = 0)) + store.save(entry(id = "x", state = EntryState.ERROR)) + controller.pause() + assertTrue(settings.load().paused) + assertEquals(listOf("paused", "paused", "paused", "paused", "error"), listOf("q", "r", "a", "s", "x").map { store.load(it)!!.state.wire }) + assertEquals(setOf("q", "r", "a", "s"), scheduler.cancelled.toSet()) + assertEquals(emptyList(), journal.unacknowledged()) + + // A header change while paused makes one parked entry's generation stale. + store.save(store.load("s")!!.copy(parkedGeneration = -1)) + controller.resume() + assertFalse(settings.load().paused) + assertEquals(EntryState.QUEUED, store.load("q")!!.state) + assertEquals(EntryState.QUEUED, store.load("r")!!.state) + assertEquals(EntryState.AWAITING_AUTH, store.load("a")!!.state) + assertEquals(EntryState.QUEUED, store.load("s")!!.state) + assertTrue(scheduler.scheduled.containsAll(listOf("q", "r", "s"))) + assertEquals(listOf("a" to FAR_FUTURE), scheduler.wakes) // the parked one waits for its expiry + } + + @Test + fun `setWifiOnly persists`() { + controller.setWifiOnly(true) + assertTrue(QueueSettingsStore(File(root, "settings.json")).load().wifiOnly) + } + + @Test + fun `updateHeaders patches every entry, bumps the generation, and requeues the parked`() { + store.save(entry(id = "p", state = EntryState.AWAITING_AUTH, parkedGeneration = 0)) + store.save(entry(id = "x", state = EntryState.ERROR)) + store.save(entry(id = "c", state = EntryState.QUEUED, descriptor = desc(url = null, file = "/f", + parts = listOf(Part("https://p/1", mapOf("authorization" to "stale"), 0, 10))))) + controller.updateHeaders(mapOf("Authorization" to "Bearer new")) + assertEquals(1, settings.load().headerGeneration) + val p = store.load("p")!! + assertEquals(EntryState.QUEUED, p.state) + assertNull(p.parkedGeneration) + assertEquals("Bearer new", p.descriptor!!.headers["Authorization"]) + assertEquals(1, p.headerGeneration) + assertEquals("Bearer new", store.load("x")!!.descriptor!!.headers["Authorization"]) + assertEquals(mapOf("Authorization" to "Bearer new"), store.load("c")!!.descriptor!!.parts!![0].headers) + assertEquals(listOf("p"), scheduler.scheduled) + assertEquals(listOf("state:p:queued"), events.log) + } + + // MARK: - ack and replay + + @Test + fun `ack forgets a completed entry of the current generation only`() { + val current = "00000000-0000-0000-0000-00000000000c" + val old = "00000000-0000-0000-0000-00000000000d" + store.save(entry(state = EntryState.COMPLETED, settledEventId = current, generation = 2)) + journal.append(record(old, generation = 1)) + journal.append(record(current, generation = 2)) + controller.ack(listOf(old, "unknown", "../../x")) + assertNotNull(store.load("e1")) + controller.ack(listOf(current)) + assertNull(store.load("e1")) + assertEquals(listOf("e1"), scheduler.cancelled) + controller.ack(listOf(current)) // idempotent + } + + @Test + fun `ack of an error removes the record and keeps the row`() { + val id = "00000000-0000-0000-0000-00000000000e" + store.save(entry(state = EntryState.ERROR, settledEventId = id)) + journal.append(record(id, kind = EventJournal.KIND_ERROR)) + controller.ack(listOf(id)) + assertNull(journal.find(id)) + assertNotNull(store.load("e1")) + } + + // A settle journaled and emitted its record, but the store write failed: + // the entry is still live at the record's generation. + + @Test + fun `ack of a completed record on a still-running entry settles and forgets it, so the sweep does not re-run it`() { + val id = "00000000-0000-0000-0000-000000000011" + store.save(entry(state = EntryState.RUNNING)) + journal.append(record(id)) + controller.ack(listOf(id)) + assertNull(store.load("e1")) + assertNull(journal.find(id)) + controller.sweep() + assertEquals(emptyList(), scheduler.scheduled) + } + + @Test + fun `ack of an error record on a still-running entry settles it as error and keeps the row`() { + val id = "00000000-0000-0000-0000-000000000012" + store.save(entry(state = EntryState.RUNNING)) + journal.append(record(id, kind = EventJournal.KIND_ERROR)) + controller.ack(listOf(id)) + val e = store.load("e1")!! + assertEquals(EntryState.ERROR, e.state) + assertEquals(id, e.settledEventId) + assertNull(journal.find(id)) + assertEquals(listOf("state:e1:error"), events.log) + controller.sweep() + assertEquals(emptyList(), scheduler.scheduled) + } + + @Test + fun `ack of a record of an older generation does not settle the live entry`() { + val id = "00000000-0000-0000-0000-000000000013" + store.save(entry(state = EntryState.QUEUED, generation = 2)) + journal.append(record(id, generation = 1)) + controller.ack(listOf(id)) + assertEquals(EntryState.QUEUED, store.load("e1")!!.state) + assertNull(journal.find(id)) + } + + @Test + fun `when the ack repair can not save, the record stays for the sweep`() { + val id = "00000000-0000-0000-0000-000000000014" + store.save(entry(state = EntryState.RUNNING)) + journal.append(record(id)) + val dir = store.entryDir("e1") + dir.setWritable(false) + try { + controller.ack(listOf(id)) + } finally { + dir.setWritable(true) + } + assertNotNull(journal.find(id)) + assertEquals(EntryState.RUNNING, store.load("e1")!!.state) + controller.sweep() + assertEquals(EntryState.COMPLETED, store.load("e1")!!.state) + } + + @Test + fun `each replay counts one more delivery`() { + journal.append(record("00000000-0000-0000-0000-00000000000f")) + assertEquals(2, controller.unacknowledged().single().deliveries) + assertEquals(3, controller.unacknowledged().single().deliveries) + } + + // MARK: - boot sweep + + @Test + fun `sweep applies a record that the store transition missed`() { + val id = "00000000-0000-0000-0000-000000000010" + store.save(entry(state = EntryState.RUNNING)) + journal.append(record(id)) + controller.sweep() + val e = store.load("e1")!! + assertEquals(EntryState.COMPLETED, e.state) + assertEquals(id, e.settledEventId) + assertEquals(listOf("state:e1:completed"), events.log) + } + + @Test + fun `sweep applies a cancel whose store save was lost`() { + val id = "00000000-0000-0000-0000-000000000011" + store.save(entry(state = EntryState.QUEUED)) + journal.append(record(id, kind = EventJournal.KIND_CANCELLED)) + controller.sweep() + assertEquals(EntryState.CANCELLED, store.load("e1")!!.state) + assertEquals(emptyList(), scheduler.scheduled) + } + + @Test + fun `sweep queues a running entry with no worker and schedules queued work`() { + store.save(entry(id = "r", state = EntryState.RUNNING)) + store.save(entry(id = "q", state = EntryState.QUEUED)) + store.save(entry(id = "w", state = EntryState.QUEUED, nextAttemptAt = now + 3_600_000)) + store.save(entry(id = "p", state = EntryState.AWAITING_AUTH, expiresAt = now + 50)) + controller.sweep() + assertEquals(EntryState.QUEUED, store.load("r")!!.state) + assertEquals(setOf("r", "q"), scheduler.scheduled.toSet()) + assertEquals(setOf("w" to now + 3_600_000, "p" to now + 50), scheduler.wakes.toSet()) + } + + @Test + fun `sweep skips an entry whose worker runs in this process`() { + store.save(entry(state = EntryState.RUNNING)) + journal.append(record("00000000-0000-0000-0000-000000000012")) + running += "e1" + controller.sweep() + assertEquals(EntryState.RUNNING, store.load("e1")!!.state) + } + + @Test + fun `sweep forgets a completed entry whose record was acked, and acks orphans`() { + store.save(entry(id = "done", state = EntryState.COMPLETED, settledEventId = "gone")) + val own = "00000000-0000-0000-0000-000000000013" + val orphan = "00000000-0000-0000-0000-000000000014" + store.save(entry(id = "c", state = EntryState.CANCELLED, settledEventId = own)) + journal.append(record(own, id = "c", kind = EventJournal.KIND_CANCELLED)) + journal.append(record(orphan, id = "c")) + controller.sweep() + assertNull(store.load("done")) + assertEquals(listOf(own), journal.unacknowledged().map { it.eventId }) + assertNotNull(store.load("c")) + } + + @Test + fun `sweep leaves paused entries alone`() { + controller.pause() + store.save(entry(state = EntryState.PAUSED)) + controller.sweep() + assertEquals(EntryState.PAUSED, store.load("e1")!!.state) + assertEquals(emptyList(), scheduler.scheduled) + } + + @Test + fun `sweep finishes a pause or resume that a process death cut short`() { + // resume() saved the setting, then died before the rows. + store.save(entry(id = "p", state = EntryState.PAUSED)) + controller.sweep() + assertEquals(EntryState.QUEUED, store.load("p")!!.state) + assertEquals(listOf("p"), scheduler.scheduled) + + // pause() saved the setting, then died before the rows. + settings.update { it.copy(paused = true) } + store.save(entry(id = "q", state = EntryState.QUEUED)) + store.save(entry(id = "r", state = EntryState.RUNNING)) + controller.sweep() + assertEquals(EntryState.PAUSED, store.load("q")!!.state) + assertEquals(EntryState.PAUSED, store.load("r")!!.state) + assertEquals(listOf("p"), scheduler.scheduled) + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/QueueEntryTest.kt b/android/src/test/java/ai/openspace/backgroundupload/QueueEntryTest.kt new file mode 100644 index 00000000..c0b286b3 --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/QueueEntryTest.kt @@ -0,0 +1,103 @@ +package ai.openspace.backgroundupload + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test + +class QueueEntryTest { + + private val form = listOf(FormPart("photo", "image/jpeg", null, "/p.jpg", null)) + + @Test + fun `sameBodyAs compares each body kind by content`() { + val json = entry(descriptor = desc(dataJson = "{\"a\":1}")) + assertTrue(json.sameBodyAs(desc(dataJson = "{\"a\":1}", headers = mapOf("New" to "h")))) + assertFalse(json.sameBodyAs(desc(dataJson = "{\"a\":2}"))) + + val multipart = entry(descriptor = desc(form = form)) + assertTrue(multipart.sameBodyAs(desc(form = form))) + assertFalse(multipart.sameBodyAs(desc(form = form.map { it.copy(name = "other") }))) + + val file = entry(descriptor = desc(file = "/a.bin")) + assertTrue(file.sameBodyAs(desc(file = "/a.bin"))) + assertFalse(file.sameBodyAs(desc(file = "/b.bin"))) + + val none = entry(descriptor = desc(method = "DELETE")) + assertTrue(none.sameBodyAs(desc(method = "DELETE"))) + } + + @Test + fun `chunked compares parts and ignores the path`() { + val parts = listOf(part(0, 100), part(100, 250)) + val chunked = entry(descriptor = desc(url = null, method = "PUT", file = "/moved.bin", parts = parts)) + assertTrue(chunked.sameBodyAs(desc(url = null, method = "PUT", file = "/elsewhere.bin", parts = parts.reversed()))) + assertFalse(chunked.sameBodyAs(desc(url = null, method = "PUT", file = "/moved.bin", parts = listOf(part(0, 250))))) + } + + @Test + fun `a kind change, url change, or method change is a different body`() { + val json = entry(descriptor = desc(dataJson = "{}")) + assertFalse(json.sameBodyAs(desc(form = form))) + assertFalse(json.sameBodyAs(desc(url = "https://example.com/other", dataJson = "{}"))) + assertFalse(json.sameBodyAs(desc(method = "PUT", dataJson = "{}"))) + assertFalse(entry(descriptor = null, legacy = true).sameBodyAs(desc(dataJson = "{}"))) + } + + @Test + fun `withHeadersPatched matches names in any case and keeps the patch spelling`() { + val e = entry(descriptor = desc(headers = mapOf("authorization" to "Bearer old", "X-Keep" to "1"))) + val patched = e.withHeadersPatched(mapOf("Authorization" to "Bearer new", "X-Add" to "2"), generation = 3) + assertEquals( + mapOf("X-Keep" to "1", "Authorization" to "Bearer new", "X-Add" to "2"), + patched.descriptor!!.headers, + ) + assertEquals(3, patched.headerGeneration) + } + + @Test + fun `withHeadersPatched replaces a part's own copy of a patched header only`() { + val parts = listOf( + Part("https://p/1", mapOf("AUTHORIZATION" to "Bearer stale", "Content-Range" to "0-99"), 0, 100), + Part("https://p/2", mapOf("Content-Range" to "100-249"), 100, 250), + ) + val e = entry(descriptor = desc(url = null, file = "/f", parts = parts)) + val patched = e.withHeadersPatched(mapOf("Authorization" to "Bearer new"), 1).descriptor!!.parts!! + assertEquals(mapOf("Content-Range" to "0-99", "Authorization" to "Bearer new"), patched[0].headers) + assertEquals(mapOf("Content-Range" to "100-249"), patched[1].headers) + } + + @Test + fun `toRow carries vars as an object and nextAttemptAt only when set`() { + val row = entry().toRow().toMap() + assertEquals(mapOf("n" to 1.0), row["vars"]) + assertEquals("queued", row["state"]) + assertFalse(row.containsKey("nextAttemptAt")) + assertEquals( + setOf("id", "key", "vars", "state", "bytesSent", "totalBytes", "attempts", "updatedAt"), + row.keys, + ) + val waiting = entry(nextAttemptAt = 9_000).toRow().toMap() + assertEquals(9_000.0, waiting["nextAttemptAt"]) + } + + @Test + fun `a malformed vars text reads as null in the row`() { + assertNull(entry().copy(varsJson = "{bad").toRow().vars) + } + + @Test + fun `isLive covers the four live states`() { + val live = EntryState.values().filter { it.isLive }.toSet() + assertEquals(setOf(EntryState.QUEUED, EntryState.RUNNING, EntryState.AWAITING_AUTH, EntryState.PAUSED), live) + } + + @Test + fun `header maps match names without regard to case`() { + assertTrue(HeaderMap.contains(mapOf("Content-Type" to "x"), "content-type")) + assertEquals("x", HeaderMap.get(mapOf("content-type" to "x"), "Content-Type")) + assertEquals(mapOf("B" to "2", "a" to "3"), HeaderMap.merge(mapOf("A" to "1", "B" to "2"), mapOf("a" to "3"))) + assertEquals(mapOf("B" to "2"), HeaderMap.without(mapOf("a" to "1", "B" to "2"), "A")) + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/QueueSettingsTest.kt b/android/src/test/java/ai/openspace/backgroundupload/QueueSettingsTest.kt new file mode 100644 index 00000000..6dbcfff7 --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/QueueSettingsTest.kt @@ -0,0 +1,68 @@ +package ai.openspace.backgroundupload + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import java.io.File + +class QueueSettingsTest { + @get:Rule + val tmp = TemporaryFolder() + + @Test + fun `defaults when there is no file`() { + val s = QueueSettingsStore(File(tmp.newFolder(), "settings.json")).load() + assertEquals(QueueSettings(), s) + assertEquals(RetryDefaults(1_000, 7_200_000, 0.2, listOf(404)), s.retry) + } + + @Test + fun `update persists and reloads in a new instance`() { + val file = File(tmp.newFolder(), "settings.json") + QueueSettingsStore(file).update { it.copy(wifiOnly = true, paused = true) } + val reloaded = QueueSettingsStore(file).load() + assertTrue(reloaded.wifiOnly) + assertTrue(reloaded.paused) + } + + @Test + fun `a corrupt file reads as the defaults`() { + val file = File(tmp.newFolder(), "settings.json").apply { writeText("{not json") } + assertEquals(QueueSettings(), QueueSettingsStore(file).load()) + } + + @Test + fun `a file missing fields keeps the defaults for them`() { + val file = File(tmp.newFolder(), "settings.json").apply { writeText("""{"wifiOnly":true}""") } + val s = QueueSettingsStore(file).load() + assertTrue(s.wifiOnly) + assertFalse(s.paused) + assertEquals(RetryDefaults(), s.retry) + } + + @Test + fun `header generation increments`() { + val store = QueueSettingsStore(File(tmp.newFolder(), "settings.json")) + assertEquals(1, store.update { it.copy(headerGeneration = it.headerGeneration + 1) }.headerGeneration) + assertEquals(2, store.update { it.copy(headerGeneration = it.headerGeneration + 1) }.headerGeneration) + } + + @Test + fun `a failed write keeps the old value`() { + // A regular file where the directory should be: the write fails. + val notADir = tmp.newFile() + val store = QueueSettingsStore(File(notADir, "settings.json")) + runCatching { store.update { it.copy(paused = true) } } + assertFalse(store.load().paused) + } + + @Test + fun `configure retry fills absent fields with the library defaults`() { + val d = QueueSettingsStore.retryDefaults(mapOf("backoff" to mapOf("baseMs" to 500.0), "terminalHttp" to mapOf("exempt" to listOf()))) + assertEquals(RetryDefaults(baseMs = 500, exempt = emptyList()), d) + assertEquals(RetryDefaults(), QueueSettingsStore.retryDefaults(null)) + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/QueueStoreTest.kt b/android/src/test/java/ai/openspace/backgroundupload/QueueStoreTest.kt new file mode 100644 index 00000000..a4ae7232 --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/QueueStoreTest.kt @@ -0,0 +1,245 @@ +package ai.openspace.backgroundupload + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNotNull +import org.junit.Assert.assertNull +import org.junit.Assert.assertThrows +import org.junit.Assert.assertTrue +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import java.io.File +import java.io.IOException +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit + +class QueueStoreTest { + @get:Rule + val tmp = TemporaryFolder() + + private fun store(dir: File = tmp.newFolder(), index: RequestIndex = RequestIndex()) = QueueStore(dir, index) + + @Test + fun `save then load round-trips across store instances`() { + val dir = tmp.newFolder() + val e = entry( + descriptor = desc(url = null, method = "PUT", file = "/f", parts = listOf(part(0, 10, accepted = true), part(10, 20))), + body = StagedBody(StagedBody.CHUNKED, "blob", null, 20), + nextAttemptAt = 5, + parkedGeneration = 2, + ) + store(dir).save(e) + // A new instance over the same dir is what a process relaunch looks like. + assertEquals(e, store(dir).load("e1")) + } + + @Test + fun `the state enum is stored by its wire string`() { + val s = store() + s.save(entry(state = EntryState.AWAITING_AUTH)) + assertTrue(File(s.entryDir("e1"), QueueStore.ENTRY_FILE).readText().contains("\"awaiting-auth\"")) + } + + @Test + fun `ids with filesystem-hostile characters round-trip`() { + val s = store() + val id = "a/b:c dü..\\e" + s.save(entry(id = id)) + assertEquals(id, s.load(id)!!.id) + s.remove(id) + assertNull(s.load(id)) + } + + @Test + fun `a corrupt entry reads as absent`() { + val s = store() + s.save(entry()) + File(s.entryDir("e1"), QueueStore.ENTRY_FILE).writeText("{not json") + assertNull(s.load("e1")) + assertEquals(emptyList(), s.all()) + } + + @Test + fun `an entry missing a required field reads as absent`() { + val s = store() + s.entryDir("e1").mkdirs() + File(s.entryDir("e1"), QueueStore.ENTRY_FILE).writeText("""{"id":"e1","key":"k","state":"queued"}""") + assertNull(s.load("e1")) // not legacy, and no descriptor or body + } + + @Test + fun `an older file gets safe defaults`() { + val s = store() + s.entryDir("e1").mkdirs() + File(s.entryDir("e1"), QueueStore.ENTRY_FILE).writeText( + """{"id":"e1","key":"k","state":"queued","descriptor":{"url":"https://x"},"body":{"kind":"none","totalBytes":0}}""", + ) + val e = s.load("e1")!! + assertEquals("null", e.varsJson) + assertEquals("POST", e.descriptor!!.method) + assertEquals(emptyMap(), e.descriptor!!.headers) + assertEquals(1, e.generation) + } + + @Test + fun `all skips v9-only directories and legacyManifest reads them`() { + val s = store() + val dir = s.entryDir("v9").apply { mkdirs() } + File(dir, QueueStore.V9_MANIFEST_FILE).writeText( + """{"id":"v9","sourcePath":"/x/blob","parts":[{"url":"https://p/1","headers":{},"start":0,"end":10,"accepted":true}],""" + + """"accept":[{"status":409}],"expiresAt":99,"wifiOnly":false,"noNotification":true,"createdAt":1}""", + ) + s.save(entry(id = "v10")) + assertEquals(listOf("v10"), s.all().map { it.id }) + val m = s.legacyManifest("v9")!! + assertEquals(listOf(Part("https://p/1", mapOf(), 0, 10, accepted = true)), m.parts) + assertEquals(listOf(UploadOutcome.AcceptRule(409)), m.accept) + assertNull(s.legacyManifest("v10")) + assertNull(s.legacyManifest("nope")) + } + + @Test + fun `compute holds the lock across load, transform, and save`() { + // A module transition and a worker's update race. If the lock did not + // span all three steps, the update could land between load and save and + // be erased. Serialized, both effects survive. + val s = store() + s.save(entry(descriptor = desc(url = null, file = "/f", parts = listOf(part(0, 10), part(10, 20))))) + val inTransform = CountDownLatch(1) + val computing = Thread { + s.compute("e1") { e -> + inTransform.countDown() + Thread.sleep(300) + e!!.copy(expiresAt = 99_000) + } + }.apply { start() } + assertTrue(inTransform.await(5, TimeUnit.SECONDS)) + val updating = Thread { + s.update("e1") { e -> e.copy(descriptor = e.descriptor!!.copy(parts = ChunkedParts.withAccepted(e.descriptor.parts!!, 0))) } + }.apply { start() } + computing.join() + updating.join() + val final = s.load("e1")!! + assertEquals(99_000, final.expiresAt) + assertTrue(final.descriptor!!.parts!![0].accepted) + } + + @Test + fun `a throwing transform writes nothing`() { + val s = store() + s.save(entry()) + assertThrows(IllegalStateException::class.java) { s.compute("e1") { throw IllegalStateException("no") } } + assertEquals(entry(), s.load("e1")) + } + + @Test + fun `compute returning the same object or null writes nothing`() { + val s = store() + assertNull(s.compute("e1") { null }) + assertFalse(s.entryDir("e1").exists() && File(s.entryDir("e1"), QueueStore.ENTRY_FILE).exists()) + s.save(entry()) + val file = File(s.entryDir("e1"), QueueStore.ENTRY_FILE) + file.setLastModified(1_000) + s.compute("e1") { it } + assertEquals(1_000, file.lastModified()) + } + + @Test + fun `remove deletes the row and every staged byte`() { + val index = RequestIndex() + val s = store(index = index) + s.save(entry()) + File(s.entryDir("e1"), "body-1.json").writeText("{}") + File(s.entryDir("e1"), "blob").writeText("bytes") + s.remove("e1") + assertNull(s.load("e1")) + assertFalse(s.entryDir("e1").exists()) + assertNull(index.get("e1")) + } + + @Test + fun `the index follows every save and remove, and loads on start`() { + val dir = tmp.newFolder() + val index = RequestIndex() + val s = store(dir, index) + s.save(entry(id = "a")) + s.save(entry(id = "b", state = EntryState.ERROR)) + assertEquals(listOf("a", "b"), index.snapshot().map { it.id }) + s.remove("a") + assertEquals(listOf("b"), index.snapshot().map { it.id }) + // A process relaunch: a fresh index loaded from disk. + val fresh = RequestIndex() + QueueStore(dir, fresh).loadIndex() + assertEquals("error", fresh.get("b")!!.state) + } + + @Test + fun `pruneUnreferenced keeps only the entry file and its body`() { + val s = store() + val e = entry(body = StagedBody(StagedBody.JSON, "body-2.json", null, 2)) + s.save(e) + val dir = s.entryDir("e1") + listOf("body-1.json", "body-2.json", "blob", "manifest.json", "entry.json.tmp").forEach { File(dir, it).writeText("x") } + s.pruneUnreferenced(e) + assertEquals(setOf("entry.json", "body-2.json"), dir.list()!!.toSet()) + } + + // MARK: - crash mid-write + + @Test + fun `crash mid-write (a) a partial entry tmp next to a valid entry`() { + val s = store() + s.save(entry(attempts = 1)) + // The process died while writing the next version: only the tmp is partial. + File(s.entryDir("e1"), "entry.json.tmp").writeText("""{"id":"e1","key":"no""") + assertEquals(1, s.load("e1")!!.attempts) + s.save(entry(attempts = 2)) + assertEquals(2, s.load("e1")!!.attempts) + assertFalse(File(s.entryDir("e1"), "entry.json.tmp").exists()) + } + + @Test + fun `crash mid-write (b) a staged body with no entry is not a row`() { + val s = store() + val dir = s.entryDir("e1").apply { mkdirs() } + File(dir, "body-1.json.tmp").writeText("{\"a\":") + File(dir, "body-1.json").writeText("{\"a\":1}") + assertEquals(emptyList(), s.all()) + assertNull(s.load("e1")) + // The next create saves an entry and prunes what it does not use. + val e = entry(body = StagedBody(StagedBody.JSON, "body-1.json", null, 7)) + s.save(e) + s.pruneUnreferenced(e) + assertEquals(setOf("entry.json", "body-1.json"), dir.list()!!.toSet()) + } + + @Test + fun `crash mid-write (c) a write that throws leaves the old target intact`() { + val target = File(tmp.newFolder(), "entry.json").apply { writeText("old") } + assertThrows(IOException::class.java) { + AtomicFiles.writeAtomically(target) { out -> + out.write("new, half".toByteArray()) + throw IOException("disk full") + } + } + assertEquals("old", target.readText()) + assertFalse(AtomicFiles.tmpFor(target).exists()) + } + + @Test + fun `a save into an unwritable directory throws`() { + val notADir = tmp.newFile() + val s = QueueStore(notADir, RequestIndex()) + assertThrows(IOException::class.java) { s.save(entry()) } + } + + @Test + fun `update is best effort`() { + val s = store() + assertNull(s.update("nope") { it }) + s.save(entry()) + assertNotNull(s.update("e1") { it.copy(attempts = 3) }) + assertEquals(3, s.load("e1")!!.attempts) + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/RetryClassifierTest.kt b/android/src/test/java/ai/openspace/backgroundupload/RetryClassifierTest.kt new file mode 100644 index 00000000..cb21bf72 --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/RetryClassifierTest.kt @@ -0,0 +1,102 @@ +package ai.openspace.backgroundupload + +import ai.openspace.backgroundupload.RetryClassifier.Verdict +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test +import java.io.IOException +import kotlin.random.Random + +class RetryClassifierTest { + private val defaultExempt = listOf(404) + + private fun classify(code: Int, body: String = "", accept: List = emptyList(), exempt: List = defaultExempt) = + RetryClassifier.classifyResponse(code, body, accept, exempt) + + @Test + fun `the retry table`() { + assertEquals(Verdict.Accepted, classify(200)) + assertEquals(Verdict.Accepted, classify(204)) + assertEquals(Verdict.Accepted, classify(409, "upload already completed", listOf(UploadOutcome.AcceptRule(409, "already completed")))) + assertEquals(Verdict.Terminal("http", "HTTP 409"), classify(409, "conflict", listOf(UploadOutcome.AcceptRule(409, "already completed")))) + assertEquals(Verdict.Auth, classify(401)) + assertEquals(Verdict.Auth, classify(403)) + assertEquals(Verdict.Transient, classify(408)) + assertEquals(Verdict.Transient, classify(429)) + assertEquals(Verdict.Transient, classify(500)) + assertEquals(Verdict.Transient, classify(599)) + assertEquals(Verdict.Transient, classify(404)) // default exempt + assertEquals(Verdict.Terminal("http", "HTTP 400"), classify(400)) + assertEquals(Verdict.Terminal("http", "HTTP 409"), classify(409)) + assertEquals(Verdict.Terminal("http", "HTTP 304"), classify(304)) + assertEquals(Verdict.Terminal("http", "HTTP 101"), classify(101)) + } + + @Test + fun `a chunked part 404 with exempt empty is terminal`() { + assertEquals(Verdict.Terminal("http", "HTTP 404"), classify(404, exempt = emptyList())) + } + + @Test + fun `an auth status stays auth even when exempt lists it`() { + assertEquals(Verdict.Auth, classify(401, exempt = listOf(401))) + } + + @Test + fun `transport failures`() { + assertEquals(Verdict.Transient, RetryClassifier.classifyFailure(IOException("reset"), fileExists = true)) + val file = RetryClassifier.classifyFailure(IOException("ENOENT"), fileExists = false) + assertTrue(file is Verdict.Terminal && file.errorKind == "file") + val other = RetryClassifier.classifyFailure(IllegalArgumentException("bad url"), fileExists = true) + assertEquals(Verdict.Terminal("unknown", "bad url"), other) + assertEquals("network", RetryClassifier.failureKind(IOException(), true)) + } + + private val policy = RetryClassifier.Policy(baseMs = 1_000, maxMs = 7_200_000, jitter = 0.0, exempt = defaultExempt) + + @Test + fun `backoff doubles from base and caps at max`() { + assertEquals(1_000, RetryClassifier.backoffMs(policy, 1)) + assertEquals(2_000, RetryClassifier.backoffMs(policy, 2)) + assertEquals(4_000, RetryClassifier.backoffMs(policy, 3)) + assertEquals(7_200_000, RetryClassifier.backoffMs(policy, 14)) + assertEquals(7_200_000, RetryClassifier.backoffMs(policy, 10_000)) + assertEquals(1_000, RetryClassifier.backoffMs(policy, 0)) // defensive + } + + @Test + fun `jitter stays within bounds and under max`() { + val jittered = policy.copy(jitter = 0.2) + val random = Random(42) + repeat(1_000) { + val ms = RetryClassifier.backoffMs(jittered, 3, random) + assertTrue("$ms", ms in 3_200..4_800) + } + repeat(1_000) { + assertTrue(RetryClassifier.backoffMs(jittered, 30, random) <= 7_200_000) + } + } + + @Test + fun `nextAttemptAt clamps to expiresAt`() { + assertEquals(3_000, RetryClassifier.nextAttemptAt(now = 1_000, backoffMs = 2_000, expiresAt = 10_000)) + assertEquals(10_000, RetryClassifier.nextAttemptAt(now = 1_000, backoffMs = 7_200_000, expiresAt = 10_000)) + } + + @Test + fun `expiry is inclusive of the deadline`() { + assertFalse(RetryClassifier.isExpired(4_999, 5_000)) + assertTrue(RetryClassifier.isExpired(5_000, 5_000)) + } + + @Test + fun `policy overrides field by field`() { + val defaults = RetryDefaults() + assertEquals(RetryClassifier.Policy(1_000, 7_200_000, 0.2, listOf(404)), RetryClassifier.policy(defaults, null)) + assertEquals( + RetryClassifier.Policy(50, 7_200_000, 0.2, emptyList()), + RetryClassifier.policy(defaults, RetryOverride(baseMs = 50, maxMs = null, jitter = null, exempt = emptyList())), + ) + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/SmallPartsTest.kt b/android/src/test/java/ai/openspace/backgroundupload/SmallPartsTest.kt new file mode 100644 index 00000000..f133bc9e --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/SmallPartsTest.kt @@ -0,0 +1,167 @@ +package ai.openspace.backgroundupload + +import androidx.work.WorkInfo.State.BLOCKED +import androidx.work.WorkInfo.State.CANCELLED +import androidx.work.WorkInfo.State.ENQUEUED +import androidx.work.WorkInfo.State.FAILED +import androidx.work.WorkInfo.State.RUNNING +import androidx.work.WorkInfo.State.SUCCEEDED +import kotlinx.coroutines.async +import kotlinx.coroutines.delay +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.sync.withPermit +import kotlinx.coroutines.withTimeoutOrNull +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test + +class AttemptEventTest { + private val response = UploadResponse(401, "x".repeat(5_000), mapOf("a" to "b")) + + @Test + fun `the body is cut at 4 KB and flagged`() { + val e = AttemptEvent.ofResponse(entry(attempts = 2), "r1", "https://x", null, response, accepted = false, at = 7) + assertEquals(AttemptEvent.MAX_BODY_CHARS, e.responseBody!!.length) + assertEquals(true, e.responseBodyTruncated) + assertEquals(2, e.attempt) + } + + @Test + fun `outcome is completed only when accepted`() { + val rejected = AttemptEvent.ofResponse(entry(), "r1", "https://x", null, response, accepted = false, at = 7) + assertEquals("error", rejected.outcome) + assertEquals(401, rejected.httpCode) + assertEquals("http", rejected.errorKind) + val ok = AttemptEvent.ofResponse(entry(), "r1", "https://x", 3, response.copy(code = 200, body = "ok"), accepted = true, at = 7) + assertEquals("completed", ok.outcome) + assertNull(ok.errorKind) + assertEquals(3.0, ok.toMap()["partIndex"]) + } + + @Test + fun `partIndex and response fields are optional in the map`() { + val failure = AttemptEvent.ofFailure(entry(), "r1", "https://x", null, "network", "reset", 7).toMap() + assertEquals( + setOf("id", "key", "requestId", "attempt", "url", "method", "outcome", "errorKind", "errorMessage", "at"), + failure.keys, + ) + } +} + +class ProgressThrottleTest { + private var now = 0L + private val emitted = mutableListOf() + private val throttle = ProgressThrottle({ now }) { _, sent, _ -> emitted += sent } + + @Test + fun `the first offer emits, then at most one per second in the foreground`() { + throttle.offer("a", 1, 10, foreground = true) + now = 500 + throttle.offer("a", 2, 10, foreground = true) + assertEquals(listOf(1L), emitted) + now = 1_000 + throttle.offer("a", 3, 10, foreground = true) + assertEquals(listOf(1L, 3L), emitted) + } + + @Test + fun `the background interval is 10 minutes`() { + throttle.offer("a", 1, 10, foreground = false) + now = 599_999 + throttle.offer("a", 2, 10, foreground = false) + assertEquals(listOf(1L), emitted) + now = 600_000 + throttle.offer("a", 3, 10, foreground = false) + assertEquals(listOf(1L, 3L), emitted) + } + + @Test + fun `flush sends the held value once`() { + throttle.offer("a", 1, 10, foreground = true) + throttle.offer("a", 2, 10, foreground = true) + throttle.flush("a") + throttle.flush("a") + assertEquals(listOf(1L, 2L), emitted) + } + + @Test + fun `ids are independent, and drop forgets an id`() { + throttle.offer("a", 1, 10, foreground = true) + throttle.offer("b", 5, 10, foreground = true) + assertEquals(listOf(1L, 5L), emitted) + throttle.offer("a", 2, 10, foreground = true) + throttle.drop("a") + throttle.flush("a") + assertEquals(listOf(1L, 5L), emitted) + } +} + +class RequestIndexTest { + @Test + fun `put, remove, and snapshot order`() { + val index = RequestIndex() + index.put(entry(id = "b", createdAt = 2).toRow()) + index.put(entry(id = "a", createdAt = 2).toRow()) + index.put(entry(id = "c", createdAt = 1).toRow()) + assertEquals(listOf("c", "a", "b"), index.snapshot().map { it.id }) + index.remove("a") + assertEquals(listOf("c", "b"), index.snapshot().map { it.id }) + } + + @Test + fun `setBytes on a missing id is a no-op`() { + val index = RequestIndex() + index.setBytes("nope", 5) + assertNull(index.get("nope")) + } + + @Test + fun `a save of a running entry does not move bytes backwards`() { + val index = RequestIndex() + val running = entry(state = EntryState.RUNNING, body = StagedBody(StagedBody.FILE, "f", null, 100)) + index.put(running.toRow()) + index.setBytes("e1", 60) + index.put(running.copy(attempts = 2).toRow()) + assertEquals(60, index.get("e1")!!.bytesSent) + index.put(running.copy(state = EntryState.QUEUED).toRow()) + assertEquals(0, index.get("e1")!!.bytesSent) + } +} + +class SchedulerTest { + @Test + fun `initialDelayMs is the time left, never negative`() { + assertEquals(0, WorkManagerScheduler.initialDelayMs(null, 1_000)) + assertEquals(0, WorkManagerScheduler.initialDelayMs(500, 1_000)) + assertEquals(4_000, WorkManagerScheduler.initialDelayMs(5_000, 1_000)) + } + + @Test + fun `a queued successor suppresses another append`() { + assertTrue(WorkManagerScheduler.hasQueuedSuccessor(listOf(RUNNING, BLOCKED))) + assertTrue(WorkManagerScheduler.hasQueuedSuccessor(listOf(ENQUEUED))) + assertFalse(WorkManagerScheduler.hasQueuedSuccessor(listOf(RUNNING))) + assertFalse(WorkManagerScheduler.hasQueuedSuccessor(listOf(SUCCEEDED, FAILED, CANCELLED))) + assertFalse(WorkManagerScheduler.hasQueuedSuccessor(emptyList())) + } + + @Test + fun `the wake name differs from the main chain`() { + assertEquals("e1#wake", WorkManagerScheduler.wakeName("e1")) + } +} + +class TransferSemaphoreTest { + @Test + fun `the global cap is 4 and a fifth request waits`() = runBlocking { + assertEquals(4, MAX_TRANSFER_CONCURRENCY) + val holders = (1..4).map { async { transferSemaphore.withPermit { delay(200) } } } + delay(20) + val fifth = withTimeoutOrNull(50) { transferSemaphore.withPermit { } } + assertNull(fifth) + holders.forEach { it.await() } + assertEquals(Unit, withTimeoutOrNull(500) { transferSemaphore.withPermit { } }) + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/TestSupport.kt b/android/src/test/java/ai/openspace/backgroundupload/TestSupport.kt new file mode 100644 index 00000000..f8e405f5 --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/TestSupport.kt @@ -0,0 +1,123 @@ +package ai.openspace.backgroundupload + +// Builders and fakes shared by the JVM tests. No android.* here. + +internal const val FAR_FUTURE = 4_000_000_000_000L + +internal fun desc( + url: String? = "https://example.com/items", + method: String = "POST", + headers: Map = mapOf("Authorization" to "Bearer old"), + dataJson: String? = null, + form: List? = null, + file: String? = null, + parts: List? = null, + accept: List = emptyList(), + retry: RetryOverride? = null, + noNotification: Boolean = false, +) = Descriptor(url, method, headers, dataJson, form, file, parts, accept, retry, noNotification) + +internal fun part(start: Long, end: Long, accepted: Boolean = false, url: String? = null) = Part( + url = url ?: "https://example.com/part?start=$start", + headers = mapOf("Content-Range" to "$start-${end - 1}"), + start = start, + end = end, + accepted = accepted, +) + +internal fun entry( + id: String = "e1", + state: EntryState = EntryState.QUEUED, + descriptor: Descriptor? = desc(dataJson = """{"a":1}"""), + body: StagedBody? = StagedBody(StagedBody.JSON, "body-1.json", null, 7), + generation: Int = 1, + attempts: Int = 0, + settledEventId: String? = null, + parkedGeneration: Int? = null, + nextAttemptAt: Long? = null, + expiresAt: Long = FAR_FUTURE, + legacy: Boolean = false, + key: String = "note", + createdAt: Long = 1_000, +) = QueueEntry( + id = id, + key = key, + varsJson = """{"n":1}""", + descriptor = descriptor, + body = body, + state = state, + attempts = attempts, + bytesSent = 0, + totalBytes = body?.totalBytes ?: 0, + expiresAt = expiresAt, + createdAt = createdAt, + updatedAt = createdAt, + nextAttemptAt = nextAttemptAt, + parkedGeneration = parkedGeneration, + generation = generation, + settledEventId = settledEventId, + legacy = legacy, +) + +internal fun parsed( + id: String = "e1", + descriptor: Descriptor = desc(dataJson = """{"a":1}"""), + varsJson: String = """{"n":1}""", + expiresAt: Long = FAR_FUTURE, + key: String = "note", +) = EntryParsing.Parsed(id, key, varsJson, descriptor, expiresAt) + +internal fun record( + eventId: String, + id: String = "e1", + kind: String = EventJournal.KIND_COMPLETED, + generation: Int = 1, + at: Long = 5_000, + state: String = kind, +) = EventJournal.SettledRecord( + eventId = eventId, id = id, key = "note", varsJson = """{"n":1}""", at = at, attempts = 1, + requestId = "r1", deliveries = 1, state = state, bytesSent = 0, totalBytes = 0, + url = "https://example.com/items", method = "POST", partIndex = null, kind = kind, + response = if (kind == EventJournal.KIND_COMPLETED) EventJournal.Response(200, mapOf(), "ok", false) else null, + errorKind = if (kind == EventJournal.KIND_ERROR) "http" else null, + message = if (kind == EventJournal.KIND_ERROR) "HTTP 400" else null, + cancelReason = if (kind == EventJournal.KIND_CANCELLED) "user" else null, + generation = generation, +) + +/** Records every event in order, as "state::" and "settled::". */ +internal class RecordingEvents(var live: Boolean = true) : QueueEvents { + val log = mutableListOf() + val rows = mutableListOf() + val records = mutableListOf() + + override fun state(row: RequestRow) { + rows += row + log += "state:${row.id}:${row.state}" + } + + override fun settled(record: EventJournal.SettledRecord) { + records += record + log += "settled:${record.id}:${record.kind}" + } + + override fun canDeliver() = live +} + +internal class FakeScheduler : WorkScheduler { + val scheduled = mutableListOf() + val wakes = mutableListOf>() + val cancelled = mutableListOf() + + override fun schedule(entry: QueueEntry) { + scheduled += entry.id + } + + override fun scheduleWake(entry: QueueEntry, at: Long, replace: Boolean) { + wakes += entry.id to at + } + + override fun cancel(id: String) { + cancelled += id + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/UploadStatesTest.kt b/android/src/test/java/ai/openspace/backgroundupload/UploadStatesTest.kt deleted file mode 100644 index bef1858c..00000000 --- a/android/src/test/java/ai/openspace/backgroundupload/UploadStatesTest.kt +++ /dev/null @@ -1,84 +0,0 @@ -package ai.openspace.backgroundupload - -import androidx.work.WorkInfo.State.BLOCKED -import androidx.work.WorkInfo.State.CANCELLED -import androidx.work.WorkInfo.State.ENQUEUED -import androidx.work.WorkInfo.State.FAILED -import androidx.work.WorkInfo.State.RUNNING -import androidx.work.WorkInfo.State.SUCCEEDED -import androidx.work.ListenableWorker -import org.junit.Assert.assertEquals -import org.junit.Assert.assertFalse -import org.junit.Assert.assertTrue -import org.junit.Test - -// getAllUploads must return ONE row per upload id, although WorkManager can -// hold several rows for it (finished chains linger for roughly a day, and -// APPEND_OR_REPLACE resumes add rows). The state vocabulary is the same as -// iOS's. -class UploadStatesTest { - - @Test - fun `a live row wins for a chunked upload`() { - assertEquals("running", chunkedUploadState(listOf(CANCELLED, RUNNING), allAccepted = false)) - assertEquals("running", chunkedUploadState(listOf(RUNNING, BLOCKED), allAccepted = false)) - assertEquals("pending", chunkedUploadState(listOf(FAILED, ENQUEUED), allAccepted = false)) - assertEquals("pending", chunkedUploadState(listOf(BLOCKED), allAccepted = false)) - } - - @Test - fun `with no live row the manifest speaks, never a lingering finished row`() { - // A cancelled chunked upload keeps its manifest. Its truthful state is - // stalled-awaiting-resume ("error"), not "cancelled". iOS's getAllUploads - // never reports "cancelled" for a lingering upload. - assertEquals("error", chunkedUploadState(listOf(CANCELLED), allAccepted = false)) - assertEquals("error", chunkedUploadState(listOf(FAILED), allAccepted = false)) - assertEquals("error", chunkedUploadState(emptyList(), allAccepted = false)) - assertEquals("completed", chunkedUploadState(listOf(SUCCEEDED), allAccepted = true)) - assertEquals("completed", chunkedUploadState(emptyList(), allAccepted = true)) - // The row that a run leaves after it journals a terminal error is - // SUCCEEDED (see terminalErrorResult). The manifest, not the row, carries - // the outcome. - assertEquals("error", chunkedUploadState(listOf(SUCCEEDED), allAccepted = false)) - } - - @Test - fun `a journaled terminal error still succeeds the row`() { - // WorkManager marks the dependents of a FAILED prerequisite FAILED without - // a run. Thus a resume appended during a failing run's teardown would - // silently never run. The journal and the manifest are the outcome record, - // never the row state. - assertTrue(terminalErrorResult() is ListenableWorker.Result.Success) - } - - @Test - fun `cancel reports from the module only when no worker is running`() { - assertTrue(cancelReportsFromModule(listOf(ENQUEUED))) - // An appended chain's dependent is BLOCKED, not ENQUEUED. It is still - // never-started, and it is still owed a module-side 'cancelled'. - assertTrue(cancelReportsFromModule(listOf(BLOCKED))) - assertTrue(cancelReportsFromModule(listOf(ENQUEUED, BLOCKED))) - // A RUNNING worker's stop handler owns the report. - assertFalse(cancelReportsFromModule(listOf(RUNNING))) - assertFalse(cancelReportsFromModule(listOf(RUNNING, BLOCKED))) - assertFalse(cancelReportsFromModule(emptyList())) - } - - @Test - fun `a queued successor suppresses another append`() { - assertTrue(hasQueuedSuccessor(listOf(RUNNING, BLOCKED))) - assertTrue(hasQueuedSuccessor(listOf(ENQUEUED))) - assertFalse(hasQueuedSuccessor(listOf(RUNNING))) - assertFalse(hasQueuedSuccessor(listOf(SUCCEEDED, FAILED, CANCELLED))) - assertFalse(hasQueuedSuccessor(emptyList())) - } - - @Test - fun `a simple upload reports its live row first, then the most conclusive finished one`() { - assertEquals("running", simpleUploadState(listOf(CANCELLED, RUNNING))) - assertEquals("pending", simpleUploadState(listOf(SUCCEEDED, ENQUEUED))) - assertEquals("completed", simpleUploadState(listOf(CANCELLED, SUCCEEDED))) - assertEquals("error", simpleUploadState(listOf(CANCELLED, FAILED))) - assertEquals("cancelled", simpleUploadState(listOf(CANCELLED))) - } -} diff --git a/android/src/test/java/ai/openspace/backgroundupload/UploadTest.kt b/android/src/test/java/ai/openspace/backgroundupload/UploadTest.kt deleted file mode 100644 index a99178d1..00000000 --- a/android/src/test/java/ai/openspace/backgroundupload/UploadTest.kt +++ /dev/null @@ -1,112 +0,0 @@ -package ai.openspace.backgroundupload - -import com.google.gson.Gson -import org.junit.Assert.assertEquals -import org.junit.Assert.assertFalse -import org.junit.Assert.assertTrue -import org.junit.Test - -class UploadTest { - private val gson = Gson() - - private fun upload(noNotification: Boolean) = Upload( - id = "u1", - url = "https://example.com/upload", - path = "/tmp/file", - method = "POST", - wifiOnly = false, - accept = listOf(), - headers = mapOf(), - noNotification = noNotification, - ) - - @Test - fun `an upload notifies unless it opts out`() { - assertTrue(upload(noNotification = false).showsNotification) - assertFalse(upload(noNotification = true).showsNotification) - } - - @Test - fun `the opt-out survives a serialization round trip`() { - val json = gson.toJson(upload(noNotification = true)) - assertFalse(gson.fromJson(json, Upload::class.java).showsNotification) - } - - // WorkManager stores this model as JSON, so an upload can be enqueued by one - // build and run by the next. A job from a build without the option must keep - // its notification rather than silently losing foreground mode. - @Test - fun `a job enqueued without the option still notifies`() { - val json = gson.toJsonTree(upload(noNotification = true)).asJsonObject - json.remove(Upload::noNotification.name) - assertTrue(gson.fromJson(json, Upload::class.java).showsNotification) - } - - // One build can enqueue a WorkManager job, and the next build can replay it. - // This is the exact JSON shape that a v8 build serialized into input data - // (Gson.toJson of the v8 Upload model): `acceptStatus: List`, and no - // `accept`. Gson does not use the constructor. Thus, without normalized(), - // the replayed object's `accept` is NULL, and the worker NPEs after the file - // has fully transmitted. WorkManager then re-runs it and re-sends the whole - // file. - private val v8JobJson = """ - { - "id": "u1", - "url": "https://example.com/upload", - "path": "/tmp/file", - "method": "PUT", - "maxRetries": 5, - "wifiOnly": false, - "acceptStatus": [409, 208], - "headers": {"Authorization": "Bearer t"}, - "notificationId": 123456, - "notificationTitle": "Uploading…", - "notificationTitleNoInternet": "Waiting for connection…", - "notificationTitleNoWifi": "Waiting for Wi-Fi…", - "notificationChannel": "background-upload", - "noNotification": false - } - """ - - @Test - fun `a replayed v8 job maps acceptStatus to accept rules and is safe to run`() { - val replayed = gson.fromJson(v8JobJson, Upload::class.java).normalized() - assertEquals( - listOf(UploadOutcome.AcceptRule(409), UploadOutcome.AcceptRule(208)), - replayed.accept, - ) - // The worker-facing calls that NPE'd on the un-normalized object. - assertTrue(UploadOutcome.isAccepted(409, "duplicate", replayed.accept)) - assertFalse(UploadOutcome.isAccepted(400, "", replayed.accept)) - assertEquals("u1", replayed.id) - assertEquals(mapOf("Authorization" to "Bearer t"), replayed.headers) - assertTrue(replayed.showsNotification) - } - - @Test - fun `a replayed v8 job with an empty acceptStatus gets no rules`() { - val json = gson.fromJson(v8JobJson, com.google.gson.JsonObject::class.java) - json.add("acceptStatus", com.google.gson.JsonArray()) - val replayed = gson.fromJson(json, Upload::class.java).normalized() - assertEquals(emptyList(), replayed.accept) - assertTrue(UploadOutcome.isAccepted(200, "", replayed.accept)) - } - - @Test - fun `a job with neither accept nor acceptStatus normalizes to no rules`() { - val json = gson.fromJson(v8JobJson, com.google.gson.JsonObject::class.java) - json.remove("acceptStatus") - val replayed = gson.fromJson(json, Upload::class.java).normalized() - assertEquals(emptyList(), replayed.accept) - assertFalse(UploadOutcome.isAccepted(409, "duplicate", replayed.accept)) - } - - @Test - fun `normalized passes a current-shape job through unchanged`() { - val current = upload(noNotification = true).copy( - accept = listOf(UploadOutcome.AcceptRule(409, "already completed")), - ) - val replayed = gson.fromJson(gson.toJson(current), Upload::class.java).normalized() - assertEquals(current, replayed) - } -} diff --git a/android/src/test/java/ai/openspace/backgroundupload/WorkerGateTest.kt b/android/src/test/java/ai/openspace/backgroundupload/WorkerGateTest.kt new file mode 100644 index 00000000..dde4a6b0 --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/WorkerGateTest.kt @@ -0,0 +1,51 @@ +package ai.openspace.backgroundupload + +import org.junit.After +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test + +class WorkerGateTest { + private val a = Any() + private val b = Any() + + @After + fun tearDown() { + // A process-wide singleton. Leave nothing for other tests. + listOf("g1", "g2").forEach { id -> WorkerGate.release(id, a); WorkerGate.release(id, b) } + } + + @Test + fun `a second worker for the same id must wait`() { + assertTrue(WorkerGate.tryAcquire("g1", a)) + assertFalse(WorkerGate.tryAcquire("g1", b)) + WorkerGate.release("g1", a) + assertTrue(WorkerGate.tryAcquire("g1", b)) + } + + @Test + fun `reacquiring with the same token is idempotent`() { + assertTrue(WorkerGate.tryAcquire("g1", a)) + assertTrue(WorkerGate.tryAcquire("g1", a)) + } + + @Test + fun `a stale release can not evict a successor`() { + assertTrue(WorkerGate.tryAcquire("g1", a)) + WorkerGate.release("g1", a) + assertTrue(WorkerGate.tryAcquire("g1", b)) + WorkerGate.release("g1", a) + assertTrue(WorkerGate.isRunning("g1")) + assertFalse(WorkerGate.tryAcquire("g1", a)) + } + + @Test + fun `ids are independent`() { + assertTrue(WorkerGate.tryAcquire("g1", a)) + assertFalse(WorkerGate.isRunning("g2")) + assertTrue(WorkerGate.tryAcquire("g2", a)) + WorkerGate.release("g1", a) + assertFalse(WorkerGate.isRunning("g1")) + assertTrue(WorkerGate.isRunning("g2")) + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/WorkerOpsTest.kt b/android/src/test/java/ai/openspace/backgroundupload/WorkerOpsTest.kt new file mode 100644 index 00000000..0e7e1df9 --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/WorkerOpsTest.kt @@ -0,0 +1,283 @@ +package ai.openspace.backgroundupload + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertThrows +import org.junit.Assert.assertTrue +import org.junit.Before +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import java.io.File + +class WorkerOpsTest { + @get:Rule + val tmp = TemporaryFolder() + + private lateinit var store: QueueStore + private lateinit var journal: EventJournal + private lateinit var settings: QueueSettingsStore + private val events = RecordingEvents() + private val scheduler = FakeScheduler() + private var now = 20_000L + private lateinit var ops: WorkerOps + private lateinit var controller: QueueController + + @Before + fun setUp() { + val root = tmp.newFolder("queue") + store = QueueStore(root, RequestIndex()) + journal = EventJournal(tmp.newFolder("journal")) + settings = QueueSettingsStore(File(root, "settings.json")) + ops = WorkerOps(store, journal, settings, events, scheduler) { now } + controller = QueueController(store, journal, settings, events, scheduler, { false }, { now }) + } + + private val ok = UploadResponse(200, """{"id":7}""", mapOf("x" to "y")) + + @Test + fun `begin takes a queued entry and emits running`() { + store.save(entry(nextAttemptAt = 5)) + val e = ops.begin("e1")!! + assertEquals(EntryState.RUNNING, e.state) + assertNull(e.nextAttemptAt) + assertEquals(listOf("state:e1:running"), events.log) + } + + @Test + fun `begin does nothing for a paused, settled, or legacy entry`() { + store.save(entry(id = "p", state = EntryState.PAUSED)) + store.save(entry(id = "x", state = EntryState.ERROR)) + store.save(entry(id = "l", state = EntryState.QUEUED, legacy = true, descriptor = null, body = null)) + assertNull(ops.begin("p")) + assertNull(ops.begin("x")) + assertNull(ops.begin("l")) + assertNull(ops.begin("nope")) + } + + @Test + fun `recordAttempt persists attempts and the request id before the send`() { + store.save(entry(state = EntryState.RUNNING, attempts = 2)) + val e = ops.recordAttempt("e1", 1, "req-3") + assertEquals(3, e.attempts) + assertEquals("req-3", store.load("e1")!!.lastRequestId) + assertEquals(3, store.load("e1")!!.attempts) + } + + @Test + fun `recordAttempt refuses once the module took the entry`() { + store.save(entry(state = EntryState.PAUSED)) + assertThrows(NotOwnedException::class.java) { ops.recordAttempt("e1", 1, "r") } + store.save(entry(state = EntryState.RUNNING, generation = 2)) + assertThrows(NotOwnedException::class.java) { ops.recordAttempt("e1", 1, "r") } + } + + @Test + fun `settle journals, transitions, then emits settled before state`() { + store.save(entry(state = EntryState.RUNNING, attempts = 1).copy(lastRequestId = "req-1")) + assertTrue(ops.settle("e1", 1, Settlement.Completed(ok, "https://example.com/items", "POST"))) + val record = journal.unacknowledged().single() + assertEquals(1, record.deliveries) + assertEquals("req-1", record.requestId) + assertEquals(200, record.response!!.status) + assertEquals(1, record.generation) + val e = store.load("e1")!! + assertEquals(EntryState.COMPLETED, e.state) + assertEquals(record.eventId, e.settledEventId) + assertEquals(e.totalBytes, e.bytesSent) + assertEquals(listOf("settled:e1:completed", "state:e1:completed"), events.log) + } + + @Test + fun `a settle with JS dead starts at 0 deliveries, so the first replay is 1`() { + events.live = false + store.save(entry(state = EntryState.RUNNING)) + assertTrue(ops.settle("e1", 1, Settlement.Completed(ok, "u", "POST"))) + assertEquals(0, journal.unacknowledged().single().deliveries) + assertEquals(listOf("state:e1:completed"), events.log) // nothing to emit to + assertEquals(1, controller.unacknowledged().single().deliveries) + } + + @Test + fun `a response that lands after cancel drops its record`() { + controller.enqueue(parsed()) + ops.begin("e1") + controller.cancel("e1") + events.log.clear() + assertFalse(ops.settle("e1", 1, Settlement.Completed(ok, "u", "POST"))) + val left = journal.unacknowledged().single() + assertEquals(EventJournal.KIND_CANCELLED, left.kind) + assertEquals(EntryState.CANCELLED, store.load("e1")!!.state) + assertEquals(emptyList(), events.log) + } + + @Test + fun `a settle from an older generation is dropped`() { + store.save(entry(state = EntryState.QUEUED, generation = 2)) + assertFalse(ops.settle("e1", 1, Settlement.Completed(ok, "u", "POST"))) + assertEquals(emptyList(), journal.unacknowledged()) + } + + @Test + fun `a response that lands during pause still settles`() { + store.save(entry(state = EntryState.PAUSED)) + assertTrue(ops.settle("e1", 1, Settlement.Failed("http", "HTTP 400", ok.copy(code = 400), null, "u", "POST"))) + val e = store.load("e1")!! + assertEquals(EntryState.ERROR, e.state) + assertEquals("http", journal.unacknowledged().single().errorKind) + } + + @Test + fun `park sets awaiting-auth once and wakes at expiry`() { + store.save(entry(state = EntryState.RUNNING, expiresAt = 90_000)) + assertEquals(WorkerOps.ParkResult.PARKED, ops.park("e1", 1, headerGeneration = 0)) + val e = store.load("e1")!! + assertEquals(EntryState.AWAITING_AUTH, e.state) + assertEquals(0, e.parkedGeneration) + assertEquals(listOf("state:e1:awaiting-auth"), events.log) + assertEquals(listOf("e1" to 90_000L), scheduler.wakes) + assertEquals(emptyList(), journal.unacknowledged()) + } + + @Test + fun `a 401 sent under older headers re-issues instead of parking`() { + store.save(entry(state = EntryState.RUNNING)) + controller.updateHeaders(mapOf("Authorization" to "Bearer new")) + assertEquals(WorkerOps.ParkResult.REISSUE, ops.park("e1", 1, headerGeneration = 0)) + assertEquals(EntryState.RUNNING, store.load("e1")!!.state) + } + + @Test + fun `updateHeaders after a park requeues it`() { + store.save(entry(state = EntryState.RUNNING)) + ops.park("e1", 1, 0) + controller.updateHeaders(mapOf("Authorization" to "Bearer new")) + assertEquals(EntryState.QUEUED, store.load("e1")!!.state) + assertEquals(listOf("e1"), scheduler.scheduled) + } + + @Test + fun `release queues the entry with its wake time and streak`() { + store.save(entry(state = EntryState.RUNNING)) + assertTrue(ops.release("e1", 1, nextAttemptAt = 80_000, streak = 7)) + val e = store.load("e1")!! + assertEquals(EntryState.QUEUED, e.state) + assertEquals(80_000L, e.nextAttemptAt) + assertEquals(7, e.backoffStreak) + assertEquals(listOf("e1" to 80_000L), scheduler.wakes) + assertEquals(80_000.0, events.rows.single().toMap()["nextAttemptAt"]) + } + + @Test + fun `a system stop queues a running entry and leaves a paused one`() { + store.save(entry(state = EntryState.RUNNING)) + ops.stopped("e1", 1) + assertEquals(EntryState.QUEUED, store.load("e1")!!.state) + store.save(entry(state = EntryState.PAUSED)) + ops.stopped("e1", 1) + assertEquals(EntryState.PAUSED, store.load("e1")!!.state) + assertEquals(emptyList(), journal.unacknowledged()) + } + + @Test + fun `markAccepted persists the flag and the accepted bytes`() { + store.save(entry( + state = EntryState.RUNNING, + descriptor = desc(url = null, file = "/f", parts = listOf(part(0, 10), part(10, 25))), + body = StagedBody(StagedBody.CHUNKED, "blob", null, 25), + ).copy(backoffStreak = 3)) + ops.markAccepted("e1", 1, 1) + val e = store.load("e1")!! + assertTrue(e.descriptor!!.parts!![1].accepted) + assertEquals(15, e.bytesSent) + assertEquals(0, e.backoffStreak) + } + + // MARK: - header generation of an attempt + + @Test + fun `an attempt carries the header generation of the headers it sends`() { + store.save(entry(state = EntryState.RUNNING)) + controller.updateHeaders(mapOf("Authorization" to "Bearer new")) + val e = ops.recordAttempt("e1", 1, "r1") + assertEquals(1, e.headerGeneration) + assertEquals("Bearer new", e.descriptor!!.headers["Authorization"]) + } + + @Test + fun `a 401 between the settings bump and the entry patch parks, and the patch requeues it`() { + store.save(entry(state = EntryState.RUNNING)) + // updateHeaders() has bumped the settings but not yet patched the entry. + settings.update { it.copy(headerGeneration = it.headerGeneration + 1) } + val sent = ops.recordAttempt("e1", 1, "r1") + assertEquals(0, sent.headerGeneration) + assertFalse(ops.hasNewerHeaders("e1", 1, sent.headerGeneration)) // no re-issue with the same old headers + assertEquals(WorkerOps.ParkResult.PARKED, ops.park("e1", 1, sent.headerGeneration)) + // The rest of updateHeaders() finds it parked. + controller.updateHeaders(mapOf("Authorization" to "Bearer new")) + val e = store.load("e1")!! + assertEquals(EntryState.QUEUED, e.state) + assertEquals("Bearer new", e.descriptor!!.headers["Authorization"]) + } + + @Test + fun `a re-issue sends the patched headers, and a 401 on them parks with them`() { + store.save(entry(state = EntryState.RUNNING)) + val first = ops.recordAttempt("e1", 1, "r1") + controller.updateHeaders(mapOf("Authorization" to "Bearer new")) + assertTrue(ops.hasNewerHeaders("e1", 1, first.headerGeneration)) + val second = ops.recordAttempt("e1", 1, "r2") + assertEquals("Bearer new", second.descriptor!!.headers["Authorization"]) + assertFalse(ops.hasNewerHeaders("e1", 1, second.headerGeneration)) + assertEquals(WorkerOps.ParkResult.PARKED, ops.park("e1", 1, second.headerGeneration)) + assertEquals(1, store.load("e1")!!.parkedGeneration) + } + + // MARK: - short backoff + + @Test + fun `a short backoff shows nextAttemptAt on the running row until the next attempt`() { + store.save(entry(state = EntryState.RUNNING)) + ops.backingOff("e1", 1, nextAttemptAt = 24_000) + val waiting = store.load("e1")!! + assertEquals(EntryState.RUNNING, waiting.state) + assertEquals(24_000L, waiting.nextAttemptAt) + assertEquals(24_000.0, events.rows.last().toMap()["nextAttemptAt"]) + ops.recordAttempt("e1", 1, "r2") + assertNull(store.load("e1")!!.nextAttemptAt) + assertEquals(listOf("state:e1:running", "state:e1:running"), events.log) + assertFalse(events.rows.last().toMap().containsKey("nextAttemptAt")) + } + + @Test + fun `an attempt with no pending backoff emits no state event`() { + store.save(entry(state = EntryState.RUNNING)) + ops.recordAttempt("e1", 1, "r1") + assertEquals(emptyList(), events.log) + } + + @Test + fun `a short backoff on an entry the run no longer owns changes nothing`() { + store.save(entry(state = EntryState.PAUSED)) + ops.backingOff("e1", 1, nextAttemptAt = 24_000) + assertNull(store.load("e1")!!.nextAttemptAt) + assertEquals(emptyList(), events.log) + } + + // MARK: - deliveries + + @Test + fun `JS that subscribes between the check and the append still gets the outcome live with deliveries 1`() { + // canDeliver() is false at the record build and true right after the append. + val answers = ArrayDeque(listOf(false, true)) + val flipping = object : QueueEvents by events { + override fun canDeliver() = answers.removeFirstOrNull() ?: true + } + val flipOps = WorkerOps(store, journal, settings, flipping, scheduler) { now } + store.save(entry(state = EntryState.RUNNING)) + assertTrue(flipOps.settle("e1", 1, Settlement.Completed(ok, "u", "POST"))) + assertEquals(1, journal.unacknowledged().single().deliveries) + assertEquals(1, events.records.single().deliveries) + } +} From 49370494f2ad548be97b74ad22964c337ba2725e Mon Sep 17 00:00:00 2001 From: Dylan Murphy Date: Thu, 24 Sep 2026 16:23:31 -0400 Subject: [PATCH 2/3] Android: apply the native slices review Reliability: begin() applies an own-generation journal record instead of re-running, so a crash between the journal write and the store update cannot double-send. A failed journal write on settle holds the record in memory and retries; nothing is acked that was not written. cancel() stops the work in a finally block, journals before it saves, and applies an existing record of its generation rather than adding a second outcome. Listening state and the deliveries count are decided under one journal lock, and a live settle goes to the listener that drained, not the newest module instance. Contract: vars and data arrive as JSON text; "null" is a body; GET with a body rejects E_INVALID. Attempt events are completed or error only; pause, cancel, and supersede emit none. A settled entry's cancel forgets its unacked outcomes. attempts reset only on reopen. Under pause only an accepted response settles. A same-id enqueue over a legacy row adopts its v9 manifest. The response body cap applies while streaming. A prune never deletes an eventId a row or a new record names. Rows carry live bytesSent. Platform: a headless run stopped at JobScheduler's 10-minute limit takes the transient path with backoff. Header validation messages carry the name and offset, never the value. README documents the headless limit and the allowBackup requirement. Simplification: one EntryRun attempt path shared by simple and chunked transfers; classifyFailure is the only failure rule; dead code removed. Tests: 289 (was 217), with a TransferHost seam for the run loop. Co-Authored-By: Claude Fable 5.1 --- README.md | 15 + .../backgroundupload/AttemptEvent.kt | 15 +- .../ai/openspace/backgroundupload/BodyCap.kt | 67 +++ .../backgroundupload/ChunkedUploadWorker.kt | 103 ++-- .../backgroundupload/EnqueueRules.kt | 36 +- .../backgroundupload/EntryParsing.kt | 55 ++- .../ai/openspace/backgroundupload/EntryRun.kt | 325 ++++++++++++ .../backgroundupload/EntryTransitions.kt | 34 +- .../openspace/backgroundupload/EntryWorker.kt | 236 ++------- .../backgroundupload/EventJournal.kt | 206 ++++++-- .../backgroundupload/EventReporter.kt | 21 +- .../openspace/backgroundupload/JsonBridge.kt | 60 +-- .../backgroundupload/LegacyImport.kt | 9 +- .../backgroundupload/QueueController.kt | 112 +++-- .../openspace/backgroundupload/QueueStore.kt | 20 +- .../backgroundupload/RequestIndex.kt | 2 - .../backgroundupload/RetryClassifier.kt | 8 +- .../backgroundupload/TransferHost.kt | 34 ++ .../backgroundupload/UploadOutcome.kt | 12 - .../openspace/backgroundupload/UploadUtils.kt | 18 +- .../backgroundupload/UploadWorker.kt | 88 +--- .../backgroundupload/UploaderModule.kt | 33 +- .../openspace/backgroundupload/WorkerOps.kt | 168 ++++--- .../openspace/backgroundupload/BodyCapTest.kt | 70 +++ .../backgroundupload/ChunkedEngineTest.kt | 4 +- .../backgroundupload/EntryParsingTest.kt | 90 +++- .../backgroundupload/EntryRunTest.kt | 464 ++++++++++++++++++ .../backgroundupload/EntryTransitionsTest.kt | 60 ++- .../backgroundupload/EventJournalTest.kt | 148 +++++- .../backgroundupload/JsonBridgeTest.kt | 43 +- .../backgroundupload/LegacyImportTest.kt | 32 ++ .../backgroundupload/QueueControllerTest.kt | 174 ++++++- .../backgroundupload/QueueStoreTest.kt | 16 +- .../backgroundupload/RetryClassifierTest.kt | 8 +- .../backgroundupload/SmallPartsTest.kt | 39 +- .../openspace/backgroundupload/TestSupport.kt | 10 +- .../backgroundupload/UploadOutcomeTest.kt | 16 - .../backgroundupload/WorkerOpsTest.kt | 189 ++++++- 38 files changed, 2309 insertions(+), 731 deletions(-) create mode 100644 android/src/main/java/ai/openspace/backgroundupload/BodyCap.kt create mode 100644 android/src/main/java/ai/openspace/backgroundupload/EntryRun.kt create mode 100644 android/src/main/java/ai/openspace/backgroundupload/TransferHost.kt create mode 100644 android/src/test/java/ai/openspace/backgroundupload/BodyCapTest.kt create mode 100644 android/src/test/java/ai/openspace/backgroundupload/EntryRunTest.kt diff --git a/README.md b/README.md index b146c38c..c0189ddc 100644 --- a/README.md +++ b/README.md @@ -191,6 +191,21 @@ notification. That notification is also the worker's foreground-service notification, so a silent request runs as an ordinary background worker and the OS may defer or restart it. Reserve it for small payloads. +### Android platform notes + +**Headless time limit.** On API 31 and later, a WorkManager run that starts +from the background usually cannot start its foreground service. The run +then has JobScheduler's limit of about 10 minutes. A single body (`data`, +`form`, `file`) that does not finish in that time starts again from byte 0 +at the next run, after a growing backoff. A body that needs more than +10 minutes headless cannot finish that way. Use `parts` for large bodies: +accepted parts are kept across runs. iOS has no equal limit. + +**Backups.** The queue store (`files/rnbgupload-chunked/`) and the journal +(`files/rnbgupload-settled/`) hold request headers, including auth tokens, +and staged bodies. Set `android:allowBackup="false"` in the host app, or +exclude those two directories in its backup rules. + # Reliable delivery 1. **Write-ahead.** Entry, descriptor, and staged body persist before any diff --git a/android/src/main/java/ai/openspace/backgroundupload/AttemptEvent.kt b/android/src/main/java/ai/openspace/backgroundupload/AttemptEvent.kt index 37a65f0a..1274c3d8 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/AttemptEvent.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/AttemptEvent.kt @@ -6,7 +6,8 @@ import com.facebook.react.bridge.WritableMap * One HTTP attempt, before the library interprets it. Live only: never * journaled. [outcome] is `completed` when the response is accepted and * `error` otherwise, so a 401 is `error` with httpCode 401 even though the - * entry parks. + * entry parks. A transport failure is `error` with its own errorKind. Pause, + * cancel, supersede, and a system stop emit no attempt event. */ data class AttemptEvent( val id: String, @@ -23,12 +24,9 @@ data class AttemptEvent( val responseHeaders: Map?, val errorKind: String?, val errorMessage: String?, - val cancelReason: String?, val at: Long, ) { companion object { - const val MAX_BODY_CHARS = 4 * 1024 - fun ofResponse( entry: QueueEntry, requestId: String, @@ -38,16 +36,16 @@ data class AttemptEvent( accepted: Boolean, at: Long, ): AttemptEvent { - val (body, truncated) = EventJournal.capBody(response.body, MAX_BODY_CHARS) + val (body, cut) = BodyCap.cap(response.body, BodyCap.ATTEMPT_MAX_BYTES) return AttemptEvent( id = entry.id, key = entry.key, requestId = requestId, attempt = entry.attempts, url = url, method = entry.descriptor?.method ?: "POST", partIndex = partIndex, outcome = if (accepted) "completed" else "error", - httpCode = response.code, responseBody = body, responseBodyTruncated = truncated, + httpCode = response.code, responseBody = body, responseBodyTruncated = cut || response.truncated, responseHeaders = response.headers, errorKind = if (accepted) null else "http", errorMessage = if (accepted) null else "HTTP ${response.code}", - cancelReason = null, at = at, + at = at, ) } @@ -64,7 +62,7 @@ data class AttemptEvent( url = url, method = entry.descriptor?.method ?: "POST", partIndex = partIndex, outcome = "error", httpCode = null, responseBody = null, responseBodyTruncated = null, responseHeaders = null, errorKind = errorKind, errorMessage = message, - cancelReason = null, at = at, + at = at, ) } @@ -83,7 +81,6 @@ data class AttemptEvent( responseHeaders?.let { put("responseHeaders", it) } errorKind?.let { put("errorKind", it) } errorMessage?.let { put("errorMessage", it) } - cancelReason?.let { put("cancelReason", it) } put("at", at.toDouble()) } diff --git a/android/src/main/java/ai/openspace/backgroundupload/BodyCap.kt b/android/src/main/java/ai/openspace/backgroundupload/BodyCap.kt new file mode 100644 index 00000000..ade8e7df --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/BodyCap.kt @@ -0,0 +1,67 @@ +package ai.openspace.backgroundupload + +import okio.Buffer +import okio.BufferedSource +import java.nio.charset.Charset + +/** + * Response body caps, in UTF-8 bytes. A cut never splits a character: it + * backs off to the last whole one. [read] applies the cap while the body + * streams in, so a huge error page never sits in memory whole. + */ +object BodyCap { + /** 1 MB, the RawResponse cap of a settled outcome. */ + const val SETTLED_MAX_BYTES = 1_048_576 + + /** 4 KB, the body cap of a live attempt event. */ + const val ATTEMPT_MAX_BYTES = 4 * 1024 + + /** A body as text and whether the cap cut it. */ + data class Capped(val text: String, val truncated: Boolean) + + /** + * Reads at most [maxBytes] of [source]. Bytes past the cap are not read. + * A UTF-8 body is cut on a character boundary; another charset is cut at + * the byte cap and decoded as it is. + */ + fun read(source: BufferedSource, maxBytes: Int, charset: Charset = Charsets.UTF_8): Capped { + val buffer = Buffer() + val limit = maxBytes.toLong() + 1 + while (buffer.size < limit) { + if (source.read(buffer, limit - buffer.size) == -1L) break + } + val truncated = buffer.size > maxBytes + val bytes = buffer.readByteArray() + val keep = if (!truncated) bytes.size + else if (charset == Charsets.UTF_8) utf8Boundary(bytes, maxBytes) + else maxBytes + return Capped(String(bytes, 0, keep, charset), truncated) + } + + /** [text] cut to at most [maxBytes] of UTF-8. Null stays null. */ + fun cap(text: String?, maxBytes: Int): Pair { + if (text == null) return null to false + val bytes = text.toByteArray(Charsets.UTF_8) + if (bytes.size <= maxBytes) return text to false + return String(bytes, 0, utf8Boundary(bytes, maxBytes), Charsets.UTF_8) to true + } + + /** + * The longest prefix length of [bytes], at most [max], that does not end + * inside a UTF-8 sequence. A continuation byte is 10xxxxxx. + */ + internal fun utf8Boundary(bytes: ByteArray, max: Int): Int { + if (bytes.size <= max) return bytes.size + var end = max + // bytes[end] is the first byte cut off. While it continues a sequence, + // the sequence started before the cut, so drop its start too. A UTF-8 + // character is at most 4 bytes; past 3 steps the bytes are not UTF-8, + // and the cut stays at max. + var steps = 0 + while (end > 0 && steps < 3 && (bytes[end].toInt() and 0xC0) == 0x80) { + end-- + steps++ + } + return if ((bytes[end].toInt() and 0xC0) == 0x80) max else end + } +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/ChunkedUploadWorker.kt b/android/src/main/java/ai/openspace/backgroundupload/ChunkedUploadWorker.kt index 1b1fed34..d236ec9a 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/ChunkedUploadWorker.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/ChunkedUploadWorker.kt @@ -2,10 +2,7 @@ package ai.openspace.backgroundupload import android.content.Context import androidx.work.WorkerParameters -import kotlinx.coroutines.CancellationException -import kotlinx.coroutines.sync.withPermit import java.io.File -import java.util.UUID import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.atomic.AtomicLong @@ -18,12 +15,13 @@ class ChunkedUploadWorker(context: Context, params: WorkerParameters) : EntryWor * one progress stream (byte-weighted) and one outcome: completed only when * every part is accepted. * + * Each part runs [EntryRun.attempt] until a verdict ends it. * Per part: accepted → persist the flag; auth → the whole entry parks (the * sibling parts stop); transient → a short backoff waits in the part while * the siblings go on, a long one releases the whole worker (accepted parts * are kept); terminal → the entry fails with that part's index. */ -internal class ChunkedTransfer(private val host: EntryWorker) { +internal class ChunkedTransfer(private val run: EntryRun) { /** A terminal part failure. Not a CancellationException, so it stops the sibling parts. */ private class PartFailed(val settlement: Settlement.Failed) : Exception(settlement.message) @@ -43,14 +41,13 @@ internal class ChunkedTransfer(private val host: EntryWorker) { val pending = ChunkedParts.pendingIndexes(parts) // A run over an all-accepted entry that has not settled yet. if (pending.isEmpty()) return Settlement.Completed(null, lastAcceptedUrl ?: d0.reportUrl, d0.method) - val blob = host.bodyFile(start) + val blob = run.store.bodyFile(start) if (blob == null || !blob.exists()) { return Settlement.Failed("file", "the chunked file is missing", null, null, d0.reportUrl, d0.method) } total = ChunkedParts.totalBytes(parts) acceptedBytes.set(ChunkedParts.acceptedBytes(parts)) - UploadProgress.add(host.entryId, total) - UploadProgress.set(host.entryId, acceptedBytes.get()) + run.progressStarted(total, acceptedBytes.get()) try { ChunkedEngine.run(pending) { index -> executePart(index, blob, start.backoffStreak) } @@ -62,17 +59,17 @@ internal class ChunkedTransfer(private val host: EntryWorker) { return Settlement.Completed(null, lastAcceptedUrl ?: d0.reportUrl, d0.method) } - private suspend fun executePart(index: Int, blob: File, initialStreak: Int) { + internal suspend fun executePart(index: Int, blob: File, initialStreak: Int) { var streak = initialStreak while (true) { if (index in acceptedHere) return - val latest = host.ops.latest(host.entryId, host.generation) + val latest = run.ops.latest(run.entryId, run.generation) val stored = latest.descriptor!!.parts!![index] if (stored.accepted) return - if (RetryClassifier.isExpired(host.now(), latest.expiresAt)) throw EntryWorker.ExpiredException() + if (RetryClassifier.isExpired(run.host.now(), latest.expiresAt)) throw EntryRun.ExpiredException() // A range past EOF can never be sent. length() is 0 for a missing file; - // that case falls through to the transfer, which classifies it as file. + // that case falls through to the attempt, which classifies it as file. val blobLength = runCatching { blob.length() }.getOrDefault(0L) if (blobLength > 0L && stored.end > blobLength) { throw PartFailed( @@ -83,75 +80,33 @@ internal class ChunkedTransfer(private val host: EntryWorker) { ), ) } - host.waitForNetwork() + run.waitForNetwork() - val requestId = UUID.randomUUID().toString() - val entry = host.ops.recordAttempt(host.entryId, host.generation, requestId) - // The generation of the headers this attempt sends: both come from the same entry. - val headerGeneration = entry.headerGeneration - val d = entry.descriptor!! - val part = d.parts!![index] - val policy = host.policy(entry) - - val response = try { - transferSemaphore.withPermit { - okhttpSend( - uploadHttpClient, - TransferRequest( - part.url, d.method, host.headersFor(d, part, requestId), - rangeRequestBody(blob, part.start, part.end), - ), - ) { sent -> onPartProgress(index, sent) } - } - } catch (error: CancellationException) { - throw error - } catch (error: Throwable) { - onPartProgress(index, 0L) - val fileExists = runCatching { blob.exists() }.getOrDefault(true) - val message = error.message ?: error.javaClass.simpleName - EventReporter.attempt( - AttemptEvent.ofFailure( - entry, requestId, part.url, index, RetryClassifier.failureKind(error, fileExists), message, host.now(), - ), - ) - when (val verdict = RetryClassifier.classifyFailure(error, fileExists)) { - is RetryClassifier.Verdict.Terminal -> throw PartFailed( - Settlement.Failed(verdict.errorKind, verdict.message, null, index, part.url, d.method), - ) - else -> { - streak++ - host.backoffOrRelease(policy, streak, entry.expiresAt) - continue - } - } - } - - val verdict = RetryClassifier.classifyResponse(response.code, response.body, d.accept, policy.exempt) - EventReporter.attempt( - AttemptEvent.ofResponse( - entry, requestId, part.url, index, response, verdict == RetryClassifier.Verdict.Accepted, host.now(), - ), + val a = run.attempt( + partIndex = index, + body = { _, part -> rangeRequestBody(blob, part!!.start, part.end) }, + onProgress = { sent -> onPartProgress(index, sent) }, + fileExists = { blob.exists() }, ) - when (verdict) { - RetryClassifier.Verdict.Accepted -> { - markAccepted(index, part) + when (val r = a.result) { + is EntryRun.AttemptResult.Accepted -> { + markAccepted(index, a.entry.descriptor!!.parts!![index]) return } - RetryClassifier.Verdict.Auth -> { - if (host.ops.hasNewerHeaders(host.entryId, host.generation, headerGeneration)) { - streak = 0 - continue - } - throw EntryWorker.ParkException(headerGeneration) + is EntryRun.AttemptResult.Auth -> { + if (!r.reissue) throw EntryRun.ParkException(r.headerGeneration) + streak = 0 } - RetryClassifier.Verdict.Transient -> { + EntryRun.AttemptResult.Transient -> { onPartProgress(index, 0L) streak++ - host.backoffOrRelease(policy, streak, entry.expiresAt) + run.backoffOrRelease(a.policy, streak, a.entry.expiresAt) + } + is EntryRun.AttemptResult.Terminal -> { + onPartProgress(index, 0L) + val message = r.response?.let { "HTTP ${it.code} on part $index" } ?: r.message + throw PartFailed(Settlement.Failed(r.errorKind, message, r.response, index, a.url, a.method)) } - is RetryClassifier.Verdict.Terminal -> throw PartFailed( - Settlement.Failed("http", "HTTP ${response.code} on part $index", response, index, part.url, d.method), - ) } } } @@ -159,7 +114,7 @@ internal class ChunkedTransfer(private val host: EntryWorker) { private fun markAccepted(index: Int, part: Part) { // Remembered here too, so a lost flag write does not re-send the part in this run. acceptedHere += index - host.ops.markAccepted(host.entryId, host.generation, index) + run.ops.markAccepted(run.entryId, run.generation, index) acceptedBytes.addAndGet(part.size) lastAcceptedUrl = part.url partSent.remove(index) @@ -173,6 +128,6 @@ internal class ChunkedTransfer(private val host: EntryWorker) { private fun report() { val sent = (acceptedBytes.get() + partSent.values.sum()).coerceAtMost(total) - host.reportProgress(sent, total) + run.reportProgress(sent, total) } } diff --git a/android/src/main/java/ai/openspace/backgroundupload/EnqueueRules.kt b/android/src/main/java/ai/openspace/backgroundupload/EnqueueRules.kt index 5b4fdf64..4331797e 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/EnqueueRules.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/EnqueueRules.kt @@ -7,10 +7,12 @@ package ai.openspace.backgroundupload * | Stored entry | Action | * | none, no v9 manifest | Create | * | none, v9 manifest | AdoptV9: keep the blob and accepted parts | - * | legacy row | Replace (generation + 1) | + * | legacy row, v9 manifest | AdoptV9 (generation + 1) | + * | legacy row, no manifest | Replace (generation + 1) | * | same body, completed, record present | ReEmit: deliveries + 1, no re-run | * | same body, completed, record gone | Replace (it was acked) | - * | same body, any other state | Resume (a settled one reopens: gen + 1) | + * | same body, any other state | Resume (a settled one reopens: gen + 1, | + * | | attempts 0) | * | different body, running | RejectRunning (E_RUNNING) | * | different body, otherwise | Replace (generation + 1, attempts 0) | */ @@ -18,7 +20,8 @@ object EnqueueRules { sealed class Action { object Create : Action() - data class AdoptV9(val manifest: LegacyManifest) : Action() + /** [generation] is 1, or the legacy row's + 1. */ + data class AdoptV9(val manifest: LegacyManifest, val generation: Int) : Action() data class ReEmit(val eventId: String) : Action() object Resume : Action() object Replace : Action() @@ -31,8 +34,10 @@ object EnqueueRules { incoming: Descriptor, hasRecord: (eventId: String) -> Boolean, ): Action { - if (existing == null) return if (v9 != null) Action.AdoptV9(v9) else Action.Create - if (existing.legacy) return Action.Replace + if (existing == null) return if (v9 != null) Action.AdoptV9(v9, 1) else Action.Create + // An imported v9 outcome row. Its v9 chunked manifest, when present, is + // the upload's progress: adopt it, as with no row at all. + if (existing.legacy) return if (v9 != null) Action.AdoptV9(v9, existing.generation + 1) else Action.Replace if (existing.sameBodyAs(incoming)) { if (existing.state == EntryState.COMPLETED) { val eventId = existing.settledEventId @@ -100,11 +105,27 @@ object EnqueueRules { ) } + /** AdoptV9: a new entry over the v9 blob, at [generation]; over a legacy row it keeps the row's createdAt. */ + fun adopted( + p: EntryParsing.Parsed, + staged: BodyStaging.Staged, + parts: List?, + legacyRow: QueueEntry?, + generation: Int, + paused: Boolean, + headerGeneration: Int, + now: Long, + ): QueueEntry = created(p, staged, parts, paused, headerGeneration, now).copy( + createdAt = legacyRow?.createdAt ?: now, + generation = generation, + ) + /** * Same body. New headers, expiresAt, vars, accept, retry, and notification * flag replace the stored ones; the body and accepted parts stay. A settled - * entry reopens with a fresh generation. A running one stays running (the - * worker reads the new headers before its next attempt). + * entry reopens with a fresh generation and attempts 0 (attempts count the + * current generation). A running one stays running (the worker reads the + * new headers before its next attempt). */ fun resumed( existing: QueueEntry, @@ -129,6 +150,7 @@ object EnqueueRules { noNotification = p.descriptor.noNotification, ), state = if (running) EntryState.RUNNING else initialState(paused), + attempts = if (reopen) 0 else existing.attempts, bytesSent = parts?.let { ChunkedParts.acceptedBytes(it) } ?: if (running) existing.bytesSent else 0L, expiresAt = p.expiresAt, updatedAt = now, diff --git a/android/src/main/java/ai/openspace/backgroundupload/EntryParsing.kt b/android/src/main/java/ai/openspace/backgroundupload/EntryParsing.kt index 81d06fb9..f3c77ac6 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/EntryParsing.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/EntryParsing.kt @@ -8,12 +8,14 @@ import okhttp3.HttpUrl.Companion.toHttpUrlOrNull import java.net.URI /** - * Turns the EnqueueEntry `{ id, key, vars, descriptor }` into Kotlin values. - * JS has already validated the descriptor. Native checks only what it needs - * to run, and rejects anything else with E_INVALID. + * Turns the EnqueueEntry `{ id, key, varsJson, descriptor }` into Kotlin + * values. JS has already validated the descriptor. Native checks only what + * it needs to run, and rejects anything else with E_INVALID. * - * A null value is read as absent. The bridge turns a JS `undefined` into - * null, so native can not tell `data: null` from `data: undefined`. + * `varsJson` and `descriptor.dataJson` are JSON text. They cross as strings + * because React Native on iOS drops object keys whose value is null. Native + * keeps the text as it came: it is the body that goes out. A dataJson of + * "null" is the JSON body null, a real body. */ object EntryParsing { class InvalidEntryException(message: String) : IllegalArgumentException(message) @@ -31,7 +33,8 @@ object EntryParsing { fun parse(entry: ReadableMap): Parsed { val id = entry.string("id")?.takeIf { it.isNotEmpty() } ?: invalid("id is required") val key = entry.string("key")?.takeIf { it.isNotEmpty() } ?: invalid("key is required") - val varsJson = JsonBridge.toJson(JsonBridge.valueOf(entry, "vars")) + val varsJson = entry.string("varsJson") ?: invalid("varsJson is required") + if (!JsonBridge.isJson(varsJson)) invalid("varsJson is not JSON text") val d = entry.map("descriptor") ?: invalid("descriptor is required") val expiresAt = d.number("expiresAt")?.toLong() ?: invalid("descriptor.expiresAt is required") return Parsed(id, key, varsJson, descriptor(d), expiresAt) @@ -46,7 +49,13 @@ object EntryParsing { if (url == null && parts == null) invalid("url is required unless parts is set") url?.let { requireHttpUrl(it, "url") } - val dataJson = if (d.isSet("data")) JsonBridge.toJson(JsonBridge.valueOf(d, "data")) else null + // An old JS layer would send `data`. Ignoring it would send no body. + if (d.isSet("data")) invalid("data crosses as dataJson") + val dataJson = if (d.isSet("dataJson")) { + val text = d.string("dataJson") ?: invalid("dataJson must be a string") + if (!JsonBridge.isJson(text)) invalid("dataJson is not JSON text") + text + } else null val form = d.array("form")?.let { parseForm(it) } val file = d.string("file")?.let { stripFileScheme(it) } val kinds = listOfNotNull(dataJson?.let { "data" }, form?.let { "form" }, file?.let { "file" }) @@ -152,14 +161,30 @@ object EntryParsing { if (url.toHttpUrlOrNull() == null) invalid("$where is not an http(s) url: $url") } - // OkHttp throws on a header name or value it can not send. Check it here, - // so the error is an enqueue rejection and not a failure at attempt time. - private fun requireValidHeaders(headers: Map, where: String) { - try { - val builder = Headers.Builder() - headers.forEach { (k, v) -> builder.add(k, v) } - } catch (e: IllegalArgumentException) { - invalid("$where: ${e.message}") + /** + * OkHttp throws on a header name or value it can not send. Check it here, + * so the error is an enqueue rejection and not a failure at attempt time. + * The message names the header and the offset only: a value can be a + * credential, and OkHttp's own message would print it. + */ + internal fun requireValidHeaders(headers: Map, where: String) { + headers.forEach { (name, value) -> + // OkHttp's rules: a name is 1+ chars in 0x21..0x7e; a value is tab or 0x20..0x7e. + if (name.isEmpty()) invalid("$where: a header name is empty") + name.indexOfFirst { it !in '\u0021'..'\u007e' }.takeIf { it >= 0 }?.let { i -> + // Only the valid part before the offset: the rest could be a value + // pasted into the name. + invalid("$where: the header name that starts '${name.take(i)}' has an invalid character at offset $i") + } + value.indexOfFirst { it != '\t' && it !in '\u0020'..'\u007e' }.takeIf { it >= 0 }?.let { i -> + invalid("$where: the value of header '$name' has an invalid character at offset $i") + } + // A backstop for any rule OkHttp adds later. Its message is not used. + try { + Headers.Builder().add(name, value) + } catch (e: IllegalArgumentException) { + invalid("$where: the HTTP client does not accept header '$name'") + } } } diff --git a/android/src/main/java/ai/openspace/backgroundupload/EntryRun.kt b/android/src/main/java/ai/openspace/backgroundupload/EntryRun.kt new file mode 100644 index 00000000..d9b5ffbe --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/EntryRun.kt @@ -0,0 +1,325 @@ +package ai.openspace.backgroundupload + +import kotlinx.coroutines.CancellationException +import okhttp3.RequestBody +import java.io.IOException +import java.util.UUID +import kotlin.math.max +import kotlin.math.min + +/** + * The run of one queue entry, and the one attempt step both transfers use. + * No Android types: [EntryWorker] is the shell that holds the per-id gate + * and is the real [TransferHost]. + * + * The run: take the entry (queued → running), run the transfer, then + * settle, park, or release. The body kind at run time picks the transfer, + * [SimpleTransfer] or [ChunkedTransfer], so a kind change under a queued run + * is safe. The entry is read from the store at start and again before every + * attempt, so fresh headers and a new expiresAt reach a running run. + */ +internal class EntryRun( + val entryId: String, + val store: QueueStore, + val ops: WorkerOps, + val host: TransferHost, +) { + /** A 401/403 under the headers of [headerGeneration] (the entry's value the attempt sent). */ + class ParkException(val headerGeneration: Int) : Exception("awaiting auth") + + /** A backoff too long to wait here. */ + class BackoffException(val nextAttemptAt: Long, val streak: Int) : Exception("released for backoff") + + class ExpiredException : Exception("expired before completion") + + /** The queue was paused between the module's pause and the work cancel reaching us. */ + class PausedException : Exception("queue paused") + + /** How one attempt ended, after the retry table. */ + sealed class AttemptResult { + data class Accepted(val response: UploadResponse) : AttemptResult() + + /** A 401/403. [reissue]: newer headers arrived while it was in flight, so send again now. */ + data class Auth(val headerGeneration: Int, val reissue: Boolean) : AttemptResult() + + object Transient : AttemptResult() + + data class Terminal(val errorKind: String, val message: String, val response: UploadResponse?) : AttemptResult() + } + + /** One attempt: the entry it ran under, where it went, the retry policy, and how it ended. */ + class Attempt( + val entry: QueueEntry, + val url: String, + val policy: RetryClassifier.Policy, + val result: AttemptResult, + ) { + val method: String get() = entry.descriptor!!.method + } + + companion object { + /** The poll while the network is unusable (offline, or waiting for wifi). */ + const val CONNECTIVITY_POLL_MS = 10_000L + + /** The poll while a short backoff remainder runs out before the run starts. */ + private const val WAIT_POLL_MS = 10_000L + + /** The descriptor's headers, the part's over them, and X-Request-Id over all. */ + fun headersFor(d: Descriptor, part: Part?, requestId: String): Map { + val withPart = if (part == null) d.headers else HeaderMap.merge(d.headers, part.headers) + return HeaderMap.merge(withPart, mapOf("X-Request-Id" to requestId)) + } + } + + var generation = 0 + private set + + /** The bytes of the current attempt, as last reported. A failed simple entry settles with them. */ + @Volatile + var liveBytes = 0L + private set + + /** + * Returns when the run is over. Throws a CancellationException after it + * handled a stop, or an IOException after a store write failed (the + * caller returns retry; nothing settled). + */ + suspend fun run() { + val initial = store.load(entryId) ?: return // forgotten while queued + if (initial.legacy) return + if (initial.state == EntryState.AWAITING_AUTH) { + // The expiry wake of a parked entry. No attempt ran here, so the + // stored bytes stand. + if (RetryClassifier.isExpired(host.now(), initial.expiresAt)) { + ops.settle(entryId, initial.generation, expired(initial).copy(bytesSent = null)) + } + return + } + if (initial.state != EntryState.QUEUED && initial.state != EntryState.RUNNING) return + if (ops.settings().paused) return + if (!waitUntilDue(initial)) return + val entry = ops.begin(entryId) ?: return + generation = entry.generation + + var current = entry + var first = true + while (true) { + try { + if (!first) current = ops.latest(entryId, generation) + first = false + host.foreground(current) + val settlement = transfer(current) + endProgress(completed = settlement is Settlement.Completed) + ops.settle(entryId, generation, settlement) + return + } catch (park: ParkException) { + endProgress(completed = false) + // REISSUE: updateHeaders() landed while this attempt was in flight. + if (ops.park(entryId, generation, park.headerGeneration) != WorkerOps.ParkResult.REISSUE) return + } catch (backoff: BackoffException) { + endProgress(completed = false) + ops.release(entryId, generation, backoff.nextAttemptAt, backoff.streak) + return + } catch (error: ExpiredException) { + endProgress(completed = false) + ops.settle(entryId, generation, expired(current)) + return + } catch (error: NotOwnedException) { + endProgress(completed = false) + return + } catch (error: PausedException) { + endProgress(completed = false) + return + } catch (error: CancellationException) { + // A system stop moves a running entry back to queued. A pause or a + // cancel already moved it; then this does nothing. + endProgress(completed = false) + if (host.stoppedByTimeout()) releaseAfterTimeout() else ops.stopped(entryId, generation) + throw error + } catch (error: IOException) { + // A store write failed (disk full, directory briefly unwritable). + // The attempt step classifies every network IOException itself, so + // one that lands here is storage: transient, no outcome. Back to + // queued; the caller returns retry. + endProgress(completed = false) + ops.stopped(entryId, generation) + throw error + } catch (error: Throwable) { + endProgress(completed = false) + val d = current.descriptor + ops.settle( + entryId, + generation, + Settlement.Failed( + errorKind = "unknown", + message = error.message ?: error.javaClass.simpleName, + response = null, + partIndex = null, + url = d?.reportUrl ?: "", + method = d?.method ?: "POST", + bytesSent = liveBytesOf(current), + ), + ) + return + } + } + } + + private suspend fun transfer(entry: QueueEntry): Settlement = + if (entry.body?.kind == StagedBody.CHUNKED) ChunkedTransfer(this).run(entry) + else SimpleTransfer(this).run(entry) + + private fun liveBytesOf(entry: QueueEntry): Long? = + if (entry.body?.kind == StagedBody.CHUNKED) null else liveBytes + + private fun expired(entry: QueueEntry) = Settlement.Failed( + errorKind = "expired", + message = "expired before completion", + response = null, + partIndex = null, + url = entry.descriptor?.reportUrl ?: "", + method = entry.descriptor?.method ?: "POST", + bytesSent = liveBytesOf(entry), + ) + + /** + * The system stopped the run at its time limit. That happens to a + * headless run whose foreground start was denied (API 31+), after about + * 10 minutes. Starting again at once would send the body from byte 0 on + * every run, so this is one more transient failure: the next backoff + * step, then a wake. + */ + private fun releaseAfterTimeout() { + runCatching { + val e = store.load(entryId) + if (!EntryTransitions.isOwnedRun(e, generation)) return + val streak = e!!.backoffStreak + 1 + val now = host.now() + val backoff = RetryClassifier.backoffMs(policy(e), streak) + ops.release(entryId, generation, RetryClassifier.nextAttemptAt(now, backoff, e.expiresAt), streak) + }.onFailure { Diag.error("could not release '$entryId' after a timeout stop", it) } + } + + /** + * Sleeps out a short backoff remainder. False when the wait is long (the + * wake run comes back for it) or the entry is no longer queued. + */ + private suspend fun waitUntilDue(initial: QueueEntry): Boolean { + var e = initial + while (true) { + val at = e.nextAttemptAt ?: return true + val remaining = at - host.now() + if (remaining <= 0) return true + if (remaining > RetryClassifier.IN_WORKER_BACKOFF_MAX_MS) return false + host.sleep(min(remaining, WAIT_POLL_MS)) + e = store.load(entryId) ?: return false + if (e.state != EntryState.QUEUED && e.state != EntryState.RUNNING) return false + } + } + + // MARK: - helpers for the transfers + + /** + * Waits until the network fits the queue's wifi-only setting. Re-reads + * the settings and the entry at every poll. + */ + suspend fun waitForNetwork() { + while (true) { + val s = ops.settings() + if (s.paused) throw PausedException() + val entry = ops.latest(entryId, generation) + if (RetryClassifier.isExpired(host.now(), entry.expiresAt)) throw ExpiredException() + if (host.connectivity(s.wifiOnly) == Connectivity.Ok) return + host.sleep(CONNECTIVITY_POLL_MS) + } + } + + fun policy(entry: QueueEntry) = + RetryClassifier.policy(ops.settings().retry, entry.descriptor?.retry) + + /** + * One HTTP attempt, the step both transfers share: write the attempt + * ahead (attempts + 1, its X-Request-Id), send, emit the attempt event, + * and classify. [partIndex] is null for a simple entry. [body] builds the + * request body from the entry the attempt runs under. [fileExists] tells a + * missing payload from a network failure. + */ + suspend fun attempt( + partIndex: Int?, + body: (Descriptor, Part?) -> RequestBody?, + onProgress: (Long) -> Unit, + fileExists: () -> Boolean, + ): Attempt { + val requestId = UUID.randomUUID().toString() + val entry = ops.recordAttempt(entryId, generation, requestId) + // The generation of the headers this attempt sends: both come from the same entry. + val headerGeneration = entry.headerGeneration + val d = entry.descriptor!! + val part = partIndex?.let { d.parts!![it] } + val url = part?.url ?: d.url!! + val policy = policy(entry) + + val response = try { + host.send(TransferRequest(url, d.method, headersFor(d, part, requestId), body(d, part)), onProgress) + } catch (error: CancellationException) { + throw error + } catch (error: Throwable) { + // A failed probe reads as present, so a flaky check is a retryable + // network error and not a terminal "file gone". + val verdict = RetryClassifier.classifyFailure(error, runCatching(fileExists).getOrDefault(true)) + host.attempt( + AttemptEvent.ofFailure( + entry, requestId, url, partIndex, RetryClassifier.failureKind(verdict), + error.message ?: error.javaClass.simpleName, host.now(), + ), + ) + val result = if (verdict is RetryClassifier.Verdict.Terminal) { + AttemptResult.Terminal(verdict.errorKind, verdict.message, null) + } else AttemptResult.Transient + return Attempt(entry, url, policy, result) + } + + val verdict = RetryClassifier.classifyResponse(response.code, response.body, d.accept, policy.exempt) + host.attempt( + AttemptEvent.ofResponse( + entry, requestId, url, partIndex, response, verdict == RetryClassifier.Verdict.Accepted, host.now(), + ), + ) + val result = when (verdict) { + RetryClassifier.Verdict.Accepted -> AttemptResult.Accepted(response) + RetryClassifier.Verdict.Auth -> + AttemptResult.Auth(headerGeneration, ops.hasNewerHeaders(entryId, generation, headerGeneration)) + RetryClassifier.Verdict.Transient -> AttemptResult.Transient + is RetryClassifier.Verdict.Terminal -> AttemptResult.Terminal("http", verdict.message, response) + } + return Attempt(entry, url, policy, result) + } + + /** + * A short backoff waits here, with the row still running and showing + * nextAttemptAt; a long one throws [BackoffException] to release the run. + */ + suspend fun backoffOrRelease(policy: RetryClassifier.Policy, streak: Int, expiresAt: Long) { + val backoff = RetryClassifier.backoffMs(policy, streak) + val now = host.now() + if (backoff <= RetryClassifier.IN_WORKER_BACKOFF_MAX_MS) { + val wait = min(backoff, max(0L, expiresAt - now)) + ops.backingOff(entryId, generation, now + wait) + host.sleep(wait) + return + } + throw BackoffException(RetryClassifier.nextAttemptAt(now, backoff, expiresAt), streak) + } + + fun progressStarted(total: Long, sent: Long) { + liveBytes = sent + host.progressStarted(entryId, total, sent) + } + + fun reportProgress(sent: Long, total: Long) { + liveBytes = sent + host.progress(entryId, sent, total) + } + + private fun endProgress(completed: Boolean) = host.progressEnded(entryId, completed) +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/EntryTransitions.kt b/android/src/main/java/ai/openspace/backgroundupload/EntryTransitions.kt index 94a297fa..d099d8df 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/EntryTransitions.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/EntryTransitions.kt @@ -76,9 +76,37 @@ object EntryTransitions { updatedAt = now, ) - /** Whether a worker of [generation] may still settle [e]. Not after a cancel, a replace, or a settle. */ - fun canSettle(e: QueueEntry?, generation: Int) = - e != null && e.generation == generation && e.isLive + /** + * Whether a worker of [generation] may still settle [e]. Not after a + * cancel, a replace, or a settle. Under pause only an [accepted] response + * settles: the server already took it. A failure waits for resume, which + * runs the entry again. + */ + fun canSettle(e: QueueEntry?, generation: Int, accepted: Boolean) = + e != null && e.generation == generation && e.isLive && (accepted || e.state != EntryState.PAUSED) + + /** The entry state a journal record puts its entry in. */ + fun stateOf(record: EventJournal.SettledRecord): EntryState = + EntryState.values().firstOrNull { it.wire == record.state } ?: EntryState.ERROR + + /** A settled entry, and the other records of its life to ack. */ + data class Journaled(val entry: QueueEntry, val extraEventIds: List) + + /** + * A live entry with a journal record of its own generation: the settle + * journaled, then its store write was lost (a process death, a failed + * save). Apply the newest record; do not run the request again. Null when + * there is no such record. The boot sweep and a worker's begin share it. + */ + fun journaledSettle(e: QueueEntry, records: List, now: Long): Journaled? { + if (!e.isLive || e.legacy) return null + val own = records.filter { it.id == e.id && it.generation == e.generation } + val latest = own.maxByOrNull { it.at } ?: return null + return Journaled( + toSettled(e, stateOf(latest), latest.eventId, latest.bytesSent, now), + own.filter { it !== latest }.map { it.eventId }, + ) + } /** Whether a worker of [generation] still owns the running entry. */ fun isOwnedRun(e: QueueEntry?, generation: Int) = diff --git a/android/src/main/java/ai/openspace/backgroundupload/EntryWorker.kt b/android/src/main/java/ai/openspace/backgroundupload/EntryWorker.kt index 35f67957..d967862b 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/EntryWorker.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/EntryWorker.kt @@ -4,57 +4,35 @@ import android.app.NotificationManager import android.content.Context import androidx.work.CoroutineWorker import androidx.work.ForegroundInfo +import androidx.work.WorkInfo import androidx.work.WorkerParameters import kotlinx.coroutines.CancellationException import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.delay +import kotlinx.coroutines.sync.withPermit import kotlinx.coroutines.withContext -import java.io.File -import java.io.IOException -import kotlin.math.max -import kotlin.math.min /** - * Runs one queue entry. The input data holds only the entry id; the worker - * reads the entry from the store at start and again before every attempt, - * so fresh headers and a new expiresAt reach a running worker with no - * restart. + * The WorkManager shell of one entry's run. The input data holds only the + * entry id. The worker acquires the per-id gate, then [EntryRun] does the + * run; this class is its [TransferHost]: OkHttp, the clock, progress, and + * the notification. * - * The run: acquire the per-id gate, take the entry (queued → running), run - * the transfer, then settle, park, or release. The body kind picks the - * transfer: [SimpleTransfer] or [ChunkedTransfer]. [UploadWorker] and - * [ChunkedUploadWorker] are the two class names WorkManager knows; both run - * this same code, so a kind change under a queued run is safe. - * - * Every run returns success (see [WorkManagerScheduler] for why). A v9 row, - * which has no entry id, exits at once in silence. + * [UploadWorker] and [ChunkedUploadWorker] are the two class names + * WorkManager knows; both run this same code. Every run returns success + * (see [WorkManagerScheduler] for why), except a store failure before the + * run could settle, which returns retry. A v9 row, which has no entry id, + * exits at once in silence. */ open class EntryWorker(protected val context: Context, params: WorkerParameters) : CoroutineWorker(context, params) { companion object { private const val GATE_POLL_MS = 100L - /** The poll while the network is unusable (offline, or waiting for wifi). */ - const val CONNECTIVITY_POLL_MS = 10_000L - /** The poll while a short backoff remainder runs out before the run starts. */ - private const val WAIT_POLL_MS = 10_000L } - /** A 401/403 under the headers of [headerGeneration] (the entry's value the attempt sent). */ - class ParkException(val headerGeneration: Int) : Exception("awaiting auth") - - /** A backoff too long to wait here. */ - class BackoffException(val nextAttemptAt: Long, val streak: Int) : Exception("released for backoff") - - class ExpiredException : Exception("expired before completion") - - /** The queue was paused between the module's pause and the work cancel reaching us. */ - class PausedException : Exception("queue paused") - - internal lateinit var entryId: String - internal var generation = 0 - internal val store by lazy { QueueStore.get(context) } - internal val ops by lazy { + private val store by lazy { QueueStore.get(context) } + private val ops by lazy { WorkerOps( store, EventJournal.get(context), @@ -75,12 +53,11 @@ open class EntryWorker(protected val context: Context, params: WorkerParameters) final override suspend fun doWork(): Result = withContext(Dispatchers.IO) { val id = inputData.getString(WorkManagerScheduler.ENTRY_ID_KEY) ?: return@withContext Result.success() - entryId = id // Acquire before the first store read: a cancel-then-enqueue can start // this run while the old one still winds down. while (!WorkerGate.tryAcquire(id, this@EntryWorker)) delay(GATE_POLL_MS) try { - runEntry() + EntryRun(id, store, ops, host).run() Result.success() } catch (error: CancellationException) { throw error @@ -94,174 +71,45 @@ open class EntryWorker(protected val context: Context, params: WorkerParameters) } } - private suspend fun runEntry() { - val initial = store.load(entryId) ?: return // forgotten while queued - if (initial.legacy) return - if (initial.state == EntryState.AWAITING_AUTH) { - // The expiry wake of a parked entry. - if (RetryClassifier.isExpired(now(), initial.expiresAt)) { - ops.settle(entryId, initial.generation, expired(initial)) - } - return - } - if (initial.state != EntryState.QUEUED && initial.state != EntryState.RUNNING) return - if (ops.settings().paused) return - if (!waitUntilDue(initial)) return - val entry = ops.begin(entryId) ?: return - generation = entry.generation - showsNotification = entry.descriptor?.noNotification == false + private val host = object : TransferHost { + override fun now() = System.currentTimeMillis() - var current = entry - var first = true - while (true) { - try { - if (!first) current = ops.latest(entryId, generation) - first = false - startForeground() - val settlement = transfer(current) - endProgress(completed = settlement is Settlement.Completed) - ops.settle(entryId, generation, settlement) - return - } catch (park: ParkException) { - endProgress(completed = false) - // REISSUE: updateHeaders() landed while this attempt was in flight. - if (ops.park(entryId, generation, park.headerGeneration) != WorkerOps.ParkResult.REISSUE) return - } catch (backoff: BackoffException) { - endProgress(completed = false) - ops.release(entryId, generation, backoff.nextAttemptAt, backoff.streak) - return - } catch (error: ExpiredException) { - endProgress(completed = false) - ops.settle(entryId, generation, expired(current)) - return - } catch (error: NotOwnedException) { - endProgress(completed = false) - return - } catch (error: PausedException) { - endProgress(completed = false) - return - } catch (error: CancellationException) { - // A system stop moves a running entry back to queued. A pause or a - // cancel already moved it; then this does nothing. - endProgress(completed = false) - ops.stopped(entryId, generation) - throw error - } catch (error: IOException) { - // A store write failed (disk full, directory briefly unwritable). - // The transfers classify every network IOException themselves, so - // one that lands here is storage: transient, no outcome. Back to - // queued; doWork returns retry. - endProgress(completed = false) - ops.stopped(entryId, generation) - throw error - } catch (error: Throwable) { - endProgress(completed = false) - val d = current.descriptor - ops.settle( - entryId, - generation, - Settlement.Failed( - errorKind = "unknown", - message = error.message ?: error.javaClass.simpleName, - response = null, - partIndex = null, - url = d?.reportUrl ?: "", - method = d?.method ?: "POST", - ), - ) - return - } - } - } + override suspend fun sleep(ms: Long) = delay(ms) - private suspend fun transfer(entry: QueueEntry): Settlement = - if (entry.body?.kind == StagedBody.CHUNKED) ChunkedTransfer(this).run(entry) - else SimpleTransfer(this).run(entry) + override suspend fun send(request: TransferRequest, onProgress: (Long) -> Unit): UploadResponse = + transferSemaphore.withPermit { okhttpSend(uploadHttpClient, request, onProgress) } - private fun expired(entry: QueueEntry) = Settlement.Failed( - errorKind = "expired", - message = "expired before completion", - response = null, - partIndex = null, - url = entry.descriptor?.reportUrl ?: "", - method = entry.descriptor?.method ?: "POST", - ) - - /** - * Sleeps out a short backoff remainder. False when the wait is long (the - * wake run comes back for it) or the entry is no longer queued. - */ - private suspend fun waitUntilDue(initial: QueueEntry): Boolean { - var e = initial - while (true) { - val at = e.nextAttemptAt ?: return true - val remaining = at - now() - if (remaining <= 0) return true - if (remaining > RetryClassifier.IN_WORKER_BACKOFF_MAX_MS) return false - delay(min(remaining, WAIT_POLL_MS)) - e = store.load(entryId) ?: return false - if (e.state != EntryState.QUEUED && e.state != EntryState.RUNNING) return false + override fun connectivity(wifiOnly: Boolean): Connectivity { + connectivity = validateConnectivity(context, wifiOnly) + updateNotification() + return connectivity } - } - // MARK: - helpers for the transfers + override suspend fun foreground(entry: QueueEntry) { + showsNotification = entry.descriptor?.noNotification == false + startForeground() + } - internal fun now() = System.currentTimeMillis() + override fun progressStarted(id: String, total: Long, sent: Long) { + UploadProgress.add(id, total) + UploadProgress.set(id, sent) + } - /** - * Waits until the network fits the queue's wifi-only setting. Re-reads - * the settings and the entry at every poll. - */ - internal suspend fun waitForNetwork() { - while (true) { - val s = ops.settings() - if (s.paused) throw PausedException() - val entry = ops.latest(entryId, generation) - if (RetryClassifier.isExpired(now(), entry.expiresAt)) throw ExpiredException() - connectivity = validateConnectivity(context, s.wifiOnly) + override fun progress(id: String, sent: Long, total: Long) { + UploadProgress.set(id, sent) + EventReporter.progress(id, sent, total) updateNotification() - if (connectivity == Connectivity.Ok) return - delay(CONNECTIVITY_POLL_MS) } - } - - /** The descriptor's headers, the part's over them, and X-Request-Id over all. */ - internal fun headersFor(d: Descriptor, part: Part?, requestId: String): Map { - val withPart = if (part == null) d.headers else HeaderMap.merge(d.headers, part.headers) - return HeaderMap.merge(withPart, mapOf("X-Request-Id" to requestId)) - } - - internal fun policy(entry: QueueEntry) = - RetryClassifier.policy(ops.settings().retry, entry.descriptor?.retry) - /** - * A short backoff waits here, with the row still running and showing - * nextAttemptAt; a long one throws [BackoffException] to release the worker. - */ - internal suspend fun backoffOrRelease(policy: RetryClassifier.Policy, streak: Int, expiresAt: Long) { - val backoff = RetryClassifier.backoffMs(policy, streak) - val now = now() - if (backoff <= RetryClassifier.IN_WORKER_BACKOFF_MAX_MS) { - val wait = min(backoff, max(0L, expiresAt - now)) - ops.backingOff(entryId, generation, now + wait) - delay(wait) - return + override fun progressEnded(id: String, completed: Boolean) { + if (completed) UploadProgress.complete(id) else UploadProgress.remove(id) + EventReporter.flushProgress(id) + EventReporter.dropProgress(id) } - throw BackoffException(RetryClassifier.nextAttemptAt(now, backoff, expiresAt), streak) - } - - internal fun reportProgress(sent: Long, total: Long) { - UploadProgress.set(entryId, sent) - EventReporter.progress(entryId, sent, total) - updateNotification() - } - internal fun bodyFile(entry: QueueEntry): File? = store.bodyFile(entry) + override fun attempt(event: AttemptEvent) = EventReporter.attempt(event) - private fun endProgress(completed: Boolean) { - if (completed) UploadProgress.complete(entryId) else UploadProgress.remove(entryId) - EventReporter.flushProgress(entryId) - EventReporter.dropProgress(entryId) + override fun stoppedByTimeout() = stopReason == WorkInfo.STOP_REASON_TIMEOUT } // MARK: - notification @@ -269,7 +117,9 @@ open class EntryWorker(protected val context: Context, params: WorkerParameters) // v9 rules. A suppressed notification means no foreground mode. A denied // foreground start (API 31+, app in the background: the usual case for a // WorkManager relaunch) is not a failure; the transfer runs without - // foreground priority. Any other failure is logged and the run goes on. + // foreground priority, under JobScheduler's time limit (see + // [EntryRun.releaseAfterTimeout]). Any other failure is logged and the run + // goes on. private suspend fun startForeground() { if (!showsNotification) return try { diff --git a/android/src/main/java/ai/openspace/backgroundupload/EventJournal.kt b/android/src/main/java/ai/openspace/backgroundupload/EventJournal.kt index 03535d68..3f560947 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/EventJournal.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/EventJournal.kt @@ -4,6 +4,8 @@ import android.content.Context import com.facebook.react.bridge.WritableMap import com.google.gson.Gson import java.io.File +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit /** * The durable record of settled outcomes (completed, error, cancelled). A @@ -13,9 +15,15 @@ import java.io.File * record. * * [maxEntries] is a runaway guard: if nothing ever acknowledges, the oldest - * records are dropped. It only fires in that broken case. + * records are dropped, except those a row still names. It only fires in + * that broken case. */ -class EventJournal(private val dir: File, private val maxEntries: Int = MAX_ENTRIES) { +class EventJournal( + private val dir: File, + private val maxEntries: Int = MAX_ENTRIES, + /** Runs a task after a delay. Tests pass a manual one. */ + private val retryLater: (delayMs: Long, task: () -> Unit) -> Unit = ::onTimer, +) { /** RawResponse. [status] is null for a chunked completion. */ data class Response( @@ -33,8 +41,8 @@ class EventJournal(private val dir: File, private val maxEntries: Int = MAX_ENTR companion object { fun of(response: UploadResponse): Response { - val (body, truncated) = capBody(response.body) - return Response(response.code, response.headers, body, truncated) + val (body, cut) = BodyCap.cap(response.body, BodyCap.SETTLED_MAX_BYTES) + return Response(response.code, response.headers, body, response.truncated || cut) } /** A chunked completion: N parts, no one response. */ @@ -52,8 +60,9 @@ class EventJournal(private val dir: File, private val maxEntries: Int = MAX_ENTR val attempts: Int, val requestId: String?, /** - * How many times this outcome reached JS: 1 after a live emit, 0 when it - * was journaled with JS dead; +1 per later delivery (replay, re-emit). + * How many times this outcome reached a JS listener: 1 after a live + * emit, 0 when it was journaled with no listener; +1 per later delivery + * (replay, re-emit). [append] sets it. */ val deliveries: Int, /** The entry state this outcome puts it in. */ @@ -109,9 +118,11 @@ class EventJournal(private val dir: File, private val maxEntries: Int = MAX_ENTR const val KIND_ERROR = "error" const val KIND_CANCELLED = "cancelled" - /** 1 MB, the RawResponse cap. */ - const val MAX_BODY_CHARS = 1_048_576 const val MAX_ENTRIES = 1000 + + /** The first wait before a held record is written again. It doubles up to [RETRY_MAX_MS]. */ + const val RETRY_MS = 5_000L + const val RETRY_MAX_MS = 600_000L private val gson = Gson() // Event ids are UUIDs that native mints. ackEvents takes ids from JS, and @@ -120,12 +131,15 @@ class EventJournal(private val dir: File, private val maxEntries: Int = MAX_ENTR fun isValidEventId(id: String) = EVENT_ID.matches(id) - /** - * A char-count cap. It is not byte-exact: a cut on a byte boundary could - * split a surrogate pair. Returns the body and whether it was cut. - */ - fun capBody(body: String?, max: Int = MAX_BODY_CHARS): Pair = - if (body != null && body.length > max) body.substring(0, max) to true else body to false + private val timer by lazy { + Executors.newSingleThreadScheduledExecutor { r -> + Thread(r, "RNFileUploader.journal").apply { isDaemon = true } + } + } + + private fun onTimer(delayMs: Long, task: () -> Unit) { + timer.schedule(task, delayMs, TimeUnit.MILLISECONDS) + } @Volatile private var instance: EventJournal? = null @@ -141,53 +155,154 @@ class EventJournal(private val dir: File, private val maxEntries: Int = MAX_ENTR dir.mkdirs() } + /** + * The JS listener, set by [drain] and cleared by [stopListening]. While it + * is set, a new record starts at 1 delivery and the caller emits it live. + * While it is null, a record starts at 0 and the next drain delivers it. + * Both decisions take this object's lock, so a record is either in the + * drain or emitted live, never both and never neither. + */ + private var listener: Any? = null + + /** + * Records whose file write failed, by eventId. They count as journaled in + * every read and ack, and a timer writes them once the disk allows. They + * live only in memory: a process death loses them. + */ + private val held = LinkedHashMap() + private var retryScheduled = false + private fun fileFor(eventId: String) = File(dir, "$eventId.json") /** - * Never throws. The worker calls this right after the server accepted the - * request. A thrown IOException would look like a transient failure and - * re-send the request. Losing one record is the lesser harm, so a failure - * returns false and the caller goes on. + * Writes [record] with deliveries 1 when a listener is set, else 0, and + * returns it as written. The caller emits it only when deliveries > 0. + * Throws IOException when the write failed; nothing is kept then. + * + * [keep] names the records the prune must not delete (every eventId a row + * names). It runs only when the journal is over its cap, under this lock. + * Callers hold the store lock (lock order: store, then journal). */ @Synchronized - fun append(record: SettledRecord): Boolean { - val bounded = record.response?.let { r -> - val (body, truncated) = capBody(r.body) - if (truncated) record.copy(response = r.copy(body = body, bodyTruncated = true)) else record - } ?: record - val written = try { - AtomicFiles.writeText(fileFor(record.eventId), gson.toJson(bounded)) - true + fun append(record: SettledRecord, keep: () -> Set = { emptySet() }): SettledRecord { + val stamped = stamp(record) + AtomicFiles.writeText(fileFor(stamped.eventId), gson.toJson(stamped)) + pruneToMax(keep) + return stamped + } + + /** + * As [append], but never throws. A failed write holds the record in + * memory and a timer tries it again. For a worker's settle: the request + * already ran, and a thrown error would send it again. + */ + @Synchronized + fun appendOrHold(record: SettledRecord, keep: () -> Set = { emptySet() }): SettledRecord = + try { + append(record, keep) } catch (t: Throwable) { - Diag.error("journal append failed for ${record.eventId}", t) - false + Diag.error("journal append failed for ${record.eventId}; held in memory", t) + val stamped = stamp(record) + held[stamped.eventId] = stamped + scheduleRetry(RETRY_MS) + stamped + } + + private fun stamp(record: SettledRecord): SettledRecord { + val response = record.response?.let { r -> + val (body, cut) = BodyCap.cap(r.body, BodyCap.SETTLED_MAX_BYTES) + if (cut) r.copy(body = body, bodyTruncated = true) else r + } + return record.copy(deliveries = if (listener != null) 1 else 0, response = response) + } + + /** Whether [eventId] is held in memory, not yet on disk. */ + @Synchronized + fun isHeld(eventId: String) = eventId in held + + /** Writes every held record. Returns true when none is left. */ + @Synchronized + fun writeHeld(): Boolean { + val written = held.values.filter { r -> + runCatching { AtomicFiles.writeText(fileFor(r.eventId), gson.toJson(r)) }.isSuccess } - pruneToMax() - return written + written.forEach { held.remove(it.eventId) } + return held.isEmpty() } - // Prunes by file time (no parsing). Guarded for the same reason as append. - private fun pruneToMax() { + private fun scheduleRetry(delayMs: Long) { + if (retryScheduled) return + retryScheduled = true + retryLater(delayMs) { + synchronized(this) { + retryScheduled = false + if (!writeHeld()) scheduleRetry(minOf(delayMs * 2, RETRY_MAX_MS)) + } + } + } + + // Prunes by file time (no parsing), sparing the records rows name. Never + // throws: it only runs in the broken case where nothing acknowledges. + private fun pruneToMax(keep: () -> Set) { try { dir.listFiles { f -> f.name.endsWith(AtomicFiles.TMP_SUFFIX) }?.forEach { it.delete() } val files = dir.listFiles { f -> f.extension == "json" } ?: return if (files.size <= maxEntries) return - files.sortedBy { it.lastModified() }.take(files.size - maxEntries).forEach { it.delete() } + val named = keep() + files.filter { it.nameWithoutExtension !in named } + .sortedBy { it.lastModified() } + .take(files.size - maxEntries) + .forEach { it.delete() } } catch (t: Throwable) { Diag.error("journal prune failed", t) } } - /** Every record, oldest first. Corrupt files are skipped. */ + /** + * getUnacknowledgedEvents(): sets [owner] as the listener and returns + * every record, oldest first, each counted as one more delivery. One lock + * spans both, see [listener]. + */ + @Synchronized + fun drain(owner: Any, isActive: () -> Boolean = { true }): List { + // [isActive] is read under this lock. A module torn down before its + // queued drain runs does not become the listener, and counts nothing. + if (!isActive()) return emptyList() + listener = owner + return unacknowledged().mapNotNull { incrementDeliveries(it.eventId) } + } + + /** Clears the listener, only when [owner] set it: a reload builds the next module before it tears down this one. */ + @Synchronized + fun stopListening(owner: Any) { + if (listener === owner) listener = null + } + + @Synchronized + fun isListening() = listener != null + + /** The listener a live settled event goes to: the module whose JS drained last. */ + @Synchronized + fun listener(): Any? = listener + + /** + * A re-emit of a journaled outcome (same-id rule 7): one more delivery, + * returned for a live emit. Null when no listener is set (the next drain + * delivers it) or the record is gone. + */ + @Synchronized + fun redeliver(eventId: String): SettledRecord? = + if (listener == null) null else incrementDeliveries(eventId) + + /** Every record, oldest first, the held ones included. Corrupt files are skipped. */ @Synchronized fun unacknowledged(): List = - (dir.listFiles { f -> f.extension == "json" } ?: emptyArray()) - .mapNotNull { read(it) } - .sortedBy { it.at } + ((dir.listFiles { f -> f.extension == "json" } ?: emptyArray()).mapNotNull { read(it) } + held.values) + .sortedWith(compareBy { it.at }.thenBy { it.eventId }) @Synchronized fun find(eventId: String): SettledRecord? = - if (isValidEventId(eventId)) read(fileFor(eventId)) else null + if (isValidEventId(eventId)) held[eventId] ?: read(fileFor(eventId)) else null @Synchronized fun forEntry(id: String): List = unacknowledged().filter { it.id == id } @@ -201,6 +316,10 @@ class EventJournal(private val dir: File, private val maxEntries: Int = MAX_ENTR fun incrementDeliveries(eventId: String): SettledRecord? { val record = find(eventId) ?: return null val next = record.copy(deliveries = record.deliveries + 1) + if (eventId in held) { + held[eventId] = next + return next + } try { AtomicFiles.writeText(fileFor(eventId), gson.toJson(next)) } catch (t: Throwable) { @@ -212,7 +331,16 @@ class EventJournal(private val dir: File, private val maxEntries: Int = MAX_ENTR /** Idempotent. Unknown and malformed ids are ignored. */ @Synchronized fun ack(eventIds: List) { - eventIds.filter { isValidEventId(it) }.forEach { fileFor(it).delete() } + eventIds.filter { isValidEventId(it) }.forEach { + held.remove(it) + fileFor(it).delete() + } + } + + /** Acks every record of entry [id]: cancel() of a settled entry forgets its outcomes. */ + @Synchronized + fun ackEntry(id: String) { + ack(forEntry(id).map { it.eventId }) } @Suppress("SENSELESS_COMPARISON", "USELESS_ELVIS") diff --git a/android/src/main/java/ai/openspace/backgroundupload/EventReporter.kt b/android/src/main/java/ai/openspace/backgroundupload/EventReporter.kt index 68121854..ea3cc62a 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/EventReporter.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/EventReporter.kt @@ -6,16 +6,13 @@ import com.facebook.react.bridge.Arguments interface QueueEvents { fun state(row: RequestRow) - /** The caller journaled [record] first. */ - fun settled(record: EventJournal.SettledRecord) - /** - * Whether a live emit can reach a JS listener now. A record journaled - * while no listener is there (JS dead, or alive but not yet subscribed) - * starts at 0 deliveries, so its first real delivery (the replay) counts - * as 1. + * The caller journaled [record] first, and emits it only when its + * deliveries is above 0: [EventJournal] decides that under its lock. + * [listener] is [EventJournal.listener]: the module whose JS drained, so + * the delivery the journal counted goes to that JS. */ - fun canDeliver(): Boolean + fun settled(record: EventJournal.SettledRecord, listener: Any) } /** @@ -40,13 +37,13 @@ object EventReporter : QueueEvents { module.emitState(JsonBridge.toWritableMap(row.toMap())) } - override fun settled(record: EventJournal.SettledRecord) { - val module = UploaderModule.instance ?: return + // Not UploaderModule.instance: a reload sets that before the new JS + // subscribes, and the journal counted this delivery for the listener. + override fun settled(record: EventJournal.SettledRecord, listener: Any) { + val module = listener as? UploaderModule ?: return module.emitSettled(record.toWritableMap()) } - override fun canDeliver(): Boolean = UploaderModule.instance?.listening == true - /** Moves the row's bytesSent in memory and emits through the throttle. */ fun progress(id: String, sent: Long, total: Long) { RequestIndex.shared.setBytes(id, sent) diff --git a/android/src/main/java/ai/openspace/backgroundupload/JsonBridge.kt b/android/src/main/java/ai/openspace/backgroundupload/JsonBridge.kt index a84b8661..fe84fbe7 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/JsonBridge.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/JsonBridge.kt @@ -6,31 +6,29 @@ import com.facebook.react.bridge.ReadableMap import com.facebook.react.bridge.ReadableType import com.facebook.react.bridge.WritableArray import com.facebook.react.bridge.WritableMap -import com.google.gson.GsonBuilder -import com.google.gson.JsonArray +import com.google.gson.Gson import com.google.gson.JsonElement -import com.google.gson.JsonNull -import com.google.gson.JsonObject import com.google.gson.JsonParser import com.google.gson.JsonPrimitive +import com.google.gson.Strictness +import com.google.gson.stream.JsonReader +import com.google.gson.stream.JsonToken +import java.io.StringReader import kotlin.math.abs import kotlin.math.floor /** - * Moves values between three forms: the bridge (ReadableMap, WritableMap), - * plain Kotlin values (Map, List, String, Double, Boolean, null), and JSON - * text. `vars` and `data` are stored as JSON text. + * Moves values between the bridge (ReadableMap, WritableMap), plain Kotlin + * values (Map, List, String, Double, Boolean, null), and JSON text. `vars` + * and `data` cross from JS as JSON text and are stored as it came; native + * parses them back only to hand objects to JS. * - * Numbers: RN gives every JS number to Kotlin as a Double. A Double with no - * fraction is written as an integer, so `{ n: 1 }` becomes `{"n":1}`, as - * JSON.stringify writes it, not `{"n":1.0}`. - * - * Key order: the bridge does not keep the JS key order. Object keys are - * written sorted, so the same object always gives the same text. The - * same-body check compares that text. + * Numbers: RN gives every JS number to Kotlin as a Double. [numberText] + * writes a Double with no fraction as an integer, so a header value 5 is + * "5", as JSON.stringify writes it, not "5.0". */ object JsonBridge { - private val gson = GsonBuilder().serializeNulls().disableHtmlEscaping().create() + private val gson = Gson() // 2^53. Above this a Double can not hold every integer, so it keeps the // Double form. @@ -71,12 +69,20 @@ object JsonBridge { } } - /** Plain values to JSON text. */ - fun toJson(value: Any?): String = gson.toJson(toElement(value)) - /** JSON text to plain values. Throws on malformed text. Numbers come back as Double. */ fun parse(json: String): Any? = fromElement(JsonParser.parseString(json)) + /** + * Whether [text] is one strict JSON value, as JSON.stringify writes it. + * Any top-level value counts, so "null" is valid. Lenient forms (single + * quotes, bare keys, trailing text) are not. + */ + fun isJson(text: String): Boolean = runCatching { + val reader = JsonReader(StringReader(text)).apply { setStrictness(Strictness.STRICT) } + gson.getAdapter(JsonElement::class.java).read(reader) + reader.peek() == JsonToken.END_DOCUMENT + }.getOrDefault(false) + /** A number as JSON would print it: an integer when it has no fraction. */ fun numberText(d: Double): String = gson.toJson(number(d)) @@ -84,24 +90,6 @@ object JsonBridge { if (d.isFinite() && d == floor(d) && abs(d) < MAX_SAFE_INTEGER) JsonPrimitive(d.toLong()) else JsonPrimitive(d) - private fun toElement(value: Any?): JsonElement = when (value) { - null -> JsonNull.INSTANCE - is Boolean -> JsonPrimitive(value) - is Double -> number(value) - is Float -> number(value.toDouble()) - is Int, is Long, is Short, is Byte -> JsonPrimitive((value as Number).toLong()) - is Number -> JsonPrimitive(value) - is String -> JsonPrimitive(value) - is Map<*, *> -> JsonObject().apply { - value.entries - .sortedBy { it.key.toString() } - .forEach { (k, v) -> add(k.toString(), toElement(v)) } - } - is Iterable<*> -> JsonArray().apply { value.forEach { add(toElement(it)) } } - is Array<*> -> JsonArray().apply { value.forEach { add(toElement(it)) } } - else -> JsonPrimitive(value.toString()) - } - private fun fromElement(element: JsonElement): Any? = when { element.isJsonNull -> null element.isJsonObject -> LinkedHashMap().apply { diff --git a/android/src/main/java/ai/openspace/backgroundupload/LegacyImport.kt b/android/src/main/java/ai/openspace/backgroundupload/LegacyImport.kt index d00e97c5..da5b19ab 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/LegacyImport.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/LegacyImport.kt @@ -33,10 +33,13 @@ object LegacyImport { private val gson = Gson() /** Returns true when the import ran at this launch (no marker yet). */ - fun runOnce(context: Context, store: QueueStore): Boolean { - val marker = File(QueueStore.rootDir(context), MARKER) + fun runOnce(context: Context, store: QueueStore): Boolean = + runOnce(File(QueueStore.rootDir(context), MARKER), File(context.filesDir, V9_JOURNAL_DIR), store) + + /** [runOnce] over plain files, for the JVM tests. */ + internal fun runOnce(marker: File, v9Dir: File, store: QueueStore): Boolean { if (marker.exists()) return false - val complete = import(File(context.filesDir, V9_JOURNAL_DIR), store) + val complete = import(v9Dir, store) if (complete) { runCatching { AtomicFiles.writeText(marker, "1") } .onFailure { Diag.error("could not write the v9 import marker", it) } diff --git a/android/src/main/java/ai/openspace/backgroundupload/QueueController.kt b/android/src/main/java/ai/openspace/backgroundupload/QueueController.kt index 810895d7..c72cb0fe 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/QueueController.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/QueueController.kt @@ -54,7 +54,7 @@ class QueueController( } finally { if (pre != null && !preUsed) discard(p.id, pre) } - result.reEmit?.let { events.settled(it) } + result.reEmit?.let { record -> journal.listener()?.let { events.settled(record, it) } } result.entry?.let { entry -> scheduleRun(entry) events.state(entry.toRow()) @@ -65,7 +65,7 @@ class QueueController( private fun preStage(p: EntryParsing.Parsed): PreStaged? { val generation = store.locked { val existing = store.load(p.id) - val v9 = if (existing == null) store.legacyManifest(p.id) else null + val v9 = v9ManifestFor(existing, p.id) val action = EnqueueRules.decide(existing, v9, p.descriptor) { journal.find(it) != null } EnqueueRules.preStageGeneration(existing, action, p.descriptor) } ?: return null @@ -84,7 +84,7 @@ class QueueController( /** Runs under the store lock. [preStaged] returns the body staged outside the lock for a generation, if any. */ private fun decideAndCommit(p: EntryParsing.Parsed, preStaged: (generation: Int) -> BodyStaging.Staged?): Enqueued { val existing = store.load(p.id) - val v9 = if (existing == null) store.legacyManifest(p.id) else null + val v9 = v9ManifestFor(existing, p.id) val s = settings.load() val now = clock() val dir = store.entryDir(p.id) @@ -93,7 +93,8 @@ class QueueController( QueueException.E_RUNNING, "entry '${p.id}' is running; a different body is accepted once it stops", ) - is EnqueueRules.Action.ReEmit -> Enqueued(null, journal.incrementDeliveries(action.eventId)) + // With no listener yet, the next drain delivers it. + is EnqueueRules.Action.ReEmit -> Enqueued(null, journal.redeliver(action.eventId)) EnqueueRules.Action.Resume -> { val next = EnqueueRules.resumed(existing!!, p, s.paused, s.headerGeneration, now) saveOrThrow(next) @@ -108,8 +109,8 @@ class QueueController( val parts = incoming?.let { EnqueueRules.adoptedParts(action.manifest, it) } // The same parts resume over the v9 blob, as a same-body enqueue does. val keepOwned = incoming != null && ChunkedParts.sameParts(action.manifest.parts, incoming) - val staged = stageOrThrow(p.descriptor.copy(parts = parts), dir, 1, store.blobFile(p.id), keepOwned) - commit(EnqueueRules.created(p, staged, parts, s.paused, s.headerGeneration, now)) + val staged = stageOrThrow(p.descriptor.copy(parts = parts), dir, action.generation, store.blobFile(p.id), keepOwned) + commit(EnqueueRules.adopted(p, staged, parts, existing, action.generation, s.paused, s.headerGeneration, now)) } EnqueueRules.Action.Replace -> { val old = existing!! @@ -122,6 +123,10 @@ class QueueController( } } + /** A v9 manifest counts only where no v10 entry owns the id, or the owner is its legacy row. */ + private fun v9ManifestFor(existing: QueueEntry?, id: String): LegacyManifest? = + if (existing == null || existing.legacy) store.legacyManifest(id) else null + private fun stageOrThrow( d: Descriptor, dir: java.io.File, @@ -183,27 +188,59 @@ class QueueController( /** * Live: journal a 'cancelled' (user) outcome, then forget after its ack. - * Settled: forget now, row and bytes. Unknown: no-op. Unacked records of a - * forgotten entry are kept, so a handler that has not run yet still runs. + * Settled (or legacy): forget now, row, bytes, and its unacknowledged + * outcomes. Unknown: no-op. + * + * A failed journal write rejects E_STORAGE and changes nothing: the entry + * goes on, and JS can call cancel() again. + * + * A failed entry save after the journal write also rejects E_STORAGE, but + * the cancel is durable: the work is stopped and the record is emitted. + * Its ack, the next cancel(), a worker's begin or settle, or the boot + * sweep applies it. A live entry that already has a record of its own + * generation gets that record applied, not a second outcome. */ fun cancel(id: String) { - val settled = store.locked { - val e = store.load(id) ?: return@locked null - if (!e.isLive || e.legacy) { - store.remove(id) - return@locked null + var stop = false + var journaled: EventJournal.SettledRecord? = null + var saved: QueueEntry? = null + try { + store.locked { + stop = true // unknown or settled: stop any stray work, as before + val e = store.load(id) ?: return@locked + if (!e.isLive || e.legacy) { + journal.ackEntry(id) + store.remove(id) + return@locked + } + stop = false // a live entry: only once an outcome is journaled + val now = clock() + EntryTransitions.journaledSettle(e, journal.forEntry(id), now)?.let { + stop = true + saveOrThrow(it.entry) + journal.ack(it.extraEventIds) + saved = it.entry + return@locked + } + val record = cancelledRecord(e, now) + journaled = try { + journal.append(record) { store.referencedEventIds() + record.eventId } + } catch (error: IOException) { + throw QueueException(QueueException.E_STORAGE, "could not journal the cancel: ${error.message}") + } + stop = true + val next = EntryTransitions.toSettled(e, EntryState.CANCELLED, record.eventId, e.bytesSent, now) + saveOrThrow(next) + saved = next } - val now = clock() - val record = cancelledRecord(e, now) - journal.append(record) - val next = EntryTransitions.toSettled(e, EntryState.CANCELLED, record.eventId, e.bytesSent, now) - saveOrThrow(next) - next to record - } - scheduler.cancel(id) - settled?.let { (entry, record) -> - if (record.deliveries > 0) events.settled(record) - events.state(entry.toRow()) + } finally { + // Once an outcome is journaled, the worker must stop even when the + // save failed: a running request would settle a second outcome. + if (stop) scheduler.cancel(id) + journaled?.let { record -> + if (record.deliveries > 0) journal.listener()?.let { events.settled(record, it) } + } + saved?.let { events.state(it.toRow()) } } } @@ -215,7 +252,7 @@ class QueueController( at = now, attempts = e.attempts, requestId = e.lastRequestId, - deliveries = if (events.canDeliver()) 1 else 0, + deliveries = 0, // the journal sets it state = EntryState.CANCELLED.wire, bytesSent = e.bytesSent, totalBytes = e.totalBytes, @@ -263,9 +300,12 @@ class QueueController( // MARK: - journal - /** Every unacknowledged outcome, each counted as one more delivery. */ - fun unacknowledged(): List = - journal.unacknowledged().mapNotNull { journal.incrementDeliveries(it.eventId) } + /** + * getUnacknowledgedEvents(): [listener] becomes the JS listener, and every + * unacknowledged outcome is returned, each counted as one more delivery. + */ + fun unacknowledged(listener: Any, isActive: () -> Boolean = { true }): List = + journal.drain(listener, isActive) /** * Removes the records. An acked completed or cancelled outcome of the @@ -286,7 +326,7 @@ class QueueController( val record = journal.find(eventId) ?: return@forEach var e = store.load(record.id) if (e != null && e.isLive && !e.legacy && e.generation == record.generation) { - val next = EntryTransitions.toSettled(e, stateOf(record), record.eventId, record.bytesSent, clock()) + val next = EntryTransitions.toSettled(e, EntryTransitions.stateOf(record), record.eventId, record.bytesSent, clock()) if (!trySave(next)) return@forEach repaired += next e = next @@ -303,9 +343,6 @@ class QueueController( repaired.filter { it.id !in forgotten }.forEach { events.state(it.toRow()) } } - private fun stateOf(record: EventJournal.SettledRecord): EntryState = - EntryState.values().firstOrNull { it.wire == record.state } ?: EntryState.ERROR - // MARK: - boot sweep /** @@ -336,12 +373,11 @@ class QueueController( if (e.legacy || isWorkerRunning(e.id)) continue val own = records[e.id].orEmpty().filter { it.generation == e.generation } if (e.isLive) { - val latest = own.maxByOrNull { it.at } - if (latest != null) { - val next = EntryTransitions.toSettled(e, stateOf(latest), latest.eventId, latest.bytesSent, now) - if (trySave(next)) { - journal.ack(own.filter { it !== latest }.map { it.eventId }) - changed += next + val journaled = EntryTransitions.journaledSettle(e, own, now) + if (journaled != null) { + if (trySave(journaled.entry)) { + journal.ack(journaled.extraEventIds) + changed += journaled.entry } continue } diff --git a/android/src/main/java/ai/openspace/backgroundupload/QueueStore.kt b/android/src/main/java/ai/openspace/backgroundupload/QueueStore.kt index 1beb0870..3db76532 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/QueueStore.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/QueueStore.kt @@ -122,10 +122,17 @@ class QueueStore(private val dir: File, private val index: RequestIndex = Reques (dir.listFiles { f -> f.isDirectory } ?: emptyArray()) .mapNotNull { d -> File(d, ENTRY_FILE).takeIf { it.exists() }?.let { read(it) } } - /** The v9 chunked manifest for [id], when the directory has no v10 entry. */ + /** Every eventId a row names. The journal prune spares them. */ + @Synchronized + fun referencedEventIds(): Set = all().mapNotNull { it.settledEventId }.toSet() + + /** + * The v9 chunked manifest in [id]'s directory. The caller asks only when + * there is no v10 entry or the entry is a legacy row: a v10 entry + * prunes the manifest when it adopts it. + */ @Synchronized fun legacyManifest(id: String): LegacyManifest? { - if (entryFile(id).exists()) return null val file = File(entryDir(id), V9_MANIFEST_FILE) if (!file.exists()) return null val parsed = runCatching { gson.fromJson(file.readText(), LegacyManifest::class.java) }.getOrNull() @@ -181,15 +188,14 @@ class QueueStore(private val dir: File, private val index: RequestIndex = Reques } } -/** A v9 chunked manifest. The field names are the v9 ones. */ +/** + * A v9 chunked manifest, only the fields v10 reads. The field names are the + * v9 ones; Gson skips the rest of the file. + */ data class LegacyManifest( val id: String, - val sourcePath: String, val parts: List, val accept: List, - val expiresAt: Long, - val noNotification: Boolean, - val createdAt: Long, ) { companion object { @Suppress("SENSELESS_COMPARISON") diff --git a/android/src/main/java/ai/openspace/backgroundupload/RequestIndex.kt b/android/src/main/java/ai/openspace/backgroundupload/RequestIndex.kt index a59708be..be3b7b0d 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/RequestIndex.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/RequestIndex.kt @@ -36,8 +36,6 @@ class RequestIndex { rows.remove(id) } - fun get(id: String): RequestRow? = rows[id] - /** Oldest first, then by id. */ fun snapshot(): List = rows.values.sortedWith(compareBy { it.createdAt }.thenBy { it.id }) diff --git a/android/src/main/java/ai/openspace/backgroundupload/RetryClassifier.kt b/android/src/main/java/ai/openspace/backgroundupload/RetryClassifier.kt index cfd01781..d570ec61 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/RetryClassifier.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/RetryClassifier.kt @@ -60,9 +60,11 @@ object RetryClassifier { else -> Verdict.Terminal("unknown", error.message ?: error.javaClass.simpleName) } - /** The live attempt's errorKind for a failure: network, file, or unknown. */ - fun failureKind(error: Throwable, fileExists: Boolean): String = - UploadOutcome.errorKind(error, fileExists) + /** + * The live attempt's errorKind for a transport failure, from its + * [classifyFailure] verdict: file, unknown, or network for a transient one. + */ + fun failureKind(verdict: Verdict): String = (verdict as? Verdict.Terminal)?.errorKind ?: "network" fun isExpired(now: Long, expiresAt: Long) = now >= expiresAt diff --git a/android/src/main/java/ai/openspace/backgroundupload/TransferHost.kt b/android/src/main/java/ai/openspace/backgroundupload/TransferHost.kt new file mode 100644 index 00000000..d7556a3d --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/TransferHost.kt @@ -0,0 +1,34 @@ +package ai.openspace.backgroundupload + +/** + * What a run needs from the platform: time, the network, progress, the + * notification, and live attempt events. [EntryWorker] is the real one. The + * JVM tests pass a fake with a scripted [send] and a manual clock, so every + * branch of [EntryRun] runs with no device. + */ +internal interface TransferHost { + fun now(): Long + + suspend fun sleep(ms: Long) + + /** One request through the library-wide cap of 4. Throws on a transport failure. */ + suspend fun send(request: TransferRequest, onProgress: (Long) -> Unit): UploadResponse + + /** The network state for the queue's wifi-only setting. The notification shows it. */ + fun connectivity(wifiOnly: Boolean): Connectivity + + /** Foreground mode for an entry that shows the notification. Never throws. */ + suspend fun foreground(entry: QueueEntry) + + fun progressStarted(id: String, total: Long, sent: Long) + + fun progress(id: String, sent: Long, total: Long) + + /** The trailing progress event, then the progress state is dropped. */ + fun progressEnded(id: String, completed: Boolean) + + fun attempt(event: AttemptEvent) + + /** Whether the system stopped this run at its time limit (JobScheduler, about 10 minutes). */ + fun stoppedByTimeout(): Boolean +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/UploadOutcome.kt b/android/src/main/java/ai/openspace/backgroundupload/UploadOutcome.kt index 9c22321a..f1edc40a 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/UploadOutcome.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/UploadOutcome.kt @@ -1,7 +1,5 @@ package ai.openspace.backgroundupload -import java.io.IOException - // Pure classification of terminal upload outcomes. Kept free of Android/React // types so it can be unit-tested on a plain JVM — this is the highest-consequence // logic in the uploader (it decides success vs failure), so it's covered directly. @@ -26,14 +24,4 @@ object UploadOutcome { rule.status == code && (rule.bodyIncludes == null || body?.contains(rule.bodyIncludes) == true) } - - // Classify a thrown error into a stable kind for the JS layer. `fileExists` - // is passed in (not read here) to keep this pure; callers should default it to - // true when the existence check itself fails, so a flaky file probe reads as a - // retryable network error rather than a terminal "file gone". - fun errorKind(error: Throwable, fileExists: Boolean): String = when { - error is IOException && !fileExists -> "file" - error is IOException -> "network" - else -> "unknown" - } } diff --git a/android/src/main/java/ai/openspace/backgroundupload/UploadUtils.kt b/android/src/main/java/ai/openspace/backgroundupload/UploadUtils.kt index 6d84b4f0..154cac08 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/UploadUtils.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/UploadUtils.kt @@ -26,10 +26,12 @@ private const val PROGRESS_INTERVAL = 500 // milliseconds private const val RANGE_COPY_BUFFER = 64 * 1024 +/** [truncated] when the body passed [BodyCap.SETTLED_MAX_BYTES] and the rest was not read. */ data class UploadResponse( val code: Int, val body: String, - val headers: Map + val headers: Map, + val truncated: Boolean = false, ) /** One request as the worker sends it. [body] is null only for GET and DELETE with no body. */ @@ -104,13 +106,17 @@ private suspend fun awaitResponse(client: OkHttpClient, request: Request): Uploa override fun onResponse(call: Call, response: Response) { val result = try { response.use { res -> // close the response asap + // The body unchanged: an empty body stays empty. A substituted + // reason phrase would make accept `bodyIncludes` rules match text + // the server never sent. The cap applies while it streams in. + val body = res.body?.let { + BodyCap.read(it.source(), BodyCap.SETTLED_MAX_BYTES, it.contentType()?.charset() ?: Charsets.UTF_8) + } UploadResponse( res.code, - // The body unchanged: an empty body stays empty. A substituted - // reason phrase would make accept `bodyIncludes` rules match text - // the server never sent. - res.body?.string().orEmpty(), - res.headers.toMultimap().mapValues { it.value.joinToString(", ") } + body?.text.orEmpty(), + res.headers.toMultimap().mapValues { it.value.joinToString(", ") }, + body?.truncated ?: false, ) } } catch (e: IOException) { diff --git a/android/src/main/java/ai/openspace/backgroundupload/UploadWorker.kt b/android/src/main/java/ai/openspace/backgroundupload/UploadWorker.kt index ceb1ee1c..a49f7548 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/UploadWorker.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/UploadWorker.kt @@ -2,99 +2,55 @@ package ai.openspace.backgroundupload import android.content.Context import androidx.work.WorkerParameters -import kotlinx.coroutines.CancellationException -import kotlinx.coroutines.sync.withPermit import okhttp3.RequestBody import java.io.File -import java.util.UUID /** The WorkManager class for a single-body entry. The run is [EntryWorker]'s. */ class UploadWorker(context: Context, params: WorkerParameters) : EntryWorker(context, params) /** * One request with one body: none, JSON, multipart, or a copied file. One - * attempt at a time, until a verdict ends it: + * attempt at a time ([EntryRun.attempt]), until a verdict ends it: * accepted → Completed; auth → re-issue (newer headers) or park; * transient → back off (short: here; long: release); terminal → Failed. */ -internal class SimpleTransfer(private val host: EntryWorker) { +internal class SimpleTransfer(private val run: EntryRun) { suspend fun run(start: QueueEntry): Settlement { val d0 = start.descriptor!! - val file = host.bodyFile(start) + val file = run.store.bodyFile(start) // The payload probe: a staged body that is gone can never be sent. if (file != null && !file.exists()) { return Settlement.Failed("file", "the staged request body is missing", null, null, d0.reportUrl, d0.method) } val total = start.body?.totalBytes ?: 0L - UploadProgress.add(host.entryId, total) + run.progressStarted(total, 0L) var streak = start.backoffStreak while (true) { - val latest = host.ops.latest(host.entryId, host.generation) - if (RetryClassifier.isExpired(host.now(), latest.expiresAt)) throw EntryWorker.ExpiredException() - host.waitForNetwork() + val latest = run.ops.latest(run.entryId, run.generation) + if (RetryClassifier.isExpired(run.host.now(), latest.expiresAt)) throw EntryRun.ExpiredException() + run.waitForNetwork() - val requestId = UUID.randomUUID().toString() - val entry = host.ops.recordAttempt(host.entryId, host.generation, requestId) - // The generation of the headers this attempt sends: both come from the same entry. - val headerGeneration = entry.headerGeneration - val d = entry.descriptor!! - val url = d.url!! - val policy = host.policy(entry) - - val response = try { - transferSemaphore.withPermit { - okhttpSend( - uploadHttpClient, - TransferRequest(url, d.method, host.headersFor(d, null, requestId), requestBody(file, d.method)), - ) { sent -> host.reportProgress(sent, total) } - } - } catch (error: CancellationException) { - throw error - } catch (error: Throwable) { - host.reportProgress(0L, total) - val fileExists = file == null || runCatching { file.exists() }.getOrDefault(true) - val message = error.message ?: error.javaClass.simpleName - EventReporter.attempt( - AttemptEvent.ofFailure( - entry, requestId, url, null, RetryClassifier.failureKind(error, fileExists), message, host.now(), - ), - ) - when (val verdict = RetryClassifier.classifyFailure(error, fileExists)) { - is RetryClassifier.Verdict.Terminal -> - return Settlement.Failed(verdict.errorKind, verdict.message, null, null, url, d.method) - else -> { - streak++ - host.backoffOrRelease(policy, streak, entry.expiresAt) - continue - } - } - } - - val verdict = RetryClassifier.classifyResponse(response.code, response.body, d.accept, policy.exempt) - EventReporter.attempt( - AttemptEvent.ofResponse( - entry, requestId, url, null, response, verdict == RetryClassifier.Verdict.Accepted, host.now(), - ), + val a = run.attempt( + partIndex = null, + body = { d, _ -> requestBody(file, d.method) }, + onProgress = { sent -> run.reportProgress(sent, total) }, + fileExists = { file == null || file.exists() }, ) - when (verdict) { - RetryClassifier.Verdict.Accepted -> return Settlement.Completed(response, url, d.method) - RetryClassifier.Verdict.Auth -> { - // updateHeaders() landed while this attempt was in flight: re-issue now. - if (host.ops.hasNewerHeaders(host.entryId, host.generation, headerGeneration)) { - streak = 0 - continue - } - throw EntryWorker.ParkException(headerGeneration) + when (val r = a.result) { + is EntryRun.AttemptResult.Accepted -> return Settlement.Completed(r.response, a.url, a.method) + is EntryRun.AttemptResult.Auth -> { + if (!r.reissue) throw EntryRun.ParkException(r.headerGeneration) + streak = 0 // updateHeaders() landed while this attempt was in flight: re-issue now. } - RetryClassifier.Verdict.Transient -> { - host.reportProgress(0L, total) + EntryRun.AttemptResult.Transient -> { + run.reportProgress(0L, total) streak++ - host.backoffOrRelease(policy, streak, entry.expiresAt) + run.backoffOrRelease(a.policy, streak, a.entry.expiresAt) } - is RetryClassifier.Verdict.Terminal -> - return Settlement.Failed("http", verdict.message, response, null, url, d.method) + is EntryRun.AttemptResult.Terminal -> + return Settlement.Failed(r.errorKind, r.message, r.response, null, a.url, a.method, bytesSent = run.liveBytes) } } } diff --git a/android/src/main/java/ai/openspace/backgroundupload/UploaderModule.kt b/android/src/main/java/ai/openspace/backgroundupload/UploaderModule.kt index 34f6fd2c..83ad4505 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/UploaderModule.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/UploaderModule.kt @@ -53,16 +53,6 @@ class UploaderModule(context: ReactApplicationContext) : private val scheduler = WorkManagerScheduler(context) private val controller = QueueController(store, journal, settings, EventReporter, scheduler) - /** - * True once JS subscribed to onSettled. JS subscribes, then calls - * getUnacknowledgedEvents() at once, so that first call is the signal. - * Until then a settled emit reaches no listener, so it must not count as - * a delivery. Cleared on teardown. - */ - @Volatile - var listening = false - private set - // The v9 import, then the v9 work cancel, then the boot sweep. // getRequests() waits for the import only (file reads and row saves), so // the first call after an upgrade already shows the legacy rows. The @@ -84,10 +74,17 @@ class UploaderModule(context: ReactApplicationContext) : } } + // Set by invalidate(). A drain still queued on the executor then does not + // make this dead module the journal's listener. + @Volatile + private var invalidated = false + override fun invalidate() { // A reload constructs the replacement before tearing this one down, so - // only clear the pointer when it still refers to us. - listening = false + // only clear the pointer (and the listener) when it still refers to us. + // The flag goes first: the drain reads it under the journal lock. + invalidated = true + journal.stopListening(this) if (instance === this) instance = null super.invalidate() } @@ -195,16 +192,16 @@ class UploaderModule(context: ReactApplicationContext) : } /** - * Every unacknowledged outcome, each counted as one more delivery. Sets - * [listening] first: an outcome settled from here on is emitted live with - * deliveries 1; one settled before it was journaled with 0, and this - * drain makes it 1. + * Every unacknowledged outcome, each counted as one more delivery. JS + * subscribes to onSettled, then calls this at once, so this call makes + * the module the journal's listener. The flip and the scan share the + * journal lock: an outcome settled before it is in this drain (journaled + * at 0, returned at 1); one settled after is emitted live at 1. */ override fun getUnacknowledgedEvents(promise: Promise) { - listening = true onQueue(promise) { val out = Arguments.createArray() - controller.unacknowledged().forEach { out.pushMap(it.toWritableMap()) } + controller.unacknowledged(this) { !invalidated }.forEach { out.pushMap(it.toWritableMap()) } out } } diff --git a/android/src/main/java/ai/openspace/backgroundupload/WorkerOps.kt b/android/src/main/java/ai/openspace/backgroundupload/WorkerOps.kt index 4f1fe2ef..0420fada 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/WorkerOps.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/WorkerOps.kt @@ -15,6 +15,7 @@ sealed class Settlement { override val method: String, ) : Settlement() + /** [bytesSent] is the live bytes of a simple entry's last attempt; null keeps the stored value. */ data class Failed( val errorKind: String, val message: String, @@ -22,6 +23,7 @@ sealed class Settlement { val partIndex: Int?, override val url: String, override val method: String, + val bytesSent: Long? = null, ) : Settlement() } @@ -49,19 +51,36 @@ class WorkerOps( ) { enum class ParkResult { PARKED, REISSUE, NOT_OWNED } - /** Takes a queued entry. Null when there is nothing to run. */ + /** + * Takes a queued entry. Null when there is nothing to run. + * + * A journal record of the entry's own generation means a settle was + * journaled and its store write was lost (a process death between the + * two, or a failed save). WorkManager can run the entry again before any + * boot sweep. Then begin applies the record, as the sweep does, and does + * not send the request again. + */ fun begin(id: String): QueueEntry? { val now = clock() - var changed = false - val entry = store.compute(id) { e -> - if (e != null && !e.legacy && (e.state == EntryState.QUEUED || e.state == EntryState.RUNNING)) { - changed = true - EntryTransitions.toRunning(e, now) - } else e + var row: QueueEntry? = null + var taken: QueueEntry? = null + store.locked { + val e = store.load(id) ?: return@locked + if (e.legacy || (e.state != EntryState.QUEUED && e.state != EntryState.RUNNING)) return@locked + val journaled = EntryTransitions.journaledSettle(e, journal.forEntry(id), now) + if (journaled != null) { + store.save(journaled.entry) + journal.ack(journaled.extraEventIds) + row = journaled.entry + return@locked + } + val next = EntryTransitions.toRunning(e, now) + store.save(next) + row = next + taken = next } - if (entry == null || entry.state != EntryState.RUNNING) return null - if (changed) events.state(entry.toRow()) - return entry + row?.let { events.state(it.toRow()) } + return taken } /** The stored entry, while this run still owns it. */ @@ -126,73 +145,88 @@ class WorkerOps( } /** - * Journal, then transition, then emit. When a cancel or a replace landed - * first, the record is an orphan: it is acked at once and nothing is - * emitted. Returns whether this run's outcome stands. + * Journal, then transition, then emit, all under the store lock, so a + * cancel, pause, or replace lands either before (this run's outcome is + * dropped) or after. Returns whether this run's outcome stands. + * + * A failed journal write holds the record in memory ([EventJournal.appendOrHold]): + * the request already ran, and a retry would send it twice. A failed + * store write leaves the record for the ack, the next [begin], or the + * boot sweep to apply. + * + * A record of this generation already in the journal (a cancel whose + * entry save failed) wins: it is applied, as [begin] does, and this run's + * outcome is dropped. One life has one outcome. */ fun settle(id: String, generation: Int, s: Settlement): Boolean { - val e = store.load(id) - if (!EntryTransitions.canSettle(e, generation)) return false - e!! val now = clock() val completed = s is Settlement.Completed - val state = if (completed) EntryState.COMPLETED else EntryState.ERROR - val bytesSent = if (completed) e.totalBytes else e.bytesSent val failed = s as? Settlement.Failed - val record = EventJournal.SettledRecord( - eventId = UUID.randomUUID().toString(), - id = e.id, - key = e.key, - varsJson = e.varsJson, - at = now, - attempts = e.attempts, - requestId = e.lastRequestId, - deliveries = if (events.canDeliver()) 1 else 0, - state = state.wire, - bytesSent = bytesSent, - totalBytes = e.totalBytes, - url = s.url, - method = s.method, - partIndex = failed?.partIndex, - kind = if (completed) EventJournal.KIND_COMPLETED else EventJournal.KIND_ERROR, - response = when (s) { - is Settlement.Completed -> s.response?.let { EventJournal.Response.of(it) } ?: EventJournal.Response.NONE - is Settlement.Failed -> s.response?.let { EventJournal.Response.of(it) } - }, - errorKind = failed?.errorKind, - message = failed?.message, - cancelReason = null, - generation = generation, - ) - // 1. The durable outcome. It never throws. - journal.append(record) - // JS can subscribe between the check above and the append, and its first - // drain can miss this record. Count it as a live delivery then. - val live = if (record.deliveries == 0 && events.canDeliver()) { - journal.incrementDeliveries(record.eventId) ?: record - } else record - // 2. The transition, atomic against cancel(). - var applied = false - val next = try { - store.compute(id) { cur -> - if (EntryTransitions.canSettle(cur, generation)) { - applied = true - EntryTransitions.toSettled(cur!!, state, record.eventId, bytesSent, now) - } else cur + var delivered: EventJournal.SettledRecord? = null + var settled: QueueEntry? = null + store.locked { + val e = store.load(id) + if (!EntryTransitions.canSettle(e, generation, completed)) return@locked + e!! + val journaled = EntryTransitions.journaledSettle(e, journal.forEntry(id), now) + if (journaled != null) { + try { + store.save(journaled.entry) + journal.ack(journaled.extraEventIds) + settled = journaled.entry + } catch (error: IOException) { + Diag.error("settle could not apply the journaled outcome of '$id'; its ack or the boot sweep applies it", error) + } + return@locked + } + val state = if (completed) EntryState.COMPLETED else EntryState.ERROR + // A failed simple entry keeps the live bytes of its last attempt; a + // chunked one keeps its accepted bytes (the stored value). + val bytesSent = if (completed) e.totalBytes else failed?.bytesSent ?: e.bytesSent + val record = EventJournal.SettledRecord( + eventId = UUID.randomUUID().toString(), + id = e.id, + key = e.key, + varsJson = e.varsJson, + at = now, + attempts = e.attempts, + requestId = e.lastRequestId, + deliveries = 0, // the journal sets it + state = state.wire, + bytesSent = bytesSent, + totalBytes = e.totalBytes, + url = s.url, + method = s.method, + partIndex = failed?.partIndex, + kind = if (completed) EventJournal.KIND_COMPLETED else EventJournal.KIND_ERROR, + response = when (s) { + is Settlement.Completed -> s.response?.let { EventJournal.Response.of(it) } ?: EventJournal.Response.NONE + is Settlement.Failed -> s.response?.let { EventJournal.Response.of(it) } + }, + errorKind = failed?.errorKind, + message = failed?.message, + cancelReason = null, + generation = generation, + ) + // 1. The durable outcome. It never throws. + delivered = journal.appendOrHold(record) { store.referencedEventIds() + record.eventId } + // 2. The transition. + val next = EntryTransitions.toSettled(e, state, record.eventId, bytesSent, now) + try { + store.save(next) + settled = next + } catch (error: IOException) { + Diag.error("settle could not save '$id'; its ack, the next run, or the boot sweep applies the record", error) } - } catch (error: IOException) { - // The record is durable; the boot sweep applies it to the entry. - Diag.error("settle could not save '$id'; its ack or the boot sweep repairs it", error) - if (live.deliveries > 0) events.settled(live) - return true } - if (!applied || next == null) { - journal.ack(listOf(record.eventId)) + val record = delivered + if (record == null) { + settled?.let { events.state(it.toRow()) } return false } // 3 and 4. Best effort. - if (live.deliveries > 0) events.settled(live) - events.state(next.toRow()) + if (record.deliveries > 0) journal.listener()?.let { events.settled(record, it) } + settled?.let { events.state(it.toRow()) } return true } diff --git a/android/src/test/java/ai/openspace/backgroundupload/BodyCapTest.kt b/android/src/test/java/ai/openspace/backgroundupload/BodyCapTest.kt new file mode 100644 index 00000000..53a75385 --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/BodyCapTest.kt @@ -0,0 +1,70 @@ +package ai.openspace.backgroundupload + +import okio.Buffer +import okio.BufferedSource +import okio.buffer +import okio.ForwardingSource +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Test + +class BodyCapTest { + /** A source that counts the bytes read from it. */ + private class Counting(bytes: ByteArray) : ForwardingSource(Buffer().write(bytes)) { + var read = 0L + override fun read(sink: Buffer, byteCount: Long): Long = + super.read(sink, byteCount).also { if (it > 0) read += it } + } + + private fun source(bytes: ByteArray): Pair { + val c = Counting(bytes) + return c to c.buffer() + } + + @Test + fun `a body under the cap is read whole`() { + val (_, s) = source("hello".toByteArray()) + assertEquals(BodyCap.Capped("hello", false), BodyCap.read(s, 10)) + } + + @Test + fun `a body at exactly the cap is not truncated`() { + val (_, s) = source("0123456789".toByteArray()) + assertEquals(BodyCap.Capped("0123456789", false), BodyCap.read(s, 10)) + } + + @Test + fun `a huge body stops streaming just past the cap`() { + val (counting, s) = source(ByteArray(5_000_000) { 'x'.code.toByte() }) + val capped = BodyCap.read(s, 1_000) + assertTrue(capped.truncated) + assertEquals(1_000, capped.text.length) + // okio reads in 8 KB segments; nowhere near the 5 MB body. + assertTrue("read ${counting.read}", counting.read < 64 * 1024) + } + + @Test + fun `a cut inside a UTF-8 character backs off to the last whole one`() { + val euros = "€".repeat(4).toByteArray(Charsets.UTF_8) // 12 bytes + val (_, s) = source(euros) + val capped = BodyCap.read(s, 10) + assertEquals("€".repeat(3), capped.text) + assertTrue(capped.truncated) + } + + @Test + fun `cap measures UTF-8 bytes, not characters`() { + assertEquals("ab" to false, BodyCap.cap("ab", 2)) + assertEquals("é" to true, BodyCap.cap("éé", 3)) + assertEquals(null to false, BodyCap.cap(null, 3)) + // A 4-byte character (an emoji) is never split. + assertEquals("a" to true, BodyCap.cap("a😀", 4)) + } + + @Test + fun `bytes that are not UTF-8 are cut at the cap`() { + val bad = ByteArray(8) { 0x80.toByte() } // continuation bytes only + assertEquals(5, BodyCap.utf8Boundary(bad, 5)) + assertEquals(3, BodyCap.utf8Boundary("abc".toByteArray(), 5)) + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/ChunkedEngineTest.kt b/android/src/test/java/ai/openspace/backgroundupload/ChunkedEngineTest.kt index 453bab91..c93f1dff 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/ChunkedEngineTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/ChunkedEngineTest.kt @@ -73,11 +73,11 @@ class ChunkedEngineTest { @Test fun `a park from one part stops the siblings with the park itself`() { // The worker needs the ParkException back, not a CancellationException. - val thrown = assertThrows(EntryWorker.ParkException::class.java) { + val thrown = assertThrows(EntryRun.ParkException::class.java) { runBlocking { ChunkedEngine.run((0 until 6).toList()) { index -> yield() - if (index == 1) throw EntryWorker.ParkException(4) + if (index == 1) throw EntryRun.ParkException(4) yield() } } diff --git a/android/src/test/java/ai/openspace/backgroundupload/EntryParsingTest.kt b/android/src/test/java/ai/openspace/backgroundupload/EntryParsingTest.kt index 1d0fc10c..5ed1c534 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/EntryParsingTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/EntryParsingTest.kt @@ -3,38 +3,66 @@ package ai.openspace.backgroundupload import com.facebook.react.bridge.JavaOnlyArray import com.facebook.react.bridge.JavaOnlyMap import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse import org.junit.Assert.assertNull import org.junit.Assert.assertThrows import org.junit.Test class EntryParsingTest { - private fun entryMap(descriptor: JavaOnlyMap, vars: Any? = JavaOnlyMap.of("n", 1.0)) = - JavaOnlyMap.of("id", "e1", "key", "note", "vars", vars, "descriptor", descriptor) + private fun entryMap(descriptor: JavaOnlyMap, varsJson: Any? = """{"n":1}""") = + JavaOnlyMap.of("id", "e1", "key", "note", "varsJson", varsJson, "descriptor", descriptor) private fun base(vararg extra: Any?) = JavaOnlyMap.of("url", "https://example.com/items", "expiresAt", 9_000.0, *extra) @Test - fun `a JSON POST parses with defaults`() { - val p = EntryParsing.parse(entryMap(base("data", JavaOnlyMap.of("n", 1.0, "text", "hi")))) + fun `a JSON POST parses with defaults and keeps the JSON text as JS wrote it`() { + val text = """{"text":"hi","n":1,"status":null}""" + val p = EntryParsing.parse(entryMap(base("dataJson", text), varsJson = """{"b":2,"a":null}""")) assertEquals("e1", p.id) assertEquals("note", p.key) - assertEquals("""{"n":1}""", p.varsJson) + assertEquals("""{"b":2,"a":null}""", p.varsJson) // key order and null values survive assertEquals(9_000L, p.expiresAt) assertEquals("POST", p.descriptor.method) - assertEquals("""{"n":1,"text":"hi"}""", p.descriptor.dataJson) + assertEquals(text, p.descriptor.dataJson) assertEquals(StagedBody.JSON, p.descriptor.bodyKind) } @Test - fun `null vars store as the text null`() { - assertEquals("null", EntryParsing.parse(entryMap(base(), vars = null)).varsJson) + fun `null vars cross as the text null`() { + assertEquals("null", EntryParsing.parse(entryMap(base(), varsJson = "null")).varsJson) } @Test - fun `a null data is no body, because the bridge turns undefined into null`() { - assertNull(EntryParsing.parse(entryMap(base("data", null))).descriptor.dataJson) + fun `a dataJson of null is a real JSON body`() { + val d = EntryParsing.parse(entryMap(base("dataJson", "null"))).descriptor + assertEquals("null", d.dataJson) + assertEquals(StagedBody.JSON, d.bodyKind) + } + + @Test + fun `no dataJson is no body`() { + val d = EntryParsing.parse(entryMap(base())).descriptor + assertNull(d.dataJson) + assertEquals(StagedBody.NONE, d.bodyKind) + } + + @Test + fun `a missing or malformed varsJson or dataJson is rejected`() { + val cases = listOf( + entryMap(base(), varsJson = null), + entryMap(base(), varsJson = JavaOnlyMap.of("n", 1.0)), // the old object form + entryMap(base(), varsJson = "{n:1}"), // lenient JSON + entryMap(base(), varsJson = """{"n":1} trailing"""), + entryMap(base("dataJson", "")), + entryMap(base("dataJson", "{'a':1}")), + entryMap(base("dataJson", JavaOnlyMap.of("a", 1.0))), + entryMap(base("data", JavaOnlyMap.of("a", 1.0))), // the old data form would send no body + ) + cases.forEach { m -> + assertThrows("$m", EntryParsing.InvalidEntryException::class.java) { EntryParsing.parse(m) } + } } @Test @@ -93,8 +121,10 @@ class EntryParsingTest { val cases = listOf( JavaOnlyMap.of("url", "https://example.com"), // no expiresAt JavaOnlyMap.of("expiresAt", 1.0), // no url and no parts - base("data", 1.0, "file", "/a"), // two body kinds - base("method", "GET", "data", 1.0), // GET with a body + base("dataJson", "1", "file", "/a"), // two body kinds + base("method", "GET", "dataJson", "1"), // GET with a body + base("method", "GET", "dataJson", "null"), // GET with the JSON body null + base("method", "GET", "file", "/a"), base("method", "TRACE"), JavaOnlyMap.of("url", "not a url", "expiresAt", 1.0), base("headers", JavaOnlyMap.of("Bad\nName", "v")), @@ -108,12 +138,46 @@ class EntryParsingTest { } } + @Test + fun `a GET with no body parses`() { + assertEquals("GET", EntryParsing.parse(entryMap(base("method", "GET"))).descriptor.method) + } + + @Test + fun `a bad header value is rejected with its name and offset, never its value`() { + val secret = "Bearer s3cr3t-token" + val e = assertThrows(EntryParsing.InvalidEntryException::class.java) { + EntryParsing.parse(entryMap(base("headers", JavaOnlyMap.of("Authorization", "$secret\n")))) + } + assertEquals("headers: the value of header 'Authorization' has an invalid character at offset ${secret.length}", e.message) + assertFalse(e.message!!.contains("s3cr3t")) + } + + @Test + fun `a bad header name is rejected with the valid part before the offset only`() { + val e = assertThrows(EntryParsing.InvalidEntryException::class.java) { + EntryParsing.requireValidHeaders(mapOf("Authorization: Bearer s3cr3t" to "v"), "headers") + } + assertEquals("headers: the header name that starts 'Authorization:' has an invalid character at offset 14", e.message) + assertFalse(e.message!!.contains("s3cr3t")) + assertThrows(EntryParsing.InvalidEntryException::class.java) { + EntryParsing.requireValidHeaders(mapOf("" to "v"), "headers") + } + } + + @Test + fun `the header check accepts what OkHttp sends`() { + EntryParsing.requireValidHeaders(mapOf("X-Tab" to "a\tb", "X-Tilde" to "~!#", "Content-Range" to "bytes 0-9/10"), "headers") + okhttp3.Headers.Builder().add("X-Tab", "a\tb").add("X-Tilde", "~!#") + } + @Test fun `an updateHeaders patch is checked like descriptor headers`() { assertEquals(mapOf("Authorization" to "Bearer new"), EntryParsing.headerPatch(JavaOnlyMap.of("Authorization", "Bearer new"))) - assertThrows(EntryParsing.InvalidEntryException::class.java) { + val e = assertThrows(EntryParsing.InvalidEntryException::class.java) { EntryParsing.headerPatch(JavaOnlyMap.of("Authorization", "Bearer\nnew")) } + assertFalse(e.message!!.contains("Bearer")) } @Test diff --git a/android/src/test/java/ai/openspace/backgroundupload/EntryRunTest.kt b/android/src/test/java/ai/openspace/backgroundupload/EntryRunTest.kt new file mode 100644 index 00000000..1e7e0511 --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/EntryRunTest.kt @@ -0,0 +1,464 @@ +package ai.openspace.backgroundupload + +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.runBlocking +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertThrows +import org.junit.Assert.assertTrue +import org.junit.Before +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import java.io.File +import java.io.IOException + +/** A scripted [TransferHost]: a manual clock, a send handler, and a log of what the run did. */ +internal class FakeHost(var clock: Long = 10_000L) : TransferHost { + val requests = mutableListOf() + val sleeps = mutableListOf() + val attempts = mutableListOf() + val progress = mutableListOf() + var handler: suspend (TransferRequest, (Long) -> Unit) -> UploadResponse = { _, _ -> UploadResponse(200, "ok", mapOf()) } + var onSleep: () -> Unit = {} + var network = ArrayDeque() + var timeout = false + var foregroundError: Throwable? = null + + override fun now() = clock + + override suspend fun sleep(ms: Long) { + sleeps += ms + clock += ms + onSleep() + } + + override suspend fun send(request: TransferRequest, onProgress: (Long) -> Unit): UploadResponse { + requests += request + return handler(request, onProgress) + } + + override fun connectivity(wifiOnly: Boolean) = network.removeFirstOrNull() ?: Connectivity.Ok + + override suspend fun foreground(entry: QueueEntry) { + foregroundError?.let { throw it } + } + + override fun progressStarted(id: String, total: Long, sent: Long) { + progress += "start:$total:$sent" + } + + override fun progress(id: String, sent: Long, total: Long) { + progress += "$sent" + } + + override fun progressEnded(id: String, completed: Boolean) { + progress += "end:$completed" + } + + override fun attempt(event: AttemptEvent) { + attempts += event + } + + override fun stoppedByTimeout() = timeout +} + +class EntryRunTest { + @get:Rule + val tmp = TemporaryFolder() + + private lateinit var store: QueueStore + private lateinit var journal: EventJournal + private lateinit var settings: QueueSettingsStore + private val events = RecordingEvents() + private val scheduler = FakeScheduler() + private val host = FakeHost() + private lateinit var ops: WorkerOps + private lateinit var controller: QueueController + + @Before + fun setUp() { + val root = tmp.newFolder("queue") + store = QueueStore(root, RequestIndex()) + journal = EventJournal(tmp.newFolder("journal"), retryLater = { _, _ -> }) + settings = QueueSettingsStore(File(root, "settings.json")) + ops = WorkerOps(store, journal, settings, events, scheduler) { host.clock } + controller = QueueController(store, journal, settings, events, scheduler, { false }, { host.clock }) + controller.configureRetry(RetryDefaults(baseMs = 1_000, jitter = 0.0)) + journal.drain(Any()) + } + + private fun run(id: String = "e1") = runBlocking { EntryRun(id, store, ops, host).run() } + + private fun respond(vararg codes: Int) { + val queue = ArrayDeque(codes.toList()) + host.handler = { _, onProgress -> + onProgress(7) + UploadResponse(queue.removeFirst(), "body", mapOf()) + } + } + + private fun outcome() = journal.unacknowledged().single() + + // MARK: - simple + + @Test + fun `an accepted response settles completed after one write-ahead attempt`() { + controller.enqueue(parsed()) + respond(200) + run() + val e = store.load("e1")!! + assertEquals(EntryState.COMPLETED, e.state) + assertEquals(1, e.attempts) + val request = host.requests.single() + assertEquals(e.lastRequestId, request.headers["X-Request-Id"]) + assertEquals("Bearer old", request.headers["Authorization"]) + assertEquals("application/json", request.headers["Content-Type"]) + assertEquals(7L, request.body!!.contentLength()) + assertEquals(listOf("completed"), host.attempts.map { it.outcome }) + assertEquals(EventJournal.KIND_COMPLETED, outcome().kind) + assertEquals(200, outcome().response!!.status) + assertEquals(listOf("start:7:0", "7", "end:true"), host.progress) + } + + @Test + fun `a transient response waits a short backoff in place, with nextAttemptAt on the running row`() { + controller.enqueue(parsed()) + respond(503, 200) + run() + assertEquals(listOf(1_000L), host.sleeps) + assertEquals(listOf("error", "completed"), host.attempts.map { it.outcome }) + assertEquals("http", host.attempts[0].errorKind) + assertEquals(503, host.attempts[0].httpCode) + val waiting = events.rows.first { it.toMap().containsKey("nextAttemptAt") } + assertEquals("running", waiting.state) + assertEquals(11_000.0, waiting.toMap()["nextAttemptAt"]) + assertEquals(2, store.load("e1")!!.attempts) + assertEquals(listOf("start:7:0", "7", "0", "7", "end:true"), host.progress) // bytes reset between attempts + } + + @Test + fun `a backoff longer than 30 s releases the run back to queued with a wake`() { + controller.configureRetry(RetryDefaults(baseMs = 60_000, jitter = 0.0)) + controller.enqueue(parsed()) + respond(503) + run() + val e = store.load("e1")!! + assertEquals(EntryState.QUEUED, e.state) + assertEquals(host.clock + 60_000, e.nextAttemptAt) + assertEquals(1, e.backoffStreak) + assertEquals(listOf("e1" to host.clock + 60_000), scheduler.wakes) + assertEquals(emptyList(), host.sleeps) + assertEquals("end:false", host.progress.last()) + assertEquals(emptyList(), journal.unacknowledged()) + } + + @Test + fun `a transport failure is a network attempt and retries`() { + controller.enqueue(parsed()) + var calls = 0 + host.handler = { _, _ -> if (calls++ == 0) throw IOException("reset") else UploadResponse(200, "", mapOf()) } + run() + assertEquals(listOf("network", null), host.attempts.map { it.errorKind }) + assertEquals("reset", host.attempts[0].errorMessage) + assertEquals(EntryState.COMPLETED, store.load("e1")!!.state) + } + + @Test + fun `a terminal response settles error http with the response and the live bytes`() { + controller.enqueue(parsed()) + respond(400) + run() + val r = outcome() + assertEquals(EventJournal.KIND_ERROR, r.kind) + assertEquals("http", r.errorKind) + assertEquals(400, r.response!!.status) + assertEquals(7, r.bytesSent) + assertEquals(7, store.load("e1")!!.bytesSent) + assertEquals("end:false", host.progress.last()) + } + + @Test + fun `a staged body gone before the run settles error file with no request`() { + controller.enqueue(parsed()) + store.bodyFile(store.load("e1")!!)!!.delete() + run() + assertEquals("file", outcome().errorKind) + assertEquals(emptyList(), host.requests) + } + + @Test + fun `a body that vanishes mid-send settles error file, and the attempt says file`() { + controller.enqueue(parsed()) + host.handler = { _, _ -> + store.bodyFile(store.load("e1")!!)!!.delete() + throw IOException("ENOENT") + } + run() + assertEquals("file", outcome().errorKind) + assertEquals(listOf("file"), host.attempts.map { it.errorKind }) + } + + @Test + fun `a 401 parks the entry and wakes it at expiry`() { + controller.enqueue(parsed(expiresAt = 90_000)) + respond(401) + run() + assertEquals(EntryState.AWAITING_AUTH, store.load("e1")!!.state) + assertEquals(listOf("e1" to 90_000L), scheduler.wakes) + assertEquals(listOf("error"), host.attempts.map { it.outcome }) + assertEquals(emptyList(), journal.unacknowledged()) + } + + @Test + fun `a 401 with newer headers from updateHeaders mid-flight re-issues at once with them`() { + controller.enqueue(parsed()) + var calls = 0 + host.handler = { _, _ -> + if (calls++ == 0) { + controller.updateHeaders(mapOf("Authorization" to "Bearer new")) + UploadResponse(401, "", mapOf()) + } else UploadResponse(200, "", mapOf()) + } + run() + assertEquals(listOf("Bearer old", "Bearer new"), host.requests.map { it.headers["Authorization"] }) + assertEquals(emptyList(), host.sleeps) + assertEquals(EntryState.COMPLETED, store.load("e1")!!.state) + } + + @Test + fun `the expiry wake of a parked entry settles expired and keeps the stored bytes`() { + controller.enqueue(parsed(expiresAt = 20_000)) + store.save(store.load("e1")!!.copy(state = EntryState.AWAITING_AUTH, bytesSent = 3)) + run() + assertEquals(EntryState.AWAITING_AUTH, store.load("e1")!!.state) // not yet expired + host.clock = 20_000 + run() + assertEquals("expired", outcome().errorKind) + assertEquals(3, outcome().bytesSent) + assertEquals(emptyList(), host.requests) + } + + @Test + fun `an entry past expiresAt settles error expired`() { + controller.enqueue(parsed(expiresAt = 5_000)) + run() + assertEquals("expired", outcome().errorKind) + assertEquals(emptyList(), host.requests) + } + + @Test + fun `a paused entry past expiresAt stays paused, and settles expired at resume`() { + controller.enqueue(parsed(expiresAt = 20_000)) + controller.pause() + host.clock = 30_000 + run() // a wake that fires during the pause + assertEquals(EntryState.PAUSED, store.load("e1")!!.state) + assertEquals(emptyList(), journal.unacknowledged()) + controller.resume() + assertEquals(EntryState.QUEUED, store.load("e1")!!.state) + run() + assertEquals("expired", outcome().errorKind) + assertEquals(EntryState.ERROR, store.load("e1")!!.state) + } + + @Test + fun `a failure that lands after pause is not an outcome`() { + controller.enqueue(parsed()) + host.handler = { _, _ -> + controller.pause() + UploadResponse(400, "", mapOf()) + } + run() + assertEquals(EntryState.PAUSED, store.load("e1")!!.state) + assertEquals(emptyList(), journal.unacknowledged()) + } + + @Test + fun `an accepted response that lands after pause settles completed`() { + controller.enqueue(parsed()) + host.handler = { _, _ -> + controller.pause() + UploadResponse(200, "", mapOf()) + } + run() + assertEquals(EntryState.COMPLETED, store.load("e1")!!.state) + } + + @Test + fun `a transient response after cancel stops the run with only the cancel outcome`() { + controller.enqueue(parsed()) + host.handler = { _, _ -> + controller.cancel("e1") + UploadResponse(503, "", mapOf()) + } + run() + assertEquals(EventJournal.KIND_CANCELLED, outcome().kind) + assertEquals("end:false", host.progress.last()) + } + + @Test + fun `a system stop moves the entry back to queued with no outcome and no attempt event`() { + controller.enqueue(parsed()) + host.handler = { _, _ -> throw CancellationException("stopped") } + assertThrows(CancellationException::class.java) { run() } + val e = store.load("e1")!! + assertEquals(EntryState.QUEUED, e.state) + assertNull(e.nextAttemptAt) + assertEquals(emptyList(), host.attempts) + assertEquals(emptyList(), journal.unacknowledged()) + assertEquals("end:false", host.progress.last()) + } + + @Test + fun `a timeout stop takes one more backoff step instead of restarting at once`() { + controller.enqueue(parsed()) + store.save(store.load("e1")!!.copy(backoffStreak = 2)) + host.timeout = true + host.handler = { _, _ -> throw CancellationException("timeout") } + assertThrows(CancellationException::class.java) { run() } + val e = store.load("e1")!! + assertEquals(EntryState.QUEUED, e.state) + assertEquals(3, e.backoffStreak) + assertEquals(host.clock + 4_000, e.nextAttemptAt) // base 1 s * 2^(3-1) + assertEquals(listOf("e1" to host.clock + 4_000), scheduler.wakes) + } + + @Test + fun `a store write that fails mid-run throws with no outcome`() { + controller.enqueue(parsed()) + respond(503, 200) + val dir = store.entryDir("e1") + host.onSleep = { dir.setWritable(false) } // the next attempt's write-ahead fails + try { + assertThrows(IOException::class.java) { run() } + } finally { + dir.setWritable(true) + } + assertEquals(emptyList(), journal.unacknowledged()) + assertEquals(1, host.requests.size) + } + + @Test + fun `an unexpected error settles error unknown`() { + controller.enqueue(parsed()) + host.foregroundError = IllegalStateException("boom") + run() + assertEquals("unknown", outcome().errorKind) + assertEquals("boom", outcome().message) + assertEquals(listOf("end:false"), host.progress) + } + + @Test + fun `a re-run after a lost settle write applies the record and sends nothing`() { + controller.enqueue(parsed()) + store.save(store.load("e1")!!.copy(state = EntryState.RUNNING)) + journal.append(record("00000000-0000-0000-0000-000000000041")) + run() + assertEquals(EntryState.COMPLETED, store.load("e1")!!.state) + assertEquals(emptyList(), host.requests) + } + + @Test + fun `a short remaining backoff is slept out before the run, a long one is left to the wake`() { + controller.enqueue(parsed()) + store.save(store.load("e1")!!.copy(nextAttemptAt = host.clock + 5_000)) + run() + assertEquals(listOf(5_000L), host.sleeps) + assertEquals(EntryState.COMPLETED, store.load("e1")!!.state) + + controller.enqueue(parsed(id = "later")) + store.save(store.load("later")!!.copy(nextAttemptAt = host.clock + 60_000)) + run("later") + assertEquals(EntryState.QUEUED, store.load("later")!!.state) + assertEquals(1, host.requests.size) + } + + @Test + fun `no usable network polls until it returns`() { + controller.enqueue(parsed()) + host.network = ArrayDeque(listOf(Connectivity.NoWifi, Connectivity.NoInternet)) + run() + assertEquals(listOf(EntryRun.CONNECTIVITY_POLL_MS, EntryRun.CONNECTIVITY_POLL_MS), host.sleeps) + assertEquals(EntryState.COMPLETED, store.load("e1")!!.state) + } + + // MARK: - chunked + + private fun chunked(size: Int = 30) { + val src = File(tmp.newFolder(), "video.bin").apply { writeBytes(ByteArray(size) { it.toByte() }) } + controller.enqueue(parsed(descriptor = desc(url = null, method = "PUT", file = src.path, + parts = listOf(part(0, 10), part(10, 20), part(20, 30))))) + } + + @Test + fun `every part accepted settles completed with no status, one attempt per part`() { + chunked() + run() + val e = store.load("e1")!! + assertEquals(EntryState.COMPLETED, e.state) + assertEquals(3, e.attempts) + assertTrue(e.descriptor!!.parts!!.all { it.accepted }) + assertNull(outcome().response!!.status) + val byUrl = host.requests.associateBy { it.url } + assertEquals("20-29", byUrl["https://example.com/part?start=20"]!!.headers["Content-Range"]) + assertEquals(10L, byUrl["https://example.com/part?start=20"]!!.body!!.contentLength()) + assertEquals(listOf(0, 1, 2), host.attempts.map { it.partIndex }.sortedBy { it }) + } + + @Test + fun `a part that fails terminally settles error with its index, and the other parts stop`() { + chunked() + host.handler = { r, _ -> + UploadResponse(if (r.url.endsWith("start=10")) 400 else 200, "", mapOf()) + } + run() + val r = outcome() + assertEquals("http", r.errorKind) + assertEquals(1, r.partIndex) + assertEquals("HTTP 400 on part 1", r.message) + assertEquals("https://example.com/part?start=10", r.url) + val accepted = store.load("e1")!!.descriptor!!.parts!!.map { it.accepted } + assertFalse(accepted[1]) + } + + @Test + fun `a part 503 backs off in that part while the others go on`() { + chunked() + var failed = false + host.handler = { r, _ -> + if (r.url.endsWith("start=0") && !failed) { + failed = true + UploadResponse(503, "", mapOf()) + } else UploadResponse(200, "", mapOf()) + } + run() + assertEquals(EntryState.COMPLETED, store.load("e1")!!.state) + assertEquals(4, host.requests.size) + assertEquals(listOf(1_000L), host.sleeps) + } + + @Test + fun `a part 401 parks the whole entry and keeps the accepted parts`() { + chunked() + host.handler = { r, _ -> + UploadResponse(if (r.url.endsWith("start=20")) 401 else 200, "", mapOf()) + } + run() + val e = store.load("e1")!! + assertEquals(EntryState.AWAITING_AUTH, e.state) + assertEquals(listOf(true, true, false), e.descriptor!!.parts!!.map { it.accepted }) + } + + @Test + fun `a failed chunked settle keeps the accepted bytes, not the in-flight ones`() { + chunked() + host.handler = { r, onProgress -> + onProgress(5) + UploadResponse(if (r.url.endsWith("start=20")) 400 else 200, "", mapOf()) + } + run() + assertEquals(20, outcome().bytesSent) + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/EntryTransitionsTest.kt b/android/src/test/java/ai/openspace/backgroundupload/EntryTransitionsTest.kt index e5a37f73..a40a156f 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/EntryTransitionsTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/EntryTransitionsTest.kt @@ -10,12 +10,35 @@ class EntryTransitionsTest { @Test fun `a settle on a cancelled entry or an older generation is not allowed`() { - assertTrue(EntryTransitions.canSettle(entry(state = EntryState.RUNNING), 1)) - assertTrue(EntryTransitions.canSettle(entry(state = EntryState.PAUSED), 1)) // an in-flight response under pause - assertFalse(EntryTransitions.canSettle(entry(state = EntryState.CANCELLED), 1)) - assertFalse(EntryTransitions.canSettle(entry(state = EntryState.COMPLETED), 1)) - assertFalse(EntryTransitions.canSettle(entry(state = EntryState.RUNNING, generation = 2), 1)) - assertFalse(EntryTransitions.canSettle(null, 1)) + for (accepted in listOf(true, false)) { + assertTrue(EntryTransitions.canSettle(entry(state = EntryState.RUNNING), 1, accepted)) + assertTrue(EntryTransitions.canSettle(entry(state = EntryState.AWAITING_AUTH), 1, accepted)) // the expiry wake + assertFalse(EntryTransitions.canSettle(entry(state = EntryState.CANCELLED), 1, accepted)) + assertFalse(EntryTransitions.canSettle(entry(state = EntryState.COMPLETED), 1, accepted)) + assertFalse(EntryTransitions.canSettle(entry(state = EntryState.RUNNING, generation = 2), 1, accepted)) + assertFalse(EntryTransitions.canSettle(null, 1, accepted)) + } + } + + @Test + fun `under pause only an accepted response settles`() { + assertTrue(EntryTransitions.canSettle(entry(state = EntryState.PAUSED), 1, accepted = true)) + assertFalse(EntryTransitions.canSettle(entry(state = EntryState.PAUSED), 1, accepted = false)) + } + + @Test + fun `a live entry with a record of its own generation settles from the newest one`() { + val live = entry(state = EntryState.RUNNING, generation = 2) + val older = record("00000000-0000-0000-0000-0000000000a1", generation = 2, at = 1) + val newest = record("00000000-0000-0000-0000-0000000000a2", kind = EventJournal.KIND_ERROR, generation = 2, at = 2) + val otherLife = record("00000000-0000-0000-0000-0000000000a3", generation = 1, at = 3) + val otherId = record("00000000-0000-0000-0000-0000000000a4", id = "x", generation = 2, at = 4) + val j = EntryTransitions.journaledSettle(live, listOf(older, newest, otherLife, otherId), now = 9)!! + assertEquals(EntryState.ERROR, j.entry.state) + assertEquals(newest.eventId, j.entry.settledEventId) + assertEquals(listOf(older.eventId), j.extraEventIds) + assertNull(EntryTransitions.journaledSettle(live, listOf(otherLife, otherId), 9)) + assertNull(EntryTransitions.journaledSettle(entry(state = EntryState.ERROR, generation = 2), listOf(newest), 9)) } @Test @@ -84,9 +107,12 @@ class EnqueueRulesTest { @Test fun `the same-id table`() { assertEquals(EnqueueRules.Action.Create, decide(null)) - val v9 = LegacyManifest("e1", "/b", listOf(part(0, 10)), emptyList(), 1, false, 1) - assertEquals(EnqueueRules.Action.AdoptV9(v9), decide(null, v9 = v9)) - assertEquals(EnqueueRules.Action.Replace, decide(entry(legacy = true, descriptor = null, body = null))) + val v9 = LegacyManifest("e1", listOf(part(0, 10)), emptyList()) + assertEquals(EnqueueRules.Action.AdoptV9(v9, 1), decide(null, v9 = v9)) + val legacy = entry(legacy = true, descriptor = null, body = null) + assertEquals(EnqueueRules.Action.Replace, decide(legacy)) + // A legacy row over a v9 manifest: adopt it, one generation up. + assertEquals(EnqueueRules.Action.AdoptV9(v9, 2), decide(legacy, v9 = v9)) assertEquals(EnqueueRules.Action.ReEmit("ev"), decide(entry(state = EntryState.COMPLETED, settledEventId = "ev"))) assertEquals(EnqueueRules.Action.Replace, decide(entry(state = EntryState.COMPLETED, settledEventId = "ev"), hasRecord = false)) EntryState.values().filter { it != EntryState.COMPLETED }.forEach { state -> @@ -128,13 +154,23 @@ class EnqueueRulesTest { } @Test - fun `resume of a settled entry reopens it with a fresh generation`() { - val next = EnqueueRules.resumed(entry(state = EntryState.ERROR, settledEventId = "ev", generation = 2), parsed(), false, 0, 9) + fun `resume of a settled entry reopens it with a fresh generation and attempts 0`() { + val settled = entry(state = EntryState.ERROR, settledEventId = "ev", generation = 2, attempts = 3) + val next = EnqueueRules.resumed(settled, parsed(), false, 0, 9) assertEquals(EntryState.QUEUED, next.state) assertEquals(3, next.generation) + assertEquals(0, next.attempts) assertNull(next.settledEventId) } + @Test + fun `resume clears a pending backoff so the entry runs now`() { + val waiting = entry(state = EntryState.QUEUED, nextAttemptAt = 99_000).copy(backoffStreak = 6) + val next = EnqueueRules.resumed(waiting, parsed(), false, 0, 9) + assertNull(next.nextAttemptAt) + assertEquals(0, next.backoffStreak) + } + @Test fun `resume of a running entry stays running, and under pause becomes paused`() { assertEquals(EntryState.RUNNING, EnqueueRules.resumed(entry(state = EntryState.RUNNING), parsed(), true, 0, 9).state) @@ -149,7 +185,7 @@ class EnqueueRulesTest { @Test fun `adopting v9 parts carries the flags only for the same parts`() { - val v9 = LegacyManifest("e1", "/b", listOf(part(0, 10, accepted = true), part(10, 20)), emptyList(), 1, false, 1) + val v9 = LegacyManifest("e1", listOf(part(0, 10, accepted = true), part(10, 20)), emptyList()) assertTrue(EnqueueRules.adoptedParts(v9, listOf(part(0, 10), part(10, 20)))[0].accepted) assertFalse(EnqueueRules.adoptedParts(v9, listOf(part(0, 20)))[0].accepted) } diff --git a/android/src/test/java/ai/openspace/backgroundupload/EventJournalTest.kt b/android/src/test/java/ai/openspace/backgroundupload/EventJournalTest.kt index b095c4c7..474c570c 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/EventJournalTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/EventJournalTest.kt @@ -3,11 +3,13 @@ package ai.openspace.backgroundupload import org.junit.Assert.assertEquals import org.junit.Assert.assertFalse import org.junit.Assert.assertNull +import org.junit.Assert.assertThrows import org.junit.Assert.assertTrue import org.junit.Rule import org.junit.Test import org.junit.rules.TemporaryFolder import java.io.File +import java.io.IOException class EventJournalTest { @get:Rule @@ -16,12 +18,66 @@ class EventJournalTest { private val id1 = "00000000-0000-0000-0000-000000000001" private val id2 = "00000000-0000-0000-0000-000000000002" + private val listener = Any() + @Test fun `append then read returns the record`() { val journal = EventJournal(tmp.newFolder()) - assertTrue(journal.append(record(id1))) - val events = journal.unacknowledged() - assertEquals(listOf(record(id1)), events) + journal.drain(listener) + assertEquals(record(id1), journal.append(record(id1))) + assertEquals(listOf(record(id1)), journal.unacknowledged()) + } + + @Test + fun `append starts at 1 delivery with a listener and at 0 without one`() { + val journal = EventJournal(tmp.newFolder()) + assertFalse(journal.isListening()) + assertEquals(0, journal.append(record(id1)).deliveries) + assertEquals(listOf(1), journal.drain(listener).map { it.deliveries }) // the drain delivers it + assertTrue(journal.isListening()) + assertEquals(1, journal.append(record(id2)).deliveries) + } + + @Test + fun `a drain for a torn-down module sets no listener and counts nothing`() { + val journal = EventJournal(tmp.newFolder()) + journal.append(record(id1)) + assertEquals(emptyList(), journal.drain(listener) { false }) + assertFalse(journal.isListening()) + assertEquals(0, journal.find(id1)!!.deliveries) + } + + @Test + fun `listener is the owner of the last drain`() { + val journal = EventJournal(tmp.newFolder()) + assertNull(journal.listener()) + journal.drain(listener) + assertTrue(journal.listener() === listener) + val next = Any() + journal.drain(next) + assertTrue(journal.listener() === next) + } + + @Test + fun `stopListening clears only its own listener`() { + val journal = EventJournal(tmp.newFolder()) + val next = Any() + journal.drain(listener) + journal.drain(next) // a reload: the next module drains before the old one is torn down + journal.stopListening(listener) + assertTrue(journal.isListening()) + journal.stopListening(next) + assertFalse(journal.isListening()) + } + + @Test + fun `redeliver counts a delivery only with a listener`() { + val journal = EventJournal(tmp.newFolder()) + journal.append(record(id1)) + assertNull(journal.redeliver(id1)) + assertEquals(0, journal.find(id1)!!.deliveries) + journal.drain(listener) // 1 + assertEquals(2, journal.redeliver(id1)!!.deliveries) } @Test @@ -49,13 +105,23 @@ class EventJournalTest { } @Test - fun `a body over 1 MB is cut and flagged`() { + fun `a body over 1 MB of UTF-8 is cut on a character and flagged`() { val journal = EventJournal(tmp.newFolder()) - val big = "x".repeat(EventJournal.MAX_BODY_CHARS + 100) + // 2-byte characters, so a char cap would keep 2 MB. + val big = "\u00e9".repeat(BodyCap.SETTLED_MAX_BYTES) journal.append(record(id1).copy(response = EventJournal.Response(200, null, big, false))) val read = journal.unacknowledged()[0].response!! assertTrue(read.bodyTruncated) - assertEquals(EventJournal.MAX_BODY_CHARS, read.body!!.length) + assertEquals(BodyCap.SETTLED_MAX_BYTES, read.body!!.toByteArray(Charsets.UTF_8).size) + assertEquals(BodyCap.SETTLED_MAX_BYTES / 2, read.body!!.length) + } + + @Test + fun `a response the stream cap cut stays flagged`() { + val r = EventJournal.Response.of(UploadResponse(500, "partial", mapOf(), truncated = true)) + assertEquals("partial", r.body) + assertTrue(r.bodyTruncated) + assertFalse(EventJournal.Response.of(UploadResponse(500, "whole", mapOf())).bodyTruncated) } @Test @@ -77,12 +143,63 @@ class EventJournalTest { } @Test - fun `append never throws, and says whether it wrote`() { + fun `append throws when it could not write and keeps nothing`() { val journal = EventJournal(tmp.newFile()) // a file where the directory should be - assertFalse(journal.append(record(id1))) + assertThrows(IOException::class.java) { journal.append(record(id1)) } assertEquals(emptyList(), journal.unacknowledged()) } + @Test + fun `appendOrHold holds a record it could not write, and every read and ack sees it`() { + val tasks = mutableListOf Unit>>() + val dir = tmp.newFolder() + val journal = EventJournal(dir, retryLater = { delay, task -> tasks += delay to task }) + dir.setWritable(false) + try { + val held = journal.appendOrHold(record(id1)) + assertTrue(journal.isHeld(id1)) + assertEquals(listOf(held), journal.unacknowledged()) + assertEquals(held, journal.find(id1)) + assertEquals(listOf(id1), journal.forEntry("e1").map { it.eventId }) + assertEquals(1, journal.drain(listener).single().deliveries) + // The retry fails while the disk is full, and waits twice as long. + tasks.removeAt(0).also { (delay, task) -> assertEquals(EventJournal.RETRY_MS, delay); task() } + assertEquals(EventJournal.RETRY_MS * 2, tasks.single().first) + } finally { + dir.setWritable(true) + } + tasks.removeAt(0).second() + assertFalse(journal.isHeld(id1)) + assertTrue(File(dir, "$id1.json").exists()) + assertEquals(1, EventJournal(dir).find(id1)!!.deliveries) + assertEquals(emptyList Unit>>(), tasks) + } + + @Test + fun `an ack removes a held record`() { + val dir = tmp.newFolder() + val journal = EventJournal(dir, retryLater = { _, _ -> }) + dir.setWritable(false) + try { + journal.appendOrHold(record(id1)) + } finally { + dir.setWritable(true) + } + journal.ack(listOf(id1)) + assertFalse(journal.isHeld(id1)) + assertEquals(emptyList(), journal.unacknowledged()) + } + + @Test + fun `ackEntry removes every record of one entry`() { + val journal = EventJournal(tmp.newFolder()) + journal.append(record(id1, id = "a")) + journal.append(record(id2, id = "a", generation = 2)) + journal.append(record("00000000-0000-0000-0000-000000000003", id = "b")) + journal.ackEntry("a") + assertEquals(listOf("b"), journal.unacknowledged().map { it.id }) + } + @Test fun `prunes the oldest records beyond the cap`() { val dir = tmp.newFolder() @@ -98,10 +215,25 @@ class EventJournalTest { assertFalse(left.contains(ids[0])) } + @Test + fun `the prune never deletes a record a row names`() { + val dir = tmp.newFolder() + val journal = EventJournal(dir, maxEntries = 3) + val ids = (1..4).map { "00000000-0000-0000-0000-00000000000$it" } + ids.take(3).forEachIndexed { i, id -> + journal.append(record(id)) + File(dir, "$id.json").setLastModified(1_000L * (i + 1)) + } + // The oldest is named by a row; the next oldest goes instead. + journal.append(record(ids[3])) { setOf(ids[0]) } + assertEquals(setOf(ids[0], ids[2], ids[3]), journal.unacknowledged().map { it.eventId }.toSet()) + } + @Test fun `incrementDeliveries persists`() { val dir = tmp.newFolder() val journal = EventJournal(dir) + journal.drain(listener) journal.append(record(id1)) assertEquals(2, journal.incrementDeliveries(id1)!!.deliveries) assertEquals(3, journal.incrementDeliveries(id1)!!.deliveries) diff --git a/android/src/test/java/ai/openspace/backgroundupload/JsonBridgeTest.kt b/android/src/test/java/ai/openspace/backgroundupload/JsonBridgeTest.kt index bb420821..928540e7 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/JsonBridgeTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/JsonBridgeTest.kt @@ -11,36 +11,25 @@ class JsonBridgeTest { @Test fun `integral doubles print as integers, as JSON stringify does`() { - assertEquals("""{"n":1,"neg":-3,"zero":0}""", JsonBridge.toJson(mapOf("n" to 1.0, "neg" to -3.0, "zero" to -0.0))) - assertEquals("12345678901", JsonBridge.toJson(12_345_678_901.0)) + assertEquals("1", JsonBridge.numberText(1.0)) + assertEquals("-3", JsonBridge.numberText(-3.0)) + assertEquals("0", JsonBridge.numberText(-0.0)) + assertEquals("12345678901", JsonBridge.numberText(12_345_678_901.0)) } @Test fun `fractions and very large magnitudes keep a decimal form`() { - assertEquals("1.5", JsonBridge.toJson(1.5)) - assertEquals("0.1", JsonBridge.toJson(0.1)) + assertEquals("1.5", JsonBridge.numberText(1.5)) + assertEquals("0.1", JsonBridge.numberText(0.1)) // Above 2^53 a double can not hold every integer, so it stays a double. - assertEquals(1e20, (JsonBridge.parse(JsonBridge.toJson(1e20)) as Double), 0.0) + assertEquals(1e20, (JsonBridge.parse(JsonBridge.numberText(1e20)) as Double), 0.0) } @Test - fun `nested maps and lists round trip`() { + fun `JSON text parses to plain values`() { val value = mapOf("a" to listOf(1.0, "x", true, null, mapOf("b" to 2.5)), "c" to mapOf()) - val text = JsonBridge.toJson(value) - assertEquals("""{"a":[1,"x",true,null,{"b":2.5}],"c":{}}""", text) - assertEquals(value, JsonBridge.parse(text)) - } - - @Test - fun `keys are sorted so the same object always gives the same text`() { - assertEquals(JsonBridge.toJson(mapOf("b" to 1.0, "a" to 2.0)), JsonBridge.toJson(mapOf("a" to 2.0, "b" to 1.0))) - } - - @Test - fun `null is the text null, and HTML characters are not escaped`() { - assertEquals("null", JsonBridge.toJson(null)) + assertEquals(value, JsonBridge.parse("""{"a":[1,"x",true,null,{"b":2.5}],"c":{}}""")) assertNull(JsonBridge.parse("null")) - assertEquals("\"\"", JsonBridge.toJson("")) } @Test @@ -78,4 +67,18 @@ class JsonBridgeTest { assertEquals(true, out.getMap("nested")!!.getBoolean("k")) assertEquals(true, out.isNull("none")) } + + @Test + fun `isJson accepts one strict JSON value of any kind`() { + listOf("null", "1", "-0.5e3", "\"s\"", "true", "[]", "{}", """{"a":[1,null,{"b":"c"}]}""", " {\"a\":1} ").forEach { + assertEquals(it, true, JsonBridge.isJson(it)) + } + } + + @Test + fun `isJson rejects lenient and malformed text`() { + listOf("", " ", "{a:1}", "{'a':1}", "[1,]", "{\"a\":1} x", "undefined", "NaN", "{\"a\":1}{}").forEach { + assertEquals(it, false, JsonBridge.isJson(it)) + } + } } diff --git a/android/src/test/java/ai/openspace/backgroundupload/LegacyImportTest.kt b/android/src/test/java/ai/openspace/backgroundupload/LegacyImportTest.kt index 9f7be8ef..1386c2cb 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/LegacyImportTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/LegacyImportTest.kt @@ -1,6 +1,7 @@ package ai.openspace.backgroundupload import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse import org.junit.Assert.assertNull import org.junit.Assert.assertTrue import org.junit.Rule @@ -74,6 +75,37 @@ class LegacyImportTest { assertTrue(File(v9Dir, "a.json").exists()) } + @Test + fun `runOnce imports, then writes the marker, and a second launch does nothing`() { + val v9Dir = tmp.newFolder() + val marker = File(tmp.newFolder(), LegacyImport.MARKER) + val store = QueueStore(tmp.newFolder(), RequestIndex()) + v9(v9Dir, "a", "up-1", "completed", 100) + assertTrue(LegacyImport.runOnce(marker, v9Dir, store)) + assertTrue(marker.exists()) + assertEquals(listOf("up-1"), store.all().map { it.id }) + // A v9 file that shows up later is not imported: the marker says done. + v9(v9Dir, "b", "up-2", "error", 200) + store.remove("up-1") + assertFalse(LegacyImport.runOnce(marker, v9Dir, store)) + assertEquals(emptyList(), store.all()) + assertTrue(File(v9Dir, "b.json").exists()) + } + + @Test + fun `runOnce with a failed row save writes no marker, so the next launch tries again`() { + val v9Dir = tmp.newFolder() + val marker = File(tmp.newFolder(), LegacyImport.MARKER) + v9(v9Dir, "a", "up-1", "error", 100) + val broken = QueueStore(tmp.newFile(), RequestIndex()) // a file where the directory should be + assertTrue(LegacyImport.runOnce(marker, v9Dir, broken)) // it ran, so v9 work is cancelled + assertFalse(marker.exists()) + val store = QueueStore(tmp.newFolder(), RequestIndex()) + assertTrue(LegacyImport.runOnce(marker, v9Dir, store)) + assertTrue(marker.exists()) + assertEquals(listOf("up-1"), store.all().map { it.id }) + } + @Test fun `an unknown type makes no row`() { assertNull(LegacyImport.legacyRow(LegacyImport.V9Entry("a", "up", "progress", 1))) diff --git a/android/src/test/java/ai/openspace/backgroundupload/QueueControllerTest.kt b/android/src/test/java/ai/openspace/backgroundupload/QueueControllerTest.kt index 92663412..6cfc408f 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/QueueControllerTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/QueueControllerTest.kt @@ -26,14 +26,16 @@ class QueueControllerTest { private val running = mutableSetOf() private var now = 10_000L private lateinit var controller: QueueController + private val listener = Any() @Before fun setUp() { root = tmp.newFolder("queue") store = QueueStore(root, RequestIndex()) - journal = EventJournal(tmp.newFolder("journal")) + journal = EventJournal(tmp.newFolder("journal"), retryLater = { _, _ -> }) settings = QueueSettingsStore(File(root, "settings.json")) controller = QueueController(store, journal, settings, events, scheduler, { it in running }, { now }) + journal.drain(listener) // JS is subscribed } private fun source(name: String, size: Int) = File(tmp.newFolder(), name).apply { writeBytes(ByteArray(size) { it.toByte() }) } @@ -165,14 +167,50 @@ class QueueControllerTest { } @Test - fun `same body on an error entry reopens it and keeps attempts`() { + fun `same body on an error entry reopens it with attempts 0`() { controller.enqueue(parsed()) store.save(store.load("e1")!!.copy(state = EntryState.ERROR, attempts = 3, settledEventId = "ev")) controller.enqueue(parsed(expiresAt = FAR_FUTURE + 1)) val e = store.load("e1")!! assertEquals(EntryState.QUEUED, e.state) assertEquals(2, e.generation) - assertEquals(3, e.attempts) + assertEquals(0, e.attempts) // attempts count the current generation + } + + @Test + fun `same body on a live entry keeps its attempts`() { + controller.enqueue(parsed()) + store.save(store.load("e1")!!.copy(attempts = 3)) + controller.enqueue(parsed(expiresAt = FAR_FUTURE + 1)) + assertEquals(3, store.load("e1")!!.attempts) + } + + @Test + fun `a same-id resume of a queued entry waiting out a backoff clears it and runs now`() { + controller.enqueue(parsed()) + store.save(store.load("e1")!!.copy(nextAttemptAt = now + 3_600_000, backoffStreak = 9)) + scheduler.scheduled.clear() + controller.enqueue(parsed()) + val e = store.load("e1")!! + assertNull(e.nextAttemptAt) + assertEquals(0, e.backoffStreak) + assertEquals(listOf("e1"), scheduler.scheduled) + assertEquals(emptyList>(), scheduler.wakes) + assertFalse(events.rows.last().toMap().containsKey("nextAttemptAt")) + } + + @Test + fun `a completed unacked re-emit with no listener yet waits for the drain`() { + controller.enqueue(parsed()) + val eventId = "00000000-0000-0000-0000-00000000001a" + journal.append(record(eventId)) + store.save(store.load("e1")!!.copy(state = EntryState.COMPLETED, settledEventId = eventId)) + journal.stopListening(listener) + events.log.clear() + controller.enqueue(parsed()) + assertEquals(emptyList(), events.log) + assertEquals(1, journal.find(eventId)!!.deliveries) + assertEquals(2, controller.unacknowledged(listener).single().deliveries) } @Test @@ -185,6 +223,34 @@ class QueueControllerTest { assertEquals(EntryState.QUEUED, e.state) } + // MARK: - legacy row over a v9 manifest + + @Test + fun `a same-id enqueue over a legacy row with a v9 manifest adopts it one generation up`() { + v9Dir("e1", v9Parts, 20) + store.save(LegacyImport.legacyRow(LegacyImport.V9Entry("x", "e1", "error", 5))!!) + val legacy = store.load("e1")!! + assertEquals(0L to 0L, legacy.bytesSent to legacy.totalBytes) // legacy rows report 0/0 bytes + controller.enqueue(parsed(descriptor = desc(url = null, method = "PUT", file = "/gone.bin", parts = listOf(part(0, 10), part(10, 20))))) + val e = store.load("e1")!! + assertFalse(e.legacy) + assertEquals(2, e.generation) + assertEquals(5, e.createdAt) + assertTrue(e.descriptor!!.parts!![0].accepted) // the v9 progress is kept + assertFalse(e.descriptor!!.parts!![1].accepted) + assertEquals(10, e.bytesSent) + assertEquals(setOf("entry.json", "blob"), dirFiles()) // the manifest is pruned + } + + @Test + fun `a same-id enqueue over a legacy row with no manifest replaces it`() { + store.save(LegacyImport.legacyRow(LegacyImport.V9Entry("x", "e1", "error", 5))!!) + controller.enqueue(parsed()) + val e = store.load("e1")!! + assertEquals(2, e.generation) + assertEquals(0, e.bytesSent) + } + // MARK: - chunked replace (a present file wins over the old blob) private fun chunkedErrorEntry(): File { @@ -360,23 +426,99 @@ class QueueControllerTest { assertEquals(record.eventId, e.settledEventId) assertEquals(listOf("e1"), scheduler.cancelled) assertEquals(listOf("settled:e1:cancelled", "state:e1:cancelled"), events.log) + assertTrue(events.listeners.single() === listener) controller.ack(listOf(record.eventId)) assertNull(store.load("e1")) assertFalse(store.entryDir("e1").exists()) } @Test - fun `cancel of a settled entry forgets it now and keeps its unacked record`() { + fun `cancel of a settled entry forgets it now with its unacked outcomes`() { controller.enqueue(parsed()) val eventId = "00000000-0000-0000-0000-00000000000b" + val older = "00000000-0000-0000-0000-00000000001b" + journal.append(record(older, generation = 0)) journal.append(record(eventId, kind = EventJournal.KIND_ERROR)) + journal.append(record("00000000-0000-0000-0000-00000000002b", id = "other")) store.save(store.load("e1")!!.copy(state = EntryState.ERROR, settledEventId = eventId)) events.log.clear() controller.cancel("e1") assertNull(store.load("e1")) assertFalse(store.entryDir("e1").exists()) assertEquals(emptyList(), events.log) - assertNotNull(journal.find(eventId)) + assertEquals(listOf("other"), journal.unacknowledged().map { it.id }) + } + + @Test + fun `cancel of a live entry whose journal can not write rejects E_STORAGE and changes nothing`() { + controller.enqueue(parsed()) + val before = store.load("e1")!! + events.log.clear() + scheduler.cancelled.clear() + val dir = tmp.root.resolve("journal") + dir.setWritable(false) + val e = try { + assertThrows(QueueException::class.java) { controller.cancel("e1") } + } finally { + dir.setWritable(true) + } + assertEquals(QueueException.E_STORAGE, e.code) + assertEquals(before, store.load("e1")) + assertEquals(emptyList(), journal.unacknowledged()) + assertEquals(emptyList(), events.log) + assertEquals(emptyList(), scheduler.cancelled) + } + + @Test + fun `cancel whose entry save fails stops the work, emits, rejects, and a retry adds no second outcome`() { + controller.enqueue(parsed()) + events.log.clear() + scheduler.cancelled.clear() + val dir = store.entryDir("e1") + dir.setWritable(false) + val error = try { + assertThrows(QueueException::class.java) { controller.cancel("e1") } + } finally { + dir.setWritable(true) + } + assertEquals(QueueException.E_STORAGE, error.code) + val record = journal.unacknowledged().single() + assertEquals(EventJournal.KIND_CANCELLED, record.kind) + assertEquals(EntryState.QUEUED, store.load("e1")!!.state) // the save was lost + assertEquals(listOf("e1"), scheduler.cancelled) // a running request can not settle again + assertEquals(listOf("settled:e1:cancelled"), events.log) + // JS calls cancel() again: the journaled cancel is applied, not a second one. + events.log.clear() + controller.cancel("e1") + val e = store.load("e1")!! + assertEquals(EntryState.CANCELLED, e.state) + assertEquals(record.eventId, e.settledEventId) + assertEquals(listOf(record.eventId), journal.unacknowledged().map { it.eventId }) + assertEquals(listOf("state:e1:cancelled"), events.log) + } + + @Test + fun `cancel over the journal cap keeps its own record`() { + val small = EventJournal(tmp.newFolder("small"), maxEntries = 1) + val smallController = QueueController(store, small, settings, events, scheduler, { false }, { now }) + val named = "00000000-0000-0000-0000-000000000042" + small.append(record(named, id = "done")) + store.save(entry(id = "done", state = EntryState.ERROR, settledEventId = named)) + smallController.enqueue(parsed()) + smallController.cancel("e1") + val own = store.load("e1")!!.settledEventId!! + assertNotNull(small.find(own)) // without it, the sweep forgets the row with no outcome + assertNotNull(small.find(named)) + } + + @Test + fun `cancel with no listener journals at 0 and does not emit the outcome`() { + controller.enqueue(parsed()) + journal.stopListening(listener) + events.log.clear() + controller.cancel("e1") + assertEquals(listOf("state:e1:cancelled"), events.log) + assertEquals(0, journal.unacknowledged().single().deliveries) } @Test @@ -525,8 +667,8 @@ class QueueControllerTest { @Test fun `each replay counts one more delivery`() { journal.append(record("00000000-0000-0000-0000-00000000000f")) - assertEquals(2, controller.unacknowledged().single().deliveries) - assertEquals(3, controller.unacknowledged().single().deliveries) + assertEquals(2, controller.unacknowledged(listener).single().deliveries) + assertEquals(3, controller.unacknowledged(listener).single().deliveries) } // MARK: - boot sweep @@ -588,6 +730,24 @@ class QueueControllerTest { assertNotNull(store.load("c")) } + @Test + fun `sweep keeps a completed entry whose record is held in memory`() { + val eventId = "00000000-0000-0000-0000-000000000015" + val dir = tmp.root.resolve("journal") + dir.setWritable(false) + try { + journal.appendOrHold(record(eventId)) + } finally { + dir.setWritable(true) + } + store.save(entry(state = EntryState.COMPLETED, settledEventId = eventId)) + controller.sweep() + assertNotNull(store.load("e1")) + // Its ack still forgets it. + controller.ack(listOf(eventId)) + assertNull(store.load("e1")) + } + @Test fun `sweep leaves paused entries alone`() { controller.pause() diff --git a/android/src/test/java/ai/openspace/backgroundupload/QueueStoreTest.kt b/android/src/test/java/ai/openspace/backgroundupload/QueueStoreTest.kt index a4ae7232..98d107ba 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/QueueStoreTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/QueueStoreTest.kt @@ -103,14 +103,15 @@ class QueueStoreTest { fun `compute holds the lock across load, transform, and save`() { // A module transition and a worker's update race. If the lock did not // span all three steps, the update could land between load and save and - // be erased. Serialized, both effects survive. + // be erased. The update must block while the transform runs. val s = store() s.save(entry(descriptor = desc(url = null, file = "/f", parts = listOf(part(0, 10), part(10, 20))))) val inTransform = CountDownLatch(1) + val finish = CountDownLatch(1) val computing = Thread { s.compute("e1") { e -> inTransform.countDown() - Thread.sleep(300) + finish.await(5, TimeUnit.SECONDS) e!!.copy(expiresAt = 99_000) } }.apply { start() } @@ -118,6 +119,13 @@ class QueueStoreTest { val updating = Thread { s.update("e1") { e -> e.copy(descriptor = e.descriptor!!.copy(parts = ChunkedParts.withAccepted(e.descriptor.parts!!, 0))) } }.apply { start() } + // Without the lock the update finishes (TERMINATED) while the transform waits. + val deadline = System.currentTimeMillis() + 5_000 + while (updating.state != Thread.State.BLOCKED && updating.state != Thread.State.TERMINATED && + System.currentTimeMillis() < deadline + ) Thread.sleep(5) + assertEquals(Thread.State.BLOCKED, updating.state) + finish.countDown() computing.join() updating.join() val final = s.load("e1")!! @@ -155,7 +163,7 @@ class QueueStoreTest { s.remove("e1") assertNull(s.load("e1")) assertFalse(s.entryDir("e1").exists()) - assertNull(index.get("e1")) + assertEquals(emptyList(), index.snapshot()) } @Test @@ -171,7 +179,7 @@ class QueueStoreTest { // A process relaunch: a fresh index loaded from disk. val fresh = RequestIndex() QueueStore(dir, fresh).loadIndex() - assertEquals("error", fresh.get("b")!!.state) + assertEquals(listOf("error"), fresh.snapshot().map { it.state }) } @Test diff --git a/android/src/test/java/ai/openspace/backgroundupload/RetryClassifierTest.kt b/android/src/test/java/ai/openspace/backgroundupload/RetryClassifierTest.kt index cb21bf72..189bf8be 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/RetryClassifierTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/RetryClassifierTest.kt @@ -50,7 +50,13 @@ class RetryClassifierTest { assertTrue(file is Verdict.Terminal && file.errorKind == "file") val other = RetryClassifier.classifyFailure(IllegalArgumentException("bad url"), fileExists = true) assertEquals(Verdict.Terminal("unknown", "bad url"), other) - assertEquals("network", RetryClassifier.failureKind(IOException(), true)) + } + + @Test + fun `the attempt errorKind of a failure comes from its verdict`() { + assertEquals("network", RetryClassifier.failureKind(RetryClassifier.classifyFailure(IOException(), true))) + assertEquals("file", RetryClassifier.failureKind(RetryClassifier.classifyFailure(IOException(), false))) + assertEquals("unknown", RetryClassifier.failureKind(RetryClassifier.classifyFailure(RuntimeException("boom"), true))) } private val policy = RetryClassifier.Policy(baseMs = 1_000, maxMs = 7_200_000, jitter = 0.0, exempt = defaultExempt) diff --git a/android/src/test/java/ai/openspace/backgroundupload/SmallPartsTest.kt b/android/src/test/java/ai/openspace/backgroundupload/SmallPartsTest.kt index f133bc9e..a5bfda74 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/SmallPartsTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/SmallPartsTest.kt @@ -6,11 +6,7 @@ import androidx.work.WorkInfo.State.ENQUEUED import androidx.work.WorkInfo.State.FAILED import androidx.work.WorkInfo.State.RUNNING import androidx.work.WorkInfo.State.SUCCEEDED -import kotlinx.coroutines.async -import kotlinx.coroutines.delay import kotlinx.coroutines.runBlocking -import kotlinx.coroutines.sync.withPermit -import kotlinx.coroutines.withTimeoutOrNull import org.junit.Assert.assertEquals import org.junit.Assert.assertFalse import org.junit.Assert.assertNull @@ -21,11 +17,21 @@ class AttemptEventTest { private val response = UploadResponse(401, "x".repeat(5_000), mapOf("a" to "b")) @Test - fun `the body is cut at 4 KB and flagged`() { + fun `the body is cut at 4 KB of UTF-8 and flagged`() { val e = AttemptEvent.ofResponse(entry(attempts = 2), "r1", "https://x", null, response, accepted = false, at = 7) - assertEquals(AttemptEvent.MAX_BODY_CHARS, e.responseBody!!.length) + assertEquals(BodyCap.ATTEMPT_MAX_BYTES, e.responseBody!!.length) assertEquals(true, e.responseBodyTruncated) assertEquals(2, e.attempt) + // 3-byte characters: the cut backs off to a whole character. + val wide = AttemptEvent.ofResponse(entry(), "r1", "https://x", null, response.copy(body = "\u20ac".repeat(2_000)), false, 7) + assertEquals(BodyCap.ATTEMPT_MAX_BYTES / 3, wide.responseBody!!.length) + } + + @Test + fun `a response the stream cap cut is flagged even under 4 KB`() { + val e = AttemptEvent.ofResponse(entry(), "r1", "https://x", null, response.copy(body = "short", truncated = true), false, 7) + assertEquals("short", e.responseBody) + assertEquals(true, e.responseBodyTruncated) } @Test @@ -114,7 +120,7 @@ class RequestIndexTest { fun `setBytes on a missing id is a no-op`() { val index = RequestIndex() index.setBytes("nope", 5) - assertNull(index.get("nope")) + assertEquals(emptyList(), index.snapshot()) } @Test @@ -124,9 +130,9 @@ class RequestIndexTest { index.put(running.toRow()) index.setBytes("e1", 60) index.put(running.copy(attempts = 2).toRow()) - assertEquals(60, index.get("e1")!!.bytesSent) + assertEquals(60, index.snapshot().single().bytesSent) index.put(running.copy(state = EntryState.QUEUED).toRow()) - assertEquals(0, index.get("e1")!!.bytesSent) + assertEquals(0, index.snapshot().single().bytesSent) } } @@ -157,11 +163,14 @@ class TransferSemaphoreTest { @Test fun `the global cap is 4 and a fifth request waits`() = runBlocking { assertEquals(4, MAX_TRANSFER_CONCURRENCY) - val holders = (1..4).map { async { transferSemaphore.withPermit { delay(200) } } } - delay(20) - val fifth = withTimeoutOrNull(50) { transferSemaphore.withPermit { } } - assertNull(fifth) - holders.forEach { it.await() } - assertEquals(Unit, withTimeoutOrNull(500) { transferSemaphore.withPermit { } }) + repeat(4) { transferSemaphore.acquire() } + try { + assertFalse(transferSemaphore.tryAcquire()) // no fifth permit + transferSemaphore.release() + assertTrue(transferSemaphore.tryAcquire()) // one freed, one taken + } finally { + repeat(4) { transferSemaphore.release() } + } + assertEquals(4, transferSemaphore.availablePermits) } } diff --git a/android/src/test/java/ai/openspace/backgroundupload/TestSupport.kt b/android/src/test/java/ai/openspace/backgroundupload/TestSupport.kt index f8e405f5..1effda61 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/TestSupport.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/TestSupport.kt @@ -86,22 +86,24 @@ internal fun record( ) /** Records every event in order, as "state::" and "settled::". */ -internal class RecordingEvents(var live: Boolean = true) : QueueEvents { +internal class RecordingEvents : QueueEvents { val log = mutableListOf() val rows = mutableListOf() val records = mutableListOf() + /** The listener each settled event went to, in order. */ + val listeners = mutableListOf() + override fun state(row: RequestRow) { rows += row log += "state:${row.id}:${row.state}" } - override fun settled(record: EventJournal.SettledRecord) { + override fun settled(record: EventJournal.SettledRecord, listener: Any) { records += record + listeners += listener log += "settled:${record.id}:${record.kind}" } - - override fun canDeliver() = live } internal class FakeScheduler : WorkScheduler { diff --git a/android/src/test/java/ai/openspace/backgroundupload/UploadOutcomeTest.kt b/android/src/test/java/ai/openspace/backgroundupload/UploadOutcomeTest.kt index 187349ed..9976f240 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/UploadOutcomeTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/UploadOutcomeTest.kt @@ -5,7 +5,6 @@ import org.junit.Assert.assertEquals import org.junit.Assert.assertFalse import org.junit.Assert.assertTrue import org.junit.Test -import java.io.IOException class UploadOutcomeTest { @@ -57,19 +56,4 @@ class UploadOutcomeTest { assertTrue(UploadOutcome.isAccepted(409, "already completed", rules)) assertFalse(UploadOutcome.isAccepted(410, "already completed", rules)) } - - @Test - fun `IOException with a missing file is a file error`() { - assertEquals("file", UploadOutcome.errorKind(IOException("gone"), fileExists = false)) - } - - @Test - fun `IOException with the file present is a network error`() { - assertEquals("network", UploadOutcome.errorKind(IOException("reset"), fileExists = true)) - } - - @Test - fun `a non-IO error is unknown`() { - assertEquals("unknown", UploadOutcome.errorKind(RuntimeException("boom"), fileExists = true)) - } } diff --git a/android/src/test/java/ai/openspace/backgroundupload/WorkerOpsTest.kt b/android/src/test/java/ai/openspace/backgroundupload/WorkerOpsTest.kt index 0e7e1df9..5c28fff6 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/WorkerOpsTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/WorkerOpsTest.kt @@ -2,6 +2,7 @@ package ai.openspace.backgroundupload import org.junit.Assert.assertEquals import org.junit.Assert.assertFalse +import org.junit.Assert.assertNotNull import org.junit.Assert.assertNull import org.junit.Assert.assertThrows import org.junit.Assert.assertTrue @@ -10,6 +11,7 @@ import org.junit.Rule import org.junit.Test import org.junit.rules.TemporaryFolder import java.io.File +import java.util.concurrent.CyclicBarrier class WorkerOpsTest { @get:Rule @@ -23,15 +25,28 @@ class WorkerOpsTest { private var now = 20_000L private lateinit var ops: WorkerOps private lateinit var controller: QueueController + private val listener = Any() + private lateinit var journalDir: File @Before fun setUp() { val root = tmp.newFolder("queue") store = QueueStore(root, RequestIndex()) - journal = EventJournal(tmp.newFolder("journal")) + journalDir = tmp.newFolder("journal") + journal = EventJournal(journalDir, retryLater = { _, _ -> }) settings = QueueSettingsStore(File(root, "settings.json")) ops = WorkerOps(store, journal, settings, events, scheduler) { now } controller = QueueController(store, journal, settings, events, scheduler, { false }, { now }) + journal.drain(listener) // JS is subscribed + } + + private fun withJournalReadOnly(block: () -> T): T { + journalDir.setWritable(false) + try { + return block() + } finally { + journalDir.setWritable(true) + } } private val ok = UploadResponse(200, """{"id":7}""", mapOf("x" to "y")) @@ -45,6 +60,30 @@ class WorkerOpsTest { assertEquals(listOf("state:e1:running"), events.log) } + @Test + fun `begin applies a record of the entry's own generation and does not run it again`() { + // The process died between the journal write and the store transition, + // and WorkManager runs the entry again before any boot sweep. + val own = "00000000-0000-0000-0000-000000000021" + val older = "00000000-0000-0000-0000-000000000022" + store.save(entry(state = EntryState.RUNNING, generation = 2)) + journal.append(record(older, generation = 2, at = 1)) + journal.append(record(own, generation = 2, at = 2)) + assertNull(ops.begin("e1")) + val e = store.load("e1")!! + assertEquals(EntryState.COMPLETED, e.state) + assertEquals(own, e.settledEventId) + assertEquals(listOf(own), journal.unacknowledged().map { it.eventId }) // the extra one is acked + assertEquals(listOf("state:e1:completed"), events.log) + } + + @Test + fun `begin runs an entry whose only record is of an older generation`() { + store.save(entry(state = EntryState.QUEUED, generation = 2)) + journal.append(record("00000000-0000-0000-0000-000000000023", generation = 1)) + assertEquals(EntryState.RUNNING, ops.begin("e1")!!.state) + } + @Test fun `begin does nothing for a paused, settled, or legacy entry`() { store.save(entry(id = "p", state = EntryState.PAUSED)) @@ -90,13 +129,94 @@ class WorkerOpsTest { } @Test - fun `a settle with JS dead starts at 0 deliveries, so the first replay is 1`() { - events.live = false + fun `a settle with no listener starts at 0 deliveries, so the first replay is 1`() { + journal.stopListening(listener) store.save(entry(state = EntryState.RUNNING)) assertTrue(ops.settle("e1", 1, Settlement.Completed(ok, "u", "POST"))) assertEquals(0, journal.unacknowledged().single().deliveries) assertEquals(listOf("state:e1:completed"), events.log) // nothing to emit to - assertEquals(1, controller.unacknowledged().single().deliveries) + assertEquals(1, controller.unacknowledged(listener).single().deliveries) + } + + @Test + fun `a settle whose journal can not write holds the record, settles, and emits`() { + store.save(entry(state = EntryState.RUNNING)) + assertTrue(withJournalReadOnly { ops.settle("e1", 1, Settlement.Completed(ok, "u", "POST")) }) + val e = store.load("e1")!! + assertEquals(EntryState.COMPLETED, e.state) + assertTrue(journal.isHeld(e.settledEventId!!)) + assertEquals(listOf("settled:e1:completed", "state:e1:completed"), events.log) + // The sweep does not forget a row whose record is held, and the ack does. + controller.sweep() + assertNotNull(store.load("e1")) + controller.ack(listOf(e.settledEventId!!)) + assertNull(store.load("e1")) + } + + @Test + fun `a settle whose store write fails leaves the record for the next run to apply`() { + store.save(entry(state = EntryState.RUNNING)) + val dir = store.entryDir("e1") + dir.setWritable(false) + val stood = try { + ops.settle("e1", 1, Settlement.Completed(ok, "u", "POST")) + } finally { + dir.setWritable(true) + } + assertTrue(stood) + assertEquals(EntryState.RUNNING, store.load("e1")!!.state) + val record = journal.unacknowledged().single() + assertEquals(listOf("settled:e1:completed"), events.log) // no state event: the row did not change + // WorkManager runs it again: begin applies the record, no second send. + assertNull(ops.begin("e1")) + assertEquals(record.eventId, store.load("e1")!!.settledEventId) + } + + @Test + fun `a settle after a cancel whose save failed applies the cancel, not a second outcome`() { + // cancel() journaled its record, then its entry save failed: the entry + // is still running, and its request comes back. + val cancelId = "00000000-0000-0000-0000-000000000041" + store.save(entry(state = EntryState.RUNNING)) + journal.append(record(cancelId, kind = EventJournal.KIND_CANCELLED)) + assertFalse(ops.settle("e1", 1, Settlement.Completed(ok, "u", "POST"))) + val e = store.load("e1")!! + assertEquals(EntryState.CANCELLED, e.state) + assertEquals(cancelId, e.settledEventId) + assertEquals(listOf(cancelId), journal.unacknowledged().map { it.eventId }) + assertEquals(listOf("state:e1:cancelled"), events.log) // no second outcome + } + + @Test + fun `a live settle goes to the listener that drained, not the newest module`() { + store.save(entry(id = "a", state = EntryState.RUNNING)) + ops.settle("a", 1, Settlement.Completed(ok, "u", "POST")) + // A reload: the next module's JS drains and takes over. + val next = Any() + controller.unacknowledged(next) + store.save(entry(id = "b", state = EntryState.RUNNING)) + ops.settle("b", 1, Settlement.Completed(ok, "u", "POST")) + assertEquals(2, events.listeners.size) + assertTrue(events.listeners[0] === listener) + assertTrue(events.listeners[1] === next) + } + + @Test + fun `a settle over the journal cap spares every record a row names`() { + val small = EventJournal(tmp.newFolder("small"), maxEntries = 2) + val smallOps = WorkerOps(store, small, settings, events, scheduler) { now } + val named = "00000000-0000-0000-0000-000000000031" + small.append(record(named, id = "done")) + store.save(entry(id = "done", state = EntryState.ERROR, settledEventId = named)) + File(tmp.root, "small/$named.json").setLastModified(1_000) + small.append(record("00000000-0000-0000-0000-000000000032", id = "orphan")) + File(tmp.root, "small/00000000-0000-0000-0000-000000000032.json").setLastModified(2_000) + store.save(entry(state = EntryState.RUNNING)) + smallOps.settle("e1", 1, Settlement.Completed(ok, "u", "POST")) + val left = small.unacknowledged().map { it.eventId } + assertTrue(left.contains(named)) + assertTrue(left.contains(store.load("e1")!!.settledEventId)) + assertEquals(2, left.size) } @Test @@ -120,12 +240,30 @@ class WorkerOpsTest { } @Test - fun `a response that lands during pause still settles`() { + fun `an accepted response that lands during pause still settles`() { store.save(entry(state = EntryState.PAUSED)) - assertTrue(ops.settle("e1", 1, Settlement.Failed("http", "HTTP 400", ok.copy(code = 400), null, "u", "POST"))) - val e = store.load("e1")!! - assertEquals(EntryState.ERROR, e.state) - assertEquals("http", journal.unacknowledged().single().errorKind) + assertTrue(ops.settle("e1", 1, Settlement.Completed(ok, "u", "POST"))) + assertEquals(EntryState.COMPLETED, store.load("e1")!!.state) + } + + @Test + fun `a failure that lands during pause does not settle`() { + store.save(entry(state = EntryState.PAUSED)) + assertFalse(ops.settle("e1", 1, Settlement.Failed("http", "HTTP 400", ok.copy(code = 400), null, "u", "POST"))) + assertEquals(EntryState.PAUSED, store.load("e1")!!.state) + assertEquals(emptyList(), journal.unacknowledged()) + assertEquals(emptyList(), events.log) + } + + @Test + fun `a failed simple settle keeps the live bytes, and a chunked one keeps its accepted bytes`() { + store.save(entry(state = EntryState.RUNNING)) + ops.settle("e1", 1, Settlement.Failed("http", "HTTP 400", ok.copy(code = 400), null, "u", "POST", bytesSent = 5)) + assertEquals(5, store.load("e1")!!.bytesSent) + assertEquals(5, journal.unacknowledged().single().bytesSent) + store.save(entry(id = "c", state = EntryState.RUNNING).copy(bytesSent = 10)) + ops.settle("c", 1, Settlement.Failed("file", "gone", null, 1, "u", "PUT")) + assertEquals(10, store.load("c")!!.bytesSent) } @Test @@ -268,16 +406,29 @@ class WorkerOpsTest { // MARK: - deliveries @Test - fun `JS that subscribes between the check and the append still gets the outcome live with deliveries 1`() { - // canDeliver() is false at the record build and true right after the append. - val answers = ArrayDeque(listOf(false, true)) - val flipping = object : QueueEvents by events { - override fun canDeliver() = answers.removeFirstOrNull() ?: true + fun `a settle racing the first drain reaches JS exactly once with deliveries 1`() { + // The drain sets the listener and scans under one journal lock, and the + // append decides 0 or 1 under it. Either the drain returns the record, + // or the settle emits it live; never both, never neither. + repeat(200) { i -> + val id = "race-$i" + val owner = Any() + journal.stopListening(listener) + journal.stopListening(owner) + store.save(entry(id = id, state = EntryState.RUNNING)) + events.records.clear() + val start = CyclicBarrier(2) + var drained: List = emptyList() + val drain = Thread { start.await(); drained = controller.unacknowledged(owner).filter { it.id == id } } + drain.start() + start.await() + ops.settle(id, 1, Settlement.Completed(ok, "u", "POST")) + drain.join() + val live = events.records.filter { it.id == id } + val seen = drained + live + assertEquals("iteration $i", 1, seen.size) + assertEquals("iteration $i", 1, seen.single().deliveries) + journal.ack(listOf(seen.single().eventId)) } - val flipOps = WorkerOps(store, journal, settings, flipping, scheduler) { now } - store.save(entry(state = EntryState.RUNNING)) - assertTrue(flipOps.settle("e1", 1, Settlement.Completed(ok, "u", "POST"))) - assertEquals(1, journal.unacknowledged().single().deliveries) - assertEquals(1, events.records.single().deliveries) } } From 196e3f56914fd49c9465a318e9d2d6e72bcceaf9 Mon Sep 17 00:00:00 2001 From: Dylan Murphy Date: Mon, 28 Sep 2026 13:11:04 -0400 Subject: [PATCH 3/3] Android test: clear the stale listener in the settle-vs-drain race test The test cleared the listener with stopListening(listener), but from the second iteration on the listener was the previous iteration's owner, so nothing was cleared. When the settle won the race, the journal stamped one live delivery and the drain counted a second one. CI failed about one run in five. Now each iteration clears whatever listener is set and asserts that none is set before the race starts. Co-Authored-By: Claude Fable 5.1 --- .../java/ai/openspace/backgroundupload/WorkerOpsTest.kt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/android/src/test/java/ai/openspace/backgroundupload/WorkerOpsTest.kt b/android/src/test/java/ai/openspace/backgroundupload/WorkerOpsTest.kt index 5c28fff6..4ffb7fac 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/WorkerOpsTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/WorkerOpsTest.kt @@ -413,8 +413,10 @@ class WorkerOpsTest { repeat(200) { i -> val id = "race-$i" val owner = Any() - journal.stopListening(listener) - journal.stopListening(owner) + // The previous iteration's drain left its owner as the listener. Clear + // it, so the race starts with no listener every time. + journal.listener()?.let { journal.stopListening(it) } + assertFalse(journal.isListening()) store.save(entry(id = id, state = EntryState.RUNNING)) events.records.clear() val start = CyclicBarrier(2)