diff --git a/README.md b/README.md index b146c38c..c0189ddc 100644 --- a/README.md +++ b/README.md @@ -191,6 +191,21 @@ notification. That notification is also the worker's foreground-service notification, so a silent request runs as an ordinary background worker and the OS may defer or restart it. Reserve it for small payloads. +### Android platform notes + +**Headless time limit.** On API 31 and later, a WorkManager run that starts +from the background usually cannot start its foreground service. The run +then has JobScheduler's limit of about 10 minutes. A single body (`data`, +`form`, `file`) that does not finish in that time starts again from byte 0 +at the next run, after a growing backoff. A body that needs more than +10 minutes headless cannot finish that way. Use `parts` for large bodies: +accepted parts are kept across runs. iOS has no equal limit. + +**Backups.** The queue store (`files/rnbgupload-chunked/`) and the journal +(`files/rnbgupload-settled/`) hold request headers, including auth tokens, +and staged bodies. Set `android:allowBackup="false"` in the host app, or +exclude those two directories in its backup rules. + # Reliable delivery 1. **Write-ahead.** Entry, descriptor, and staged body persist before any diff --git a/android/consumer-rules.pro b/android/consumer-rules.pro index 754b74ca..d7627d06 100644 --- a/android/consumer-rules.pro +++ b/android/consumer-rules.pro @@ -1,32 +1,43 @@ # These rules go to consumers through consumerProguardFiles. Thus a minified # release build of the host app keeps these guarantees. # -# Gson persists Upload, NotificationConfig, and EventJournal.Entry. Upload goes -# into WorkManager input data. NotificationConfig goes into SharedPreferences. -# Entry goes into the on-disk event journal. The library reads them back later, -# across app restarts AND across app updates. Gson finds fields by name through -# reflection. Gson also needs the generic Signature attribute to rebuild typed -# collections. Thus, if R8 renames a field or removes Signature, it corrupts the -# persisted state silently: +# Gson persists the queue entry (entry.json), the queue settings +# (settings.json), the settled-outcome journal, NotificationConfig +# (SharedPreferences), and reads the v9 journal and v9 chunked manifests at +# the first v10 launch. The library reads them back later, across app +# restarts AND across app updates. Gson finds fields by name through +# reflection, and it needs the generic Signature attribute to rebuild typed +# collections. Thus, if R8 renames a field or removes Signature, it corrupts +# the persisted state silently: # -# * Upload.accept is a List. Without Signature, Gson decodes the -# elements as bare maps. Then no rule ever matches, and a configured accept -# status (for example 409) is reported as an http error, not as a completed -# upload. ChunkedManifest.parts has the same shape and the same failure. -# * A journal Entry from an older build fails to parse if field names changed. -# The library then drops the Entry as malformed. This loses the terminal -# outcomes that the journal exists to keep. A ChunkedManifest is the resume -# record for a chunked upload, and it fails in the same way. +# * Descriptor.accept is a List and Descriptor.parts a +# List. Without Signature, Gson decodes the elements as bare maps. +# Then no accept rule matches, and every chunked part reads as unsent. +# * A record from an older build fails to parse if field names changed. The +# library drops it as malformed. That loses the outcomes the journal +# exists to keep, and the entries the queue exists to run. +# * EntryState is an enum persisted by its @SerializedName wire string. # -# Debug builds are not minified and round-trip correctly. Thus neither failure +# Debug builds are not minified and round-trip correctly, so neither failure # is reproducible without R8. Keep these rules. -keepattributes Signature -keepattributes *Annotation* --keep class ai.openspace.backgroundupload.Upload { *; } --keep class ai.openspace.backgroundupload.Upload$* { *; } --keep class ai.openspace.backgroundupload.NotificationConfig { *; } --keep class ai.openspace.backgroundupload.EventJournal$Entry { *; } --keep class ai.openspace.backgroundupload.ChunkedManifest { *; } --keep class ai.openspace.backgroundupload.ChunkedManifest$* { *; } +# v10 queue +-keep class ai.openspace.backgroundupload.QueueEntry { *; } +-keep class ai.openspace.backgroundupload.EntryState { *; } +-keep class ai.openspace.backgroundupload.Descriptor { *; } +-keep class ai.openspace.backgroundupload.FormPart { *; } +-keep class ai.openspace.backgroundupload.RetryOverride { *; } +-keep class ai.openspace.backgroundupload.StagedBody { *; } +-keep class ai.openspace.backgroundupload.Part { *; } +-keep class ai.openspace.backgroundupload.QueueSettings { *; } +-keep class ai.openspace.backgroundupload.RetryDefaults { *; } +-keep class ai.openspace.backgroundupload.EventJournal$SettledRecord { *; } +-keep class ai.openspace.backgroundupload.EventJournal$Response { *; } -keep class ai.openspace.backgroundupload.UploadOutcome$AcceptRule { *; } +-keep class ai.openspace.backgroundupload.NotificationConfig { *; } + +# v9 files read once at the first v10 launch +-keep class ai.openspace.backgroundupload.LegacyImport$V9Entry { *; } +-keep class ai.openspace.backgroundupload.LegacyManifest { *; } diff --git a/android/src/main/java/ai/openspace/backgroundupload/AtomicFiles.kt b/android/src/main/java/ai/openspace/backgroundupload/AtomicFiles.kt new file mode 100644 index 00000000..bf291b53 --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/AtomicFiles.kt @@ -0,0 +1,53 @@ +package ai.openspace.backgroundupload + +import java.io.File +import java.io.FileOutputStream +import java.io.IOException +import java.nio.channels.FileChannel +import java.nio.file.StandardOpenOption + +/** + * Write-ahead file writes. Every durable file in the library goes through + * [writeAtomically]: write a tmp sibling, fsync it, rename it over the + * target. A crash at any point leaves either the old target or the new one, + * never a partial file. + */ +internal object AtomicFiles { + const val TMP_SUFFIX = ".tmp" + + fun tmpFor(target: File) = File(target.parentFile, target.name + TMP_SUFFIX) + + /** Throws IOException when the target could not be replaced. The old target is then intact. */ + fun writeAtomically(target: File, write: (FileOutputStream) -> Unit) { + val parent = target.parentFile ?: throw IOException("no parent directory for ${target.path}") + if (!parent.isDirectory && !parent.mkdirs()) { + throw IOException("could not create ${parent.path}") + } + val tmp = tmpFor(target) + try { + FileOutputStream(tmp).use { out -> + write(out) + out.flush() + out.fd.sync() + } + if (!tmp.renameTo(target)) throw IOException("could not rename ${tmp.path} to ${target.name}") + } catch (error: Throwable) { + tmp.delete() + throw error + } + syncDirectory(parent) + } + + fun writeText(target: File, text: String) = + writeAtomically(target) { it.write(text.toByteArray(Charsets.UTF_8)) } + + /** + * Makes a rename durable. This works on Linux (Android). Some file systems + * do not allow it, so a failure is ignored: the rename itself is still atomic. + */ + fun syncDirectory(dir: File) { + runCatching { + FileChannel.open(dir.toPath(), StandardOpenOption.READ).use { it.force(true) } + } + } +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/AttemptEvent.kt b/android/src/main/java/ai/openspace/backgroundupload/AttemptEvent.kt new file mode 100644 index 00000000..1274c3d8 --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/AttemptEvent.kt @@ -0,0 +1,88 @@ +package ai.openspace.backgroundupload + +import com.facebook.react.bridge.WritableMap + +/** + * One HTTP attempt, before the library interprets it. Live only: never + * journaled. [outcome] is `completed` when the response is accepted and + * `error` otherwise, so a 401 is `error` with httpCode 401 even though the + * entry parks. A transport failure is `error` with its own errorKind. Pause, + * cancel, supersede, and a system stop emit no attempt event. + */ +data class AttemptEvent( + val id: String, + val key: String, + val requestId: String, + val attempt: Int, + val url: String, + val method: String, + val partIndex: Int?, + val outcome: String, + val httpCode: Int?, + val responseBody: String?, + val responseBodyTruncated: Boolean?, + val responseHeaders: Map?, + val errorKind: String?, + val errorMessage: String?, + val at: Long, +) { + companion object { + fun ofResponse( + entry: QueueEntry, + requestId: String, + url: String, + partIndex: Int?, + response: UploadResponse, + accepted: Boolean, + at: Long, + ): AttemptEvent { + val (body, cut) = BodyCap.cap(response.body, BodyCap.ATTEMPT_MAX_BYTES) + return AttemptEvent( + id = entry.id, key = entry.key, requestId = requestId, attempt = entry.attempts, + url = url, method = entry.descriptor?.method ?: "POST", partIndex = partIndex, + outcome = if (accepted) "completed" else "error", + httpCode = response.code, responseBody = body, responseBodyTruncated = cut || response.truncated, + responseHeaders = response.headers, + errorKind = if (accepted) null else "http", + errorMessage = if (accepted) null else "HTTP ${response.code}", + at = at, + ) + } + + fun ofFailure( + entry: QueueEntry, + requestId: String, + url: String, + partIndex: Int?, + errorKind: String, + message: String, + at: Long, + ) = AttemptEvent( + id = entry.id, key = entry.key, requestId = requestId, attempt = entry.attempts, + url = url, method = entry.descriptor?.method ?: "POST", partIndex = partIndex, + outcome = "error", httpCode = null, responseBody = null, responseBodyTruncated = null, + responseHeaders = null, errorKind = errorKind, errorMessage = message, + at = at, + ) + } + + fun toMap(): Map = LinkedHashMap().apply { + put("id", id) + put("key", key) + put("requestId", requestId) + put("attempt", attempt.toDouble()) + put("url", url) + put("method", method) + partIndex?.let { put("partIndex", it.toDouble()) } + put("outcome", outcome) + httpCode?.let { put("httpCode", it.toDouble()) } + responseBody?.let { put("responseBody", it) } + responseBodyTruncated?.let { put("responseBodyTruncated", it) } + responseHeaders?.let { put("responseHeaders", it) } + errorKind?.let { put("errorKind", it) } + errorMessage?.let { put("errorMessage", it) } + put("at", at.toDouble()) + } + + fun toWritableMap(): WritableMap = JsonBridge.toWritableMap(toMap()) +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/BodyCap.kt b/android/src/main/java/ai/openspace/backgroundupload/BodyCap.kt new file mode 100644 index 00000000..ade8e7df --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/BodyCap.kt @@ -0,0 +1,67 @@ +package ai.openspace.backgroundupload + +import okio.Buffer +import okio.BufferedSource +import java.nio.charset.Charset + +/** + * Response body caps, in UTF-8 bytes. A cut never splits a character: it + * backs off to the last whole one. [read] applies the cap while the body + * streams in, so a huge error page never sits in memory whole. + */ +object BodyCap { + /** 1 MB, the RawResponse cap of a settled outcome. */ + const val SETTLED_MAX_BYTES = 1_048_576 + + /** 4 KB, the body cap of a live attempt event. */ + const val ATTEMPT_MAX_BYTES = 4 * 1024 + + /** A body as text and whether the cap cut it. */ + data class Capped(val text: String, val truncated: Boolean) + + /** + * Reads at most [maxBytes] of [source]. Bytes past the cap are not read. + * A UTF-8 body is cut on a character boundary; another charset is cut at + * the byte cap and decoded as it is. + */ + fun read(source: BufferedSource, maxBytes: Int, charset: Charset = Charsets.UTF_8): Capped { + val buffer = Buffer() + val limit = maxBytes.toLong() + 1 + while (buffer.size < limit) { + if (source.read(buffer, limit - buffer.size) == -1L) break + } + val truncated = buffer.size > maxBytes + val bytes = buffer.readByteArray() + val keep = if (!truncated) bytes.size + else if (charset == Charsets.UTF_8) utf8Boundary(bytes, maxBytes) + else maxBytes + return Capped(String(bytes, 0, keep, charset), truncated) + } + + /** [text] cut to at most [maxBytes] of UTF-8. Null stays null. */ + fun cap(text: String?, maxBytes: Int): Pair { + if (text == null) return null to false + val bytes = text.toByteArray(Charsets.UTF_8) + if (bytes.size <= maxBytes) return text to false + return String(bytes, 0, utf8Boundary(bytes, maxBytes), Charsets.UTF_8) to true + } + + /** + * The longest prefix length of [bytes], at most [max], that does not end + * inside a UTF-8 sequence. A continuation byte is 10xxxxxx. + */ + internal fun utf8Boundary(bytes: ByteArray, max: Int): Int { + if (bytes.size <= max) return bytes.size + var end = max + // bytes[end] is the first byte cut off. While it continues a sequence, + // the sequence started before the cut, so drop its start too. A UTF-8 + // character is at most 4 bytes; past 3 steps the bytes are not UTF-8, + // and the cut stays at max. + var steps = 0 + while (end > 0 && steps < 3 && (bytes[end].toInt() and 0xC0) == 0x80) { + end-- + steps++ + } + return if ((bytes[end].toInt() and 0xC0) == 0x80) max else end + } +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/BodyStaging.kt b/android/src/main/java/ai/openspace/backgroundupload/BodyStaging.kt new file mode 100644 index 00000000..e65d4647 --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/BodyStaging.kt @@ -0,0 +1,212 @@ +package ai.openspace.backgroundupload + +import java.io.BufferedOutputStream +import java.io.File +import java.io.OutputStream +import java.nio.file.Files +import java.nio.file.StandardCopyOption +import java.util.UUID + +/** + * Writes a request body into the entry directory before the entry is saved. + * After enqueue resolves, every byte the request needs is in the library + * directory, so the caller may delete its source. + * + * | Descriptor | Staged file | Content-Type | + * | none | none | none | + * | data | body-.json | application/json unless the caller set one | + * | form | body-.multipart| multipart/form-data; boundary=… (replaces the caller's) | + * | file | file- (copy) | the caller's own | + * | file + parts | blob / blob- (move) | the caller's own | + * + * Each staged file has a name per generation, so a replace writes a new file + * next to the old one. The old body is deleted only after the new entry is + * saved. Every source is checked before anything is written. A chunked blob + * of generation 1 keeps the v9 name `blob`, so v9 blobs are found in place. + */ +object BodyStaging { + const val JSON_CONTENT_TYPE = "application/json" + private const val BUFFER = 64 * 1024 + private val CRLF = "\r\n".toByteArray() + + data class Staged(val body: StagedBody, val headers: Map) + + /** + * @param ownedBlob the chunked blob the entry (or a v9 manifest) already + * owns. The parts run over it when the caller's file is gone (v9 recreate). + * @param keepOwned run over [ownedBlob] even when the caller's file is + * present. Only for the same parts, whose accepted flags carry over. + */ + fun stage( + d: Descriptor, + dir: File, + generation: Int, + ownedBlob: File? = null, + keepOwned: Boolean = false, + ): Staged { + checkSources(d, dir, generation, ownedBlob, keepOwned) + val body = when (d.bodyKind) { + StagedBody.NONE -> StagedBody(StagedBody.NONE, null, null, 0) + StagedBody.JSON -> { + val name = "body-$generation.json" + val target = File(dir, name) + writeJson(d.dataJson!!, target) + StagedBody(StagedBody.JSON, name, null, target.length()) + } + StagedBody.MULTIPART -> { + val name = "body-$generation.multipart" + val target = File(dir, name) + val boundary = newBoundary() + writeMultipart(d.form!!, boundary, target) + StagedBody(StagedBody.MULTIPART, name, boundary, target.length()) + } + StagedBody.FILE -> { + val name = "file-$generation" + val target = File(dir, name) + copyFile(File(d.file!!), target) + StagedBody(StagedBody.FILE, name, null, target.length()) + } + else -> { + val parts = d.parts!! + val source = File(d.file!!) + val runOver = chunkedInput(d, dir, generation, ownedBlob, keepOwned) + // Checked before the move, so a rejected plan moves nothing. + if (!ChunkedParts.tilesExactly(parts, runOver.length())) { + throw QueueException( + QueueException.E_INVALID, + "parts must tile the file exactly: [0, ${runOver.length()})", + ) + } + val blob = if (runOver.path == source.path) { + File(dir, blobName(generation)).also { takeOwnership(source, it) } + } else runOver + StagedBody(StagedBody.CHUNKED, blob.name, null, ChunkedParts.totalBytes(parts)) + } + } + return Staged(body, headersFor(d.headers, body)) + } + + /** The Content-Type rule of the table above. */ + fun headersFor(headers: Map, body: StagedBody): Map = + when (body.kind) { + StagedBody.JSON -> + if (HeaderMap.contains(headers, "Content-Type")) headers + else headers + ("Content-Type" to JSON_CONTENT_TYPE) + StagedBody.MULTIPART -> + HeaderMap.without(headers, "Content-Type") + + ("Content-Type" to "multipart/form-data; boundary=${body.boundary}") + else -> headers + } + + /** The chunked blob name of [generation]. Generation 1 keeps the v9 name. */ + fun blobName(generation: Int) = + if (generation <= 1) QueueStore.BLOB_FILE else "${QueueStore.BLOB_FILE}-$generation" + + /** + * The file a chunked plan runs over, before anything moves: + * 1. [ownedBlob] when [keepOwned] and it exists; + * 2. the caller's file when present (a new file wins over an old blob); + * 3. this generation's blob, left by a crash after the move; + * 4. [ownedBlob], when the caller's file was moved away at an earlier enqueue. + */ + internal fun chunkedInput(d: Descriptor, dir: File, generation: Int, ownedBlob: File?, keepOwned: Boolean): File { + val source = File(d.file!!) + val leftover = File(dir, blobName(generation)) + return when { + keepOwned && ownedBlob != null && ownedBlob.exists() -> ownedBlob + source.isFile -> source + leftover.exists() -> leftover + ownedBlob != null && ownedBlob.exists() -> ownedBlob + else -> throw QueueException(QueueException.E_FILE_MISSING, "file does not exist: ${source.path}") + } + } + + /** Every source must exist and be readable before any write. */ + internal fun checkSources(d: Descriptor, dir: File, generation: Int, ownedBlob: File?, keepOwned: Boolean) { + d.form?.forEach { part -> + part.path?.let { requireReadable(File(it), "form part '${part.name}'") } + } + when (d.bodyKind) { + StagedBody.FILE -> requireReadable(File(d.file!!), "file") + StagedBody.CHUNKED -> chunkedInput(d, dir, generation, ownedBlob, keepOwned) + } + } + + private fun requireReadable(file: File, what: String) { + if (!file.isFile || !file.canRead()) { + throw QueueException(QueueException.E_FILE_MISSING, "$what does not exist or can not be read: ${file.path}") + } + } + + internal fun writeJson(dataJson: String, target: File) = + AtomicFiles.writeText(target, dataJson) + + /** + * RFC 7578. Per part: the boundary line, Content-Disposition with the name + * (and a filename for a file part), Content-Type, a blank line, the bytes, + * CRLF. Then the closing delimiter. File parts stream from disk. + */ + internal fun writeMultipart(form: List, boundary: String, target: File) { + AtomicFiles.writeAtomically(target) { raw -> + val out = BufferedOutputStream(raw, BUFFER) + for (part in form) { + out.ascii("--$boundary") + out.write(CRLF) + val disposition = StringBuilder("Content-Disposition: form-data; name=\"") + .append(escapeQuoted(part.name)).append('"') + if (part.path != null) { + val fileName = part.fileName ?: File(part.path).name + disposition.append("; filename=\"").append(escapeQuoted(fileName)).append('"') + } + out.write(disposition.toString().toByteArray(Charsets.UTF_8)) + out.write(CRLF) + out.write("Content-Type: ${part.contentType}".toByteArray(Charsets.UTF_8)) + out.write(CRLF) + out.write(CRLF) + if (part.path != null) { + File(part.path).inputStream().use { it.copyTo(out, BUFFER) } + } else { + out.write((part.string ?: "").toByteArray(Charsets.UTF_8)) + } + out.write(CRLF) + } + out.ascii("--$boundary--") + out.write(CRLF) + out.flush() + } + } + + /** The WHATWG form encoding of a quoted name: `"`, CR and LF are percent-encoded. */ + internal fun escapeQuoted(value: String): String = + value.replace("\"", "%22").replace("\r", "%0D").replace("\n", "%0A") + + internal fun copyFile(source: File, target: File) { + source.inputStream().use { input -> + AtomicFiles.writeAtomically(target) { out -> input.copyTo(out, BUFFER) } + } + } + + /** + * Moves the caller's file to [blob] (v9 verbatim). A crash between the move + * and the entry save leaves the bytes at the blob path with no entry; a + * retry whose source is gone adopts them ([chunkedInput] step 3). + */ + internal fun takeOwnership(source: File, blob: File) { + if (source.absoluteFile == blob.absoluteFile) return + if (!source.exists()) { + if (blob.exists()) return + throw QueueException(QueueException.E_FILE_MISSING, "file does not exist: ${source.path}") + } + blob.parentFile?.mkdirs() + if (blob.exists()) blob.delete() + if (!source.renameTo(blob)) { + // renameTo can not cross file systems. Files.move falls back to copy + delete. + Files.move(source.toPath(), blob.toPath(), StandardCopyOption.REPLACE_EXISTING) + } + blob.parentFile?.let { AtomicFiles.syncDirectory(it) } + } + + internal fun newBoundary(): String = "----RNBGU" + UUID.randomUUID().toString().replace("-", "") + + private fun OutputStream.ascii(text: String) = write(text.toByteArray(Charsets.US_ASCII)) +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/ChunkedEngine.kt b/android/src/main/java/ai/openspace/backgroundupload/ChunkedEngine.kt index e398c750..bbbee474 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/ChunkedEngine.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/ChunkedEngine.kt @@ -6,108 +6,22 @@ import kotlinx.coroutines.sync.Semaphore import kotlinx.coroutines.sync.withPermit /** - * The pure scheduling half of chunked execution: the window, the retry - * policy, and the backoff. It is kept free of Android and OkHttp types. Thus - * the highest-consequence invariants (at most WINDOW parts in flight, and - * never two requests for one part index) are unit-testable on a plain JVM. - * [ChunkedUploadWorker] supplies the part executor. + * The pure scheduling half of chunked execution: the window. It has no + * Android or OkHttp types, so its two invariants (at most WINDOW parts in + * flight, never two requests for one part index) are unit-testable on a + * plain JVM. The retry decisions moved to [RetryClassifier] in v10. */ object ChunkedEngine { - // The number of parts of one upload in flight at one time. This is a library - // constant, not an option. If soak data shows that a different value is - // better, this constant changes, not the API. + // Parts of one upload in flight at one time. A library constant, not an option. const val WINDOW = 3 - // The library retries a non-accepted, non-transient HTTP response this many - // times per part. Then the response becomes a terminal error and stalls the - // upload. The budget is small on purpose. A response that the server repeats - // (401, 400) does not change without a new startUpload. Only transient - // failures retry without a limit. - const val PART_HTTP_RETRIES = 3 - - // The poll interval while the network is unusable (offline, or waiting for - // wifi). The interval is constant, not exponential. We wait for conditions - // here; we do not back off a server. And expiresAt bounds the total wait. - const val CONNECTIVITY_POLL_MS = 10_000L - - private const val BACKOFF_BASE_MS = 1_000L - private const val BACKOFF_CAP_MS = 60_000L - - // A 5xx means that the server failed, not that the request is wrong. Thus it - // retries like a transport failure: without a limit, until expiresAt. - fun isTransientHttp(code: Int) = code in 500..599 - - /** What a starting worker must do for the manifest that it finds (or does not find). */ - enum class StartAction { - /** - * No manifest exists. The upload was completed and acknowledged, or it was - * explicitly removed, while this run sat in the queue. Both are legitimate - * ends, already reported (or deliberately not reported). Exit with success - * and in silence. A journaled terminal here would be a spurious 'file' - * error for an upload that nobody owns any more. - */ - NO_MANIFEST, - - /** - * Every part is already accepted: this is a trailing resume run. Re-report - * the journaled completion (never mint a second terminal event) and stop. - * Start no foreground service and no transfers. - */ - ALREADY_COMPLETE, - - /** Pending parts remain. Run the engine. */ - RUN, - } - - fun startAction(manifest: ChunkedManifest?): StartAction = when { - manifest == null -> StartAction.NO_MANIFEST - manifest.allAccepted -> StartAction.ALREADY_COMPLETE - else -> StartAction.RUN - } - - /** How a run that found (or produced) an all-accepted manifest reports the completion. */ - sealed class CompletionReport { - /** An unacknowledged 'completed' entry exists. Re-emit it. Never mint a second entry. */ - data class ReEmit(val entry: EventJournal.Entry) : CompletionReport() - - /** A fresh completion with no journal entry yet. Journal and emit a new entry. */ - object Mint : CompletionReport() - - /** - * A trailing run with nothing unacknowledged: the completion was journaled - * AND acknowledged. Nobody is owed an event. This occurs when the trailing - * run races ackEvents, which deletes the journal entry just before the - * manifest. An event minted here would be a duplicate 'completed' for an - * upload that the consumer already settled. - */ - object None : CompletionReport() - } - - fun completionReport( - unacked: List, - uploadId: String, - freshCompletion: Boolean, - ): CompletionReport { - val existing = unacked.firstOrNull { it.uploadId == uploadId && it.type == "completed" } - return when { - existing != null -> CompletionReport.ReEmit(existing) - freshCompletion -> CompletionReport.Mint - else -> CompletionReport.None - } - } - - /** Exponential backoff for transient failures: 1s, 2s, 4s, and more, capped at 60s. */ - fun backoffMs(attempt: Int): Long = - (BACKOFF_BASE_MS shl (attempt - 1).coerceIn(0, 6)).coerceAtMost(BACKOFF_CAP_MS) - /** - * Runs [executePart] exactly one time per index, with at most [window] parts - * at one time. One coroutine per part index is what guarantees that no two - * requests for the same part are in flight (concurrent PUTs of one partNum - * are verified unsafe on the server side). An executor that throws cancels - * the remaining parts, and the error propagates. Terminal classification is - * the caller's job. + * Runs [executePart] exactly one time per index, with at most [window] + * parts at one time. One coroutine per index is what guarantees no two + * requests for one part are in flight (concurrent PUTs of one partNum are + * unsafe on the server). An executor that throws cancels the other parts, + * and the error propagates. */ suspend fun run( partIndexes: List, diff --git a/android/src/main/java/ai/openspace/backgroundupload/ChunkedManifest.kt b/android/src/main/java/ai/openspace/backgroundupload/ChunkedManifest.kt deleted file mode 100644 index 6bbe523b..00000000 --- a/android/src/main/java/ai/openspace/backgroundupload/ChunkedManifest.kt +++ /dev/null @@ -1,298 +0,0 @@ -package ai.openspace.backgroundupload - -import android.content.Context -import com.facebook.react.bridge.ReadableMap -import com.google.gson.Gson -import java.io.File -import java.util.Base64 - -/** - * The durable record of one chunked upload: the moved source file, the parts - * that the consumer authored, and which of them the server has accepted. - * [ChunkedManifestStore] persists it as JSON at startUpload, BEFORE the work - * is enqueued. Thus a worker rescheduled after process death (or a startUpload - * after a crash, a stop, or a reauth) resumes from it without a call into JS. - * This manifest IS the resume mechanism. - * - * The data shape is kept free of Android and React types (Gson round-trips - * it, and JVM tests construct it directly). The ReadableMap parsing lives in - * the companion, like [Upload]'s. - */ -data class ChunkedManifest( - val id: String, - /** The library-owned copy of the bytes (the consumer's file, renamed in). */ - val sourcePath: String, - val parts: List, - val accept: List, - /** Epoch ms. After this time, the upload stops with errorKind 'expired'. */ - val expiresAt: Long, - val wifiOnly: Boolean, - val noNotification: Boolean, - val createdAt: Long, -) { - /** - * One part, exactly as the consumer authored it. The library sends the file - * bytes [start, end) as the body of a PUT to [url], with [headers] - * unchanged. It never derives or edits a protocol field. - */ - data class Part( - val url: String, - val headers: Map, - val start: Long, - val end: Long, // exclusive - val accepted: Boolean = false, - ) { - val size get() = end - start - } - - val showsNotification get() = !noNotification - val totalBytes get() = parts.sumOf { it.size } - val acceptedBytes get() = parts.filter { it.accepted }.sumOf { it.size } - - /** The server's auto-publish condition. It is the only thing that 'completed' may mean. */ - val allAccepted get() = parts.all { it.accepted } - - fun isExpired(now: Long) = now >= expiresAt - - fun pendingIndexes() = parts.indices.filter { !parts[it].accepted } - - fun withPartAccepted(index: Int) = copy( - parts = parts.mapIndexed { i, part -> if (i == index) part.copy(accepted = true) else part }, - ) - - class ReconcileException(message: String) : IllegalArgumentException(message) - - /** - * A startUpload re-call with an existing id is one of two things: - * - * **Resume** — the incoming parts are the SAME array (identical count, - * ranges, and urls). The headers, the accept rules, expiresAt, and the flags - * come from the new call. This is how fresh auth reaches stalled parts, and - * how a salvage extends the deadline. The accepted part statuses, the moved - * source, and createdAt survive from this manifest. A resume is permitted at - * any time, running or not. A running worker re-reads the stored copy before - * every attempt. - * - * **Recreate** — a DIFFERENT parts array. The consumer re-authored the - * upload under a fresh server uploadId after the old one died (it expired - * past the server's 31-day window, or it is otherwise unrecoverable). The - * owned bytes are kept. The parts are replaced as a whole, and every part - * status resets to unsent. The headers, the accept rules, and expiresAt come - * from the new call. The new ranges must tile exactly [0, blobSize). A - * partial or overlapping cover would silently upload wrong bytes. A recreate - * is accepted only while the upload is NOT running (stalled on a terminal - * error, expired, or cancelled). A different parts array while a worker - * executes is a consumer bug, not a recreate, because the in-flight requests - * belong to the old parts. - */ - fun reconcile(incoming: ChunkedManifest, running: Boolean, blobSize: Long): ChunkedManifest { - if (samePartsAs(incoming)) { - // Accepted flags follow the RANGE, not the array index. samePartsAs is - // order-independent, so the same tile can sit at a different index. - val acceptedStarts = parts.filter { it.accepted }.map { it.start }.toSet() - return incoming.copy( - sourcePath = sourcePath, - createdAt = createdAt, - parts = incoming.parts.map { it.copy(accepted = it.start in acceptedStarts) }, - ) - } - if (running) throw ReconcileException( - "chunked upload '$id' is running; a different parts array is only accepted once it stops", - ) - if (!tilesExactly(incoming.parts, blobSize)) throw ReconcileException( - "chunked upload '$id' recreate parts must tile exactly [0, $blobSize)", - ) - return incoming.copy(sourcePath = sourcePath, createdAt = createdAt) - } - - // Order-independent, like tilesExactly. The same tiles, authored in a - // different order, are the SAME upload (a resume), never a recreate. - private fun samePartsAs(incoming: ChunkedManifest): Boolean { - if (incoming.parts.size != parts.size) return false - val stored = parts.sortedBy { it.start } - val fresh = incoming.parts.sortedBy { it.start } - return stored.indices.all { i -> - fresh[i].url == stored[i].url && - fresh[i].start == stored[i].start && - fresh[i].end == stored[i].end - } - } - - companion object { - /** - * Whether [parts] cover [0, size) exactly: no gap, no overlap, and nothing - * past the end. Order-independent, like everything else about parts. - */ - fun tilesExactly(parts: List, size: Long): Boolean { - if (parts.isEmpty()) return false - val sorted = parts.sortedBy { it.start } - var cursor = 0L - for (part in sorted) { - if (part.start != cursor || part.end <= part.start) return false - cursor = part.end - } - return cursor == size - } - - /** - * Validates a first-call (create) manifest against the just-owned bytes. - * Like a recreate, the parts must tile exactly [0, blobSize). A partial or - * overlapping cover would silently upload wrong bytes. It throws BEFORE - * the manifest is saved. Thus the moved blob stays adoptable by a - * corrected retry (see UploaderModule.takeOwnership's orphan branch). - */ - fun validatedForCreate(incoming: ChunkedManifest, blobSize: Long): ChunkedManifest { - if (!tilesExactly(incoming.parts, blobSize)) throw ReconcileException( - "chunked upload '${incoming.id}' parts must tile exactly [0, $blobSize)", - ) - return incoming - } - - /** @param sourcePath the library-owned destination, not the consumer's path. */ - fun fromReadableMap(map: ReadableMap, sourcePath: String, createdAt: Long): ChunkedManifest { - val partsArr = map.getArray("parts") ?: throw Upload.MissingOptionException("parts") - if (partsArr.size() == 0) throw IllegalArgumentException("parts must be a non-empty array") - if (!map.hasKey("expiresAt")) throw Upload.MissingOptionException("expiresAt") - return ChunkedManifest( - id = map.getString("id") ?: throw Upload.MissingOptionException("id"), - sourcePath = sourcePath, - parts = (0 until partsArr.size()).map { i -> - val part = partsArr.getMap(i) ?: throw Upload.MissingOptionException("parts[$i]") - val range = part.getMap("range") ?: throw Upload.MissingOptionException("parts[$i].range") - Part( - url = part.getString("url") ?: throw Upload.MissingOptionException("parts[$i].url"), - headers = parseHeaderMap(part.getMap("headers")), - start = range.getDouble("start").toLong(), - end = range.getDouble("end").toLong(), - ) - }, - accept = parseAcceptRules(map.getArray("accept")), - expiresAt = map.getDouble("expiresAt").toLong(), - wifiOnly = if (map.hasKey("wifiOnly")) map.getBoolean("wifiOnly") else false, - noNotification = if (map.hasKey("noNotification")) map.getBoolean("noNotification") else false, - createdAt = createdAt, - ) - } - } -} - -/** - * A file-backed store: one directory per upload id, which holds - * `manifest.json` and `blob` (the moved source bytes). It has the same - * durability pattern as [EventJournal]: tmp+rename writes, and corrupt files - * read as absent. It is reachable from a bare Context, because the worker can - * run in a process where React never initialized. - */ -class ChunkedManifestStore(private val dir: File) { - - companion object { - private val gson = Gson() - - @Volatile - private var instance: ChunkedManifestStore? = null - - fun get(context: Context): ChunkedManifestStore = - instance ?: synchronized(this) { - instance ?: ChunkedManifestStore(File(context.filesDir, "rnbgupload-chunked")) - .also { instance = it } - } - } - - init { - dir.mkdirs() - } - - // Upload ids come from the consumer, and they can contain path separators or - // other filesystem-hostile characters. Thus the directory name is an encoding - // of the id, never the id itself. The id is read back from the manifest, not - // decoded from the name. - private fun uploadDir(id: String) = - File(dir, Base64.getUrlEncoder().withoutPadding().encodeToString(id.toByteArray())) - - private fun manifestFile(id: String) = File(uploadDir(id), "manifest.json") - - /** Where startUpload moves the source file for this id. */ - fun blobFile(id: String) = File(uploadDir(id), "blob") - - @Synchronized - fun load(id: String): ChunkedManifest? { - val file = manifestFile(id) - if (!file.exists()) return null - val parsed = runCatching { gson.fromJson(file.readText(), ChunkedManifest::class.java) } - .getOrNull() - return validated(parsed) - } - - /** Throws on a write failure. A manifest that did not persist must fail the startUpload call. */ - @Synchronized - fun save(manifest: ChunkedManifest) { - val dir = uploadDir(manifest.id) - dir.mkdirs() - val tmp = File(dir, "manifest.tmp") - tmp.writeText(gson.toJson(manifest)) - if (!tmp.renameTo(manifestFile(manifest.id))) { - throw java.io.IOException("failed to persist chunked manifest for '${manifest.id}'") - } - } - - /** - * An atomic read-modify-write. Thus a worker that marks a part accepted can - * never clobber a concurrent startUpload's fresh headers (or another part's - * flag). Returns null, without a throw, when the manifest is gone or the - * write failed. A caller that can continue from memory does that. - */ - @Synchronized - fun update(id: String, transform: (ChunkedManifest) -> ChunkedManifest): ChunkedManifest? = - runCatching { - val manifest = load(id) ?: return null - val next = transform(manifest) - save(next) - next - }.getOrNull() - - /** - * An atomic create-or-transform. The store lock spans load, [transform], and - * save. Thus nothing — a running worker's markAccepted included — can write - * between them and be erased. startUpload's load, reconcile, and save must - * go through here, not as three separate calls. [transform] receives null - * when no manifest exists. Unlike [update], a transform that throws (a - * reconcile rejection) or a failed write propagates, because startUpload - * must fail loudly, not continue from memory. - */ - @Synchronized - fun compute(id: String, transform: (ChunkedManifest?) -> ChunkedManifest): ChunkedManifest { - val next = transform(load(id)) - save(next) - return next - } - - /** Whether a manifest is stored for this id (without parsing it). */ - @Synchronized - fun contains(id: String): Boolean = manifestFile(id).exists() - - /** Deletes the manifest AND the moved bytes. Does nothing for an unknown id (a simple upload). */ - @Synchronized - fun remove(id: String) { - uploadDir(id).deleteRecursively() - } - - @Synchronized - fun all(): List = - (dir.listFiles { f -> f.isDirectory } ?: emptyArray()) - .mapNotNull { d -> - val file = File(d, "manifest.json") - if (!file.exists()) return@mapNotNull null - validated(runCatching { gson.fromJson(file.readText(), ChunkedManifest::class.java) }.getOrNull()) - } - - // Gson does not use the constructor. Thus a corrupt or field-renamed file can - // make non-null Kotlin fields null. Reject a file that lacks a field that the - // engine relies on. Normalize an absent accept list; do not reject it. - @Suppress("SENSELESS_COMPARISON") - private fun validated(m: ChunkedManifest?): ChunkedManifest? { - if (m == null || m.id == null || m.sourcePath == null || m.parts == null) return null - if (m.parts.isEmpty()) return null - if (m.parts.any { it == null || it.url == null || it.headers == null }) return null - return if (m.accept == null) m.copy(accept = listOf()) else m - } -} diff --git a/android/src/main/java/ai/openspace/backgroundupload/ChunkedParts.kt b/android/src/main/java/ai/openspace/backgroundupload/ChunkedParts.kt new file mode 100644 index 00000000..5d8ac2e0 --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/ChunkedParts.kt @@ -0,0 +1,60 @@ +package ai.openspace.backgroundupload + +/** + * One part of a chunked upload, as the caller wrote it. The library sends the + * file bytes [start, end) to [url]. [accepted] is set when the server + * accepted the part. The field names match the v9 manifest, so a v9 + * manifest.json reads into this class unchanged. + */ +data class Part( + val url: String, + val headers: Map, + val start: Long, + val end: Long, // exclusive + val accepted: Boolean = false, +) { + val size get() = end - start +} + +/** Pure rules over a parts list. Moved from the v9 ChunkedManifest with the same meaning. */ +object ChunkedParts { + + /** Whether [parts] cover [0, size) exactly: no gap, no overlap, nothing past the end. Order does not matter. */ + fun tilesExactly(parts: List, size: Long): Boolean { + if (parts.isEmpty()) return false + var cursor = 0L + for (part in parts.sortedBy { it.start }) { + if (part.start != cursor || part.end <= part.start) return false + cursor = part.end + } + return cursor == size + } + + /** + * The same upload: the same count, ranges, and urls, in any order. Headers + * are not compared, because a resume sends fresh headers. + */ + fun sameParts(a: List, b: List): Boolean { + if (a.size != b.size) return false + val x = a.sortedBy { it.start } + val y = b.sortedBy { it.start } + return x.indices.all { i -> + x[i].url == y[i].url && x[i].start == y[i].start && x[i].end == y[i].end + } + } + + /** The [incoming] parts with the accepted flags of [stored]. A flag follows the range start, not the index. */ + fun carryAccepted(stored: List, incoming: List): List { + val acceptedStarts = stored.filter { it.accepted }.map { it.start }.toSet() + return incoming.map { it.copy(accepted = it.start in acceptedStarts) } + } + + fun totalBytes(parts: List): Long = parts.sumOf { it.size } + + fun acceptedBytes(parts: List): Long = parts.filter { it.accepted }.sumOf { it.size } + + fun pendingIndexes(parts: List): List = parts.indices.filter { !parts[it].accepted } + + fun withAccepted(parts: List, index: Int): List = + parts.mapIndexed { i, part -> if (i == index) part.copy(accepted = true) else part } +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/ChunkedUploadWorker.kt b/android/src/main/java/ai/openspace/backgroundupload/ChunkedUploadWorker.kt index 33750073..d236ec9a 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/ChunkedUploadWorker.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/ChunkedUploadWorker.kt @@ -1,410 +1,133 @@ package ai.openspace.backgroundupload -import android.app.NotificationManager import android.content.Context -import androidx.work.CoroutineWorker -import androidx.work.ForegroundInfo -import androidx.work.ListenableWorker import androidx.work.WorkerParameters -import kotlinx.coroutines.CancellationException -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.delay -import kotlinx.coroutines.sync.withPermit -import kotlinx.coroutines.withContext import java.io.File -import java.io.IOException -import java.util.UUID import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.atomic.AtomicLong +/** The WorkManager class for a chunked entry. The run is [EntryWorker]'s. */ +class ChunkedUploadWorker(context: Context, params: WorkerParameters) : EntryWorker(context, params) + /** - * Executes one chunked upload from its durable [ChunkedManifest]. The input - * data carries only the upload id. The manifest is the record: startUpload - * persists it before this work is enqueued. Thus a worker rescheduled after - * process death resumes from disk, with no JS involved. + * The parts of one chunked entry, at most [ChunkedEngine.WINDOW] at a time, + * each part through the shared 4-request semaphore. One logical entry has + * one progress stream (byte-weighted) and one outcome: completed only when + * every part is accepted. * - * One logical upload has one event stream: byte-weighted aggregate progress, - * and one terminal event. 'completed' is journaled only when every part is - * accepted. Every other terminal keeps the manifest and the bytes, so a later - * startUpload can resume. The bytes are deleted only when a 'completed' event - * is ACKED (see UploaderModule.ackEvents). + * Each part runs [EntryRun.attempt] until a verdict ends it. + * Per part: accepted → persist the flag; auth → the whole entry parks (the + * sibling parts stop); transient → a short backoff waits in the part while + * the siblings go on, a long one releases the whole worker (accepted parts + * are kept); terminal → the entry fails with that part's index. */ -class ChunkedUploadWorker(private val context: Context, params: WorkerParameters) : - CoroutineWorker(context, params) { - - companion object { - /** - * The key for the upload id in the worker's input data. It is a string - * literal for the same reason as [UploadWorker.PARAMS_KEY]: WorkManager's - * database persists it across builds, and it must survive R8 renames and - * refactors. - */ - const val ID_KEY = "chunkedUploadId" +internal class ChunkedTransfer(private val run: EntryRun) { - /** How often a starting worker re-checks [ChunkedWorkerGate] for its id. */ - private const val GATE_POLL_MS = 100L - } + /** A terminal part failure. Not a CancellationException, so it stops the sibling parts. */ + private class PartFailed(val settlement: Settlement.Failed) : Exception(settlement.message) - private lateinit var uploadId: String - private val store by lazy { ChunkedManifestStore.get(context) } - private val config by lazy { NotificationConfig.load(context) } - private val notificationManager = - context.getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager - - // The latest known manifest. Part executors re-read the stored copy before - // every attempt (see latest()). Thus a reconciling startUpload's fresh - // headers, and an extended expiresAt, reach a worker that already runs. - @Volatile - private var manifest: ChunkedManifest? = null - - @Volatile - private var connectivity = Connectivity.Ok - - // In-flight bytes per part index, for byte-weighted aggregate progress. private val partSent = ConcurrentHashMap() + private val acceptedHere = ConcurrentHashMap.newKeySet() private val acceptedBytes = AtomicLong(0) + private var total = 0L - private class ExpiredException : Exception("upload expired") - - private class SourceMissingException(path: String) : - IOException("chunked source file missing: $path") - - private class PartRejectedException(val partIndex: Int, val response: UploadResponse) : - Exception("part $partIndex rejected with HTTP ${response.code}") - - private class PartBeyondEofException(val partIndex: Int, message: String) : Exception(message) - - override suspend fun doWork(): Result = withContext(Dispatchers.IO) { - uploadId = inputData.getString(ID_KEY) ?: throw Throwable("No upload id") - - // Acquire the per-id execution gate BEFORE the first manifest read. A - // cancel-then-start can start this worker while the cancelled one still - // winds down, and two PUTs of one partNum are unsafe. Also, the manifest - // read occurs only after the gate is held. That is what makes the module's - // recreate check race-free (see ChunkedWorkerGate and - // ChunkedManifestStore.compute). - try { - while (!ChunkedWorkerGate.tryAcquire(uploadId, this@ChunkedUploadWorker)) { - delay(GATE_POLL_MS) - } - } catch (error: CancellationException) { - // Cancelled while waiting. A user cancel still owes its terminal event. - checkAndHandleCancellation() - throw error - } - try { - runUpload() - } finally { - ChunkedWorkerGate.release(uploadId, this@ChunkedUploadWorker) - } - } - - private suspend fun runUpload(): Result { - val initial = store.load(uploadId) - when (ChunkedEngine.startAction(initial)) { - // The upload was completed-and-acknowledged, or it was removed, while - // this run sat in the queue. Both are legitimate and already settled. - // Exit in silence. A terminal journaled here would be a spurious error - // for an upload that nobody owns. - ChunkedEngine.StartAction.NO_MANIFEST -> return Result.success() - // A trailing resume of a finished-but-unacknowledged upload. Re-report - // the journaled completion. Skip the foreground service and the engine. - ChunkedEngine.StartAction.ALREADY_COMPLETE -> { - manifest = initial - journalCompleted(freshCompletion = false) - return Result.success() - } - ChunkedEngine.StartAction.RUN -> Unit + @Volatile + private var lastAcceptedUrl: String? = null + + suspend fun run(start: QueueEntry): Settlement { + val d0 = start.descriptor!! + val parts = d0.parts!! + lastAcceptedUrl = parts.lastOrNull { it.accepted }?.url + val pending = ChunkedParts.pendingIndexes(parts) + // A run over an all-accepted entry that has not settled yet. + if (pending.isEmpty()) return Settlement.Completed(null, lastAcceptedUrl ?: d0.reportUrl, d0.method) + val blob = run.store.bodyFile(start) + if (blob == null || !blob.exists()) { + return Settlement.Failed("file", "the chunked file is missing", null, null, d0.reportUrl, d0.method) } - checkNotNull(initial) // RUN implies a manifest - manifest = initial - acceptedBytes.set(initial.acceptedBytes) - UploadProgress.add(uploadId, initial.totalBytes) - UploadProgress.set(uploadId, initial.acceptedBytes) + total = ChunkedParts.totalBytes(parts) + acceptedBytes.set(ChunkedParts.acceptedBytes(parts)) + run.progressStarted(total, acceptedBytes.get()) - // Initialization. A failure here is terminal: journaled, never retried. - // The EXCEPTION is a refused foreground start, which the transfer - // survives. try { - if (initial.showsNotification) { - ensureNotificationChannel(notificationManager, config) - setForeground(getForegroundInfo()) - } - } catch (error: Throwable) { - if (!isForegroundStartDenied(error)) { - if (!checkAndHandleCancellation()) { - UploadProgress.remove(uploadId) - handleFailure(error) - } - return terminalErrorResult() - } - // The app is in the background, and API 31+ refused the foreground - // start. This is the usual state for a WorkManager relaunch (a reboot, - // or a quota resume). The upload runs correctly without foreground - // priority. A failure here would brick every headless resume. - } - - return try { - ChunkedEngine.run(initial.pendingIndexes()) { index -> executePart(index) } - // Every executor returned. An executor returns only when its part was - // accepted. That is exactly the server's auto-publish condition. - UploadProgress.complete(uploadId) - journalCompleted(freshCompletion = true) - Result.success() - } catch (error: Throwable) { - if (checkAndHandleCancellation()) throw error - UploadProgress.remove(uploadId) - handleFailure(error) - terminalErrorResult() + ChunkedEngine.run(pending) { index -> executePart(index, blob, start.backoffStreak) } + } catch (failed: PartFailed) { + return failed.settlement } + // Every executor returned, and an executor returns only when its part was + // accepted: the server's auto-publish condition. + return Settlement.Completed(null, lastAcceptedUrl ?: d0.reportUrl, d0.method) } - /** - * Uploads one part until it is accepted, or throws. Terminal conditions - * (expiry, a missing source, or a non-accepted response out of retries) - * propagate and cancel the sibling parts. Everything transient retries here, - * bounded only by expiresAt. - */ - private suspend fun executePart(index: Int) { - var rejections = 0 - var transientAttempts = 0 + internal suspend fun executePart(index: Int, blob: File, initialStreak: Int) { + var streak = initialStreak while (true) { - val current = latest() - val part = current.parts[index] - if (part.accepted) return - if (current.isExpired(System.currentTimeMillis())) throw ExpiredException() - - // A range past the blob's EOF can never transmit. The read would fail - // on every attempt until expiry. Thus it is a terminal 'file' error - // immediately (iOS classifies it the same way). length() is 0 for a - // missing file. That case falls through to the transfer, which - // classifies it as source-missing. The failed-probe-reads-as-network - // default stays intact. - val blobLength = runCatching { File(current.sourcePath).length() }.getOrDefault(0L) - if (blobLength > 0L && part.end > blobLength) throw PartBeyondEofException( - index, - "part $index range [${part.start}, ${part.end}) exceeds source size $blobLength", - ) - - if (!validateAndReportConnectivity(current.wifiOnly)) { - delay(ChunkedEngine.CONNECTIVITY_POLL_MS) - continue + if (index in acceptedHere) return + val latest = run.ops.latest(run.entryId, run.generation) + val stored = latest.descriptor!!.parts!![index] + if (stored.accepted) return + if (RetryClassifier.isExpired(run.host.now(), latest.expiresAt)) throw EntryRun.ExpiredException() + + // A range past EOF can never be sent. length() is 0 for a missing file; + // that case falls through to the attempt, which classifies it as file. + val blobLength = runCatching { blob.length() }.getOrDefault(0L) + if (blobLength > 0L && stored.end > blobLength) { + throw PartFailed( + Settlement.Failed( + "file", + "part $index range [${stored.start}, ${stored.end}) exceeds the file size $blobLength", + null, index, stored.url, latest.descriptor.method, + ), + ) } + run.waitForNetwork() - val response = try { - transferSemaphore.withPermit { - okhttpUploadPart(uploadHttpClient, part, File(current.sourcePath)) { sent -> - onPartProgress(index, sent) - } + val a = run.attempt( + partIndex = index, + body = { _, part -> rangeRequestBody(blob, part!!.start, part.end) }, + onProgress = { sent -> onPartProgress(index, sent) }, + fileExists = { blob.exists() }, + ) + when (val r = a.result) { + is EntryRun.AttemptResult.Accepted -> { + markAccepted(index, a.entry.descriptor!!.parts!![index]) + return + } + is EntryRun.AttemptResult.Auth -> { + if (!r.reissue) throw EntryRun.ParkException(r.headerGeneration) + streak = 0 + } + EntryRun.AttemptResult.Transient -> { + onPartProgress(index, 0L) + streak++ + run.backoffOrRelease(a.policy, streak, a.entry.expiresAt) + } + is EntryRun.AttemptResult.Terminal -> { + onPartProgress(index, 0L) + val message = r.response?.let { "HTTP ${it.code} on part $index" } ?: r.message + throw PartFailed(Settlement.Failed(r.errorKind, message, r.response, index, a.url, a.method)) } - } catch (error: CancellationException) { - throw error - } catch (error: IOException) { - onPartProgress(index, 0L) - // The default is fileExists=true. Thus a failed probe reads as - // network, not file. - val fileExists = runCatching { File(current.sourcePath).exists() }.getOrDefault(true) - if (!fileExists) throw SourceMissingException(current.sourcePath) - transientAttempts++ - delay(ChunkedEngine.backoffMs(transientAttempts)) - continue - } - - if (UploadOutcome.isAccepted(response.code, response.body, current.accept)) { - markAccepted(index) - return - } - onPartProgress(index, 0L) - if (ChunkedEngine.isTransientHttp(response.code)) { - transientAttempts++ - delay(ChunkedEngine.backoffMs(transientAttempts)) - continue } - rejections++ - if (rejections > ChunkedEngine.PART_HTTP_RETRIES) throw PartRejectedException(index, response) - delay(ChunkedEngine.backoffMs(rejections)) } } - // The stored copy is the truth: a reconcile can have replaced the headers - // or expiresAt. Fall back to the in-memory copy only when the read fails. - private fun latest(): ChunkedManifest = - store.load(uploadId)?.also { manifest = it } ?: manifest!! - - private fun markAccepted(index: Int) { - // Persist the flag first, atomically against concurrent flips and - // reconciles. This is best-effort. A lost flag only re-sends this part on - // a later resume, and the consumer's accept rules absorb that ('already - // completed'). That is better than a failure of an upload that the server - // accepted. - manifest = store.update(uploadId) { it.withPartAccepted(index) } - ?: manifest?.withPartAccepted(index) - manifest?.parts?.get(index)?.let { acceptedBytes.addAndGet(it.size) } + private fun markAccepted(index: Int, part: Part) { + // Remembered here too, so a lost flag write does not re-send the part in this run. + acceptedHere += index + run.ops.markAccepted(run.entryId, run.generation, index) + acceptedBytes.addAndGet(part.size) + lastAcceptedUrl = part.url partSent.remove(index) - reportProgress() + report() } private fun onPartProgress(index: Int, sent: Long) { if (sent == 0L) partSent.remove(index) else partSent[index] = sent - reportProgress() + report() } - private fun reportProgress() { - val total = manifest?.totalBytes ?: return + private fun report() { val sent = (acceptedBytes.get() + partSent.values.sum()).coerceAtMost(total) - UploadProgress.set(uploadId, sent) - EventReporter.progress(uploadId, sent, total) - updateNotification() - } - - // A resume of a finished-but-unacknowledged upload (all parts accepted, - // 'completed' journaled, and the consumer re-called startUpload before the - // ack) must not mint a second terminal event. Re-emit the journaled one. - // Then a live listener still hears it, with the eventId that the consumer - // will acknowledge. And a trailing run whose completion was already ACKED - // reports nothing at all. See ChunkedEngine.CompletionReport. - private fun journalCompleted(freshCompletion: Boolean) { - val report = ChunkedEngine.completionReport( - EventJournal.get(context).unacknowledged(), - uploadId, - freshCompletion, - ) - when (report) { - is ChunkedEngine.CompletionReport.ReEmit -> EventReporter.emit(report.entry) - // No response fields, because no single response represents N accepted - // parts. - ChunkedEngine.CompletionReport.Mint -> journalAndEmit( - EventJournal.Entry( - eventId = UUID.randomUUID().toString(), - uploadId = uploadId, - type = "completed", - timestamp = System.currentTimeMillis(), - ), - ) - ChunkedEngine.CompletionReport.None -> Unit - } - } - - private fun handleFailure(error: Throwable) { - val entry = when (error) { - is ExpiredException -> errorEntry( - error = "upload expired before every part was accepted", - errorKind = "expired", - ) - is PartRejectedException -> { - val (body, truncated) = EventJournal.capBody(error.response.body) - errorEntry( - error = "HTTP ${error.response.code} on part ${error.partIndex}", - errorKind = "http", - partIndex = error.partIndex, - responseCode = error.response.code, - responseBody = body, - responseBodyTruncated = truncated, - responseHeaders = error.response.headers, - ) - } - is SourceMissingException -> errorEntry(error = error.message!!, errorKind = "file") - is PartBeyondEofException -> errorEntry( - error = error.message!!, - errorKind = "file", - partIndex = error.partIndex, - ) - else -> { - val fileExists = manifest?.let { m -> - runCatching { File(m.sourcePath).exists() }.getOrDefault(true) - } ?: true - errorEntry( - error = error.message ?: "Unknown exception", - errorKind = UploadOutcome.errorKind(error, fileExists), - ) - } - } - journalAndEmit(entry) - } - - private fun errorEntry( - error: String, - errorKind: String, - partIndex: Int? = null, - responseCode: Int? = null, - responseBody: String? = null, - responseBodyTruncated: Boolean = false, - responseHeaders: Map? = null, - ) = EventJournal.Entry( - eventId = UUID.randomUUID().toString(), - uploadId = uploadId, - type = "error", - timestamp = System.currentTimeMillis(), - error = error, - errorKind = errorKind, - partIndex = partIndex, - responseCode = responseCode, - responseBody = responseBody, - responseBodyTruncated = responseBodyTruncated, - responseHeaders = responseHeaders, - ) - - // The semantics are the same as UploadWorker's. Only a user cancel is - // terminal (journaled, cancelReason 'user'). A system stop emits nothing, - // because WorkManager will re-run this upload, and the manifest resumes it. - // The manifest and the bytes are kept in both cases. stopUpload's contract - // is that the next startUpload resumes. - private fun checkAndHandleCancellation(): Boolean { - if (!isStopped) return false - - UploadProgress.remove(uploadId) - - if (!UserCancellations.consume(uploadId)) return true - - journalAndEmit( - EventJournal.Entry( - eventId = UUID.randomUUID().toString(), - uploadId = uploadId, - type = "cancelled", - timestamp = System.currentTimeMillis(), - cancelReason = "user", - ), - ) - return true - } - - private fun journalAndEmit(entry: EventJournal.Entry) { - // A terminal event for an id whose manifest is gone would report an - // upload that nobody owns any more. Either removeUpload deleted it mid-run - // (its work cancel races the in-flight PUT's IOException), or a completed - // ack released it. Suppress the event; iOS's removedIds has the same idea. - // A user cancel keeps its manifest, so real 'cancelled' events pass - // through. - if (!store.contains(uploadId)) return - EventReporter.journalAndEmit(context, entry) + run.reportProgress(sent, total) } - - private fun validateAndReportConnectivity(wifiOnly: Boolean): Boolean { - connectivity = validateConnectivity(context, wifiOnly) - updateNotification() - return connectivity == Connectivity.Ok - } - - private fun updateNotification() { - if (manifest?.showsNotification != true) return - notificationManager.notify( - config.systemNotificationId, - buildUploadNotification(context, config, connectivity), - ) - } - - override suspend fun getForegroundInfo(): ForegroundInfo = - uploadForegroundInfo(config, buildUploadNotification(context, config, connectivity)) } - -/** - * The Result that a chunked run returns after it journals a terminal error: - * SUCCESS, deliberately. The journal and the manifest are the upload's outcome - * record, never the WorkManager row state. A row that finishes FAILED destroys - * every appended dependent: WorkManager marks the dependents of a failed - * prerequisite FAILED without a run. Thus a resume enqueued during the failing - * run's teardown window would silently never run (see the APPEND_OR_REPLACE - * note in UploaderModule.enqueueChunkedUpload). getAllUploads derives a - * chunked upload's state from its manifest (allAccepted), not from row states. - */ -internal fun terminalErrorResult(): ListenableWorker.Result = ListenableWorker.Result.success() diff --git a/android/src/main/java/ai/openspace/backgroundupload/ChunkedWorkerGate.kt b/android/src/main/java/ai/openspace/backgroundupload/ChunkedWorkerGate.kt deleted file mode 100644 index 424c627b..00000000 --- a/android/src/main/java/ai/openspace/backgroundupload/ChunkedWorkerGate.kt +++ /dev/null @@ -1,40 +0,0 @@ -package ai.openspace.backgroundupload - -import java.util.concurrent.ConcurrentHashMap - -/** - * At most one [ChunkedUploadWorker] EXECUTES per upload id, process-wide. - * - * The unique-work chain almost guarantees this, but not across a cancel. - * cancelUniqueWork marks the row CANCELLED immediately, while the cancelled - * worker's coroutine still winds down. Thus a startUpload that arrives right - * after a cancelUpload can enqueue (and start) a replacement worker while the - * old worker still has a part PUT in flight. Two concurrent PUTs of one - * partNum are verified unsafe on the server side. A starting worker acquires - * its id here, and a successor waits for the release. - * - * This is also the truthful "is this upload running" for the recreate rule. - * A worker registers before its first manifest read, and it releases in a - * finally block. WorkManager's row state stays RUNNING for a moment after - * doWork returns. This gate does not: it never reports a finished run as - * running. - * - * The gate is same-process only, like [UserCancellations]. A worker in a dead - * process holds nothing, and WorkManager runs our workers in the app process. - */ -object ChunkedWorkerGate { - private val holders = ConcurrentHashMap() - - /** True when [token] now holds the id, or already held it. False while another token holds it. */ - fun tryAcquire(id: String, token: Any): Boolean { - val current = holders.putIfAbsent(id, token) - return current == null || current === token - } - - /** Releases only when [token] is the holder. Thus a stale release cannot evict a successor. */ - fun release(id: String, token: Any) { - holders.remove(id, token) - } - - fun isRunning(id: String): Boolean = holders.containsKey(id) -} diff --git a/android/src/main/java/ai/openspace/backgroundupload/Diag.kt b/android/src/main/java/ai/openspace/backgroundupload/Diag.kt new file mode 100644 index 00000000..14382887 --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/Diag.kt @@ -0,0 +1,19 @@ +package ai.openspace.backgroundupload + +import android.util.Log + +/** + * Logging that is safe in the JVM unit tests. There, android.util.Log is a + * stub that throws, so every call is wrapped. + */ +internal object Diag { + const val TAG = "RNFileUploader" + + fun warn(message: String, error: Throwable? = null) { + runCatching { Log.w(TAG, message, error) } + } + + fun error(message: String, error: Throwable? = null) { + runCatching { Log.e(TAG, message, error) } + } +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/EnqueueRules.kt b/android/src/main/java/ai/openspace/backgroundupload/EnqueueRules.kt new file mode 100644 index 00000000..4331797e --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/EnqueueRules.kt @@ -0,0 +1,178 @@ +package ai.openspace.backgroundupload + +/** + * The same-id rules of enqueue() (plan 5.4), as pure functions. [decide] + * picks the action; the builders make the next entry from the staged body. + * + * | Stored entry | Action | + * | none, no v9 manifest | Create | + * | none, v9 manifest | AdoptV9: keep the blob and accepted parts | + * | legacy row, v9 manifest | AdoptV9 (generation + 1) | + * | legacy row, no manifest | Replace (generation + 1) | + * | same body, completed, record present | ReEmit: deliveries + 1, no re-run | + * | same body, completed, record gone | Replace (it was acked) | + * | same body, any other state | Resume (a settled one reopens: gen + 1, | + * | | attempts 0) | + * | different body, running | RejectRunning (E_RUNNING) | + * | different body, otherwise | Replace (generation + 1, attempts 0) | + */ +object EnqueueRules { + + sealed class Action { + object Create : Action() + /** [generation] is 1, or the legacy row's + 1. */ + data class AdoptV9(val manifest: LegacyManifest, val generation: Int) : Action() + data class ReEmit(val eventId: String) : Action() + object Resume : Action() + object Replace : Action() + object RejectRunning : Action() + } + + fun decide( + existing: QueueEntry?, + v9: LegacyManifest?, + incoming: Descriptor, + hasRecord: (eventId: String) -> Boolean, + ): Action { + if (existing == null) return if (v9 != null) Action.AdoptV9(v9, 1) else Action.Create + // An imported v9 outcome row. Its v9 chunked manifest, when present, is + // the upload's progress: adopt it, as with no row at all. + if (existing.legacy) return if (v9 != null) Action.AdoptV9(v9, existing.generation + 1) else Action.Replace + if (existing.sameBodyAs(incoming)) { + if (existing.state == EntryState.COMPLETED) { + val eventId = existing.settledEventId + return if (eventId != null && hasRecord(eventId)) Action.ReEmit(eventId) else Action.Replace + } + return Action.Resume + } + return if (existing.state == EntryState.RUNNING) Action.RejectRunning else Action.Replace + } + + /** + * The generation to stage a body for before the store lock is taken, or + * null to stage under the lock. Only a copied body (JSON, multipart, + * file), and only when no worker can change the decision meanwhile: a new + * id, or a replace of an entry that a worker can not take (not queued, + * not running). A chunked body is a move, which is fast, over a blob that + * a worker may be reading, so it always stages under the lock. + */ + fun preStageGeneration(existing: QueueEntry?, action: Action, incoming: Descriptor): Int? { + val copied = incoming.bodyKind.let { + it == StagedBody.JSON || it == StagedBody.MULTIPART || it == StagedBody.FILE + } + if (!copied) return null + return when (action) { + Action.Create -> 1 + Action.Replace -> existing + ?.takeIf { it.state != EntryState.QUEUED && it.state != EntryState.RUNNING } + ?.let { it.generation + 1 } + else -> null + } + } + + /** The parts an adopted v9 manifest runs with: its accepted flags when the parts are the same. */ + fun adoptedParts(v9: LegacyManifest, incoming: List): List = + if (ChunkedParts.sameParts(v9.parts, incoming)) ChunkedParts.carryAccepted(v9.parts, incoming) + else incoming + + private fun initialState(paused: Boolean) = if (paused) EntryState.PAUSED else EntryState.QUEUED + + /** A new entry (Create, AdoptV9). [parts] carries adopted accepted flags. */ + fun created( + p: EntryParsing.Parsed, + staged: BodyStaging.Staged, + parts: List?, + paused: Boolean, + headerGeneration: Int, + now: Long, + ): QueueEntry { + val runParts = parts ?: p.descriptor.parts + return QueueEntry( + id = p.id, + key = p.key, + varsJson = p.varsJson, + descriptor = p.descriptor.copy(headers = staged.headers, parts = runParts), + body = staged.body, + state = initialState(paused), + attempts = 0, + bytesSent = runParts?.let { ChunkedParts.acceptedBytes(it) } ?: 0L, + totalBytes = staged.body.totalBytes, + expiresAt = p.expiresAt, + createdAt = now, + updatedAt = now, + headerGeneration = headerGeneration, + generation = 1, + ) + } + + /** AdoptV9: a new entry over the v9 blob, at [generation]; over a legacy row it keeps the row's createdAt. */ + fun adopted( + p: EntryParsing.Parsed, + staged: BodyStaging.Staged, + parts: List?, + legacyRow: QueueEntry?, + generation: Int, + paused: Boolean, + headerGeneration: Int, + now: Long, + ): QueueEntry = created(p, staged, parts, paused, headerGeneration, now).copy( + createdAt = legacyRow?.createdAt ?: now, + generation = generation, + ) + + /** + * Same body. New headers, expiresAt, vars, accept, retry, and notification + * flag replace the stored ones; the body and accepted parts stay. A settled + * entry reopens with a fresh generation and attempts 0 (attempts count the + * current generation). A running one stays running (the worker reads the + * new headers before its next attempt). + */ + fun resumed( + existing: QueueEntry, + p: EntryParsing.Parsed, + paused: Boolean, + headerGeneration: Int, + now: Long, + ): QueueEntry { + val stored = existing.descriptor!! + val body = existing.body!! + val parts = stored.parts?.let { ChunkedParts.carryAccepted(it, p.descriptor.parts!!) } + val running = existing.state == EntryState.RUNNING + val reopen = existing.isSettled + return existing.copy( + key = p.key, + varsJson = p.varsJson, + descriptor = stored.copy( + headers = BodyStaging.headersFor(p.descriptor.headers, body), + parts = parts, + accept = p.descriptor.accept, + retry = p.descriptor.retry, + noNotification = p.descriptor.noNotification, + ), + state = if (running) EntryState.RUNNING else initialState(paused), + attempts = if (reopen) 0 else existing.attempts, + bytesSent = parts?.let { ChunkedParts.acceptedBytes(it) } ?: if (running) existing.bytesSent else 0L, + expiresAt = p.expiresAt, + updatedAt = now, + nextAttemptAt = null, + backoffStreak = 0, + headerGeneration = headerGeneration, + parkedGeneration = null, + generation = if (reopen) existing.generation + 1 else existing.generation, + settledEventId = if (reopen) null else existing.settledEventId, + ) + } + + /** Different body (or over a legacy or acked row). A new life over the same id. */ + fun replaced( + existing: QueueEntry, + p: EntryParsing.Parsed, + staged: BodyStaging.Staged, + paused: Boolean, + headerGeneration: Int, + now: Long, + ): QueueEntry = created(p, staged, null, paused, headerGeneration, now).copy( + createdAt = existing.createdAt, + generation = existing.generation + 1, + ) +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/EntryParsing.kt b/android/src/main/java/ai/openspace/backgroundupload/EntryParsing.kt new file mode 100644 index 00000000..f3c77ac6 --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/EntryParsing.kt @@ -0,0 +1,209 @@ +package ai.openspace.backgroundupload + +import com.facebook.react.bridge.ReadableArray +import com.facebook.react.bridge.ReadableMap +import com.facebook.react.bridge.ReadableType +import okhttp3.Headers +import okhttp3.HttpUrl.Companion.toHttpUrlOrNull +import java.net.URI + +/** + * Turns the EnqueueEntry `{ id, key, varsJson, descriptor }` into Kotlin + * values. JS has already validated the descriptor. Native checks only what + * it needs to run, and rejects anything else with E_INVALID. + * + * `varsJson` and `descriptor.dataJson` are JSON text. They cross as strings + * because React Native on iOS drops object keys whose value is null. Native + * keeps the text as it came: it is the body that goes out. A dataJson of + * "null" is the JSON body null, a real body. + */ +object EntryParsing { + class InvalidEntryException(message: String) : IllegalArgumentException(message) + + data class Parsed( + val id: String, + val key: String, + val varsJson: String, + val descriptor: Descriptor, + val expiresAt: Long, + ) + + private val METHODS = setOf("POST", "PUT", "PATCH", "DELETE", "GET") + + fun parse(entry: ReadableMap): Parsed { + val id = entry.string("id")?.takeIf { it.isNotEmpty() } ?: invalid("id is required") + val key = entry.string("key")?.takeIf { it.isNotEmpty() } ?: invalid("key is required") + val varsJson = entry.string("varsJson") ?: invalid("varsJson is required") + if (!JsonBridge.isJson(varsJson)) invalid("varsJson is not JSON text") + val d = entry.map("descriptor") ?: invalid("descriptor is required") + val expiresAt = d.number("expiresAt")?.toLong() ?: invalid("descriptor.expiresAt is required") + return Parsed(id, key, varsJson, descriptor(d), expiresAt) + } + + fun descriptor(d: ReadableMap): Descriptor { + val method = (d.string("method") ?: "POST").uppercase() + if (method !in METHODS) invalid("method $method is not supported") + + val parts = d.array("parts")?.let { parseParts(it) } + val url = d.string("url") + if (url == null && parts == null) invalid("url is required unless parts is set") + url?.let { requireHttpUrl(it, "url") } + + // An old JS layer would send `data`. Ignoring it would send no body. + if (d.isSet("data")) invalid("data crosses as dataJson") + val dataJson = if (d.isSet("dataJson")) { + val text = d.string("dataJson") ?: invalid("dataJson must be a string") + if (!JsonBridge.isJson(text)) invalid("dataJson is not JSON text") + text + } else null + val form = d.array("form")?.let { parseForm(it) } + val file = d.string("file")?.let { stripFileScheme(it) } + val kinds = listOfNotNull(dataJson?.let { "data" }, form?.let { "form" }, file?.let { "file" }) + if (kinds.size > 1) invalid("at most one of data, form, file; got ${kinds.joinToString()}") + if (parts != null && file == null) invalid("parts requires file") + if (method == "GET" && kinds.isNotEmpty()) invalid("a GET request can not carry a body") + + val headers = parseHeaderMap(d.map("headers")) + requireValidHeaders(headers, "headers") + + return Descriptor( + url = url, + method = method, + headers = headers, + dataJson = dataJson, + form = form, + file = file, + parts = parts, + accept = parseAcceptRules(d.array("accept")), + retry = d.map("retry")?.let { parseRetry(it) }, + noNotification = d.map("android")?.bool("noNotification") ?: false, + ) + } + + /** updateHeaders(patch): the header map, checked the same way as a descriptor's. */ + fun headerPatch(patch: ReadableMap): Map = + parseHeaderMap(patch).also { requireValidHeaders(it, "updateHeaders") } + + /** `file:///a%20b` → `/a b`. A plain path is returned as it is. */ + fun stripFileScheme(path: String): String { + if (!path.startsWith("file://")) return path + return runCatching { URI(path).path }.getOrNull() ?: path.removePrefix("file://") + } + + private fun parseParts(arr: ReadableArray): List { + if (arr.size() == 0) invalid("parts must be a non-empty array") + return (0 until arr.size()).map { i -> + val p = arr.getMap(i) ?: invalid("parts[$i] must be an object") + val url = p.string("url") ?: invalid("parts[$i].url is required") + requireHttpUrl(url, "parts[$i].url") + val range = p.map("range") ?: invalid("parts[$i].range is required") + val start = range.number("start")?.toLong() ?: invalid("parts[$i].range.start is required") + val end = range.number("end")?.toLong() ?: invalid("parts[$i].range.end is required") + if (start < 0 || end <= start) invalid("parts[$i].range must satisfy 0 <= start < end") + val headers = parseHeaderMap(p.map("headers")) + requireValidHeaders(headers, "parts[$i].headers") + Part(url = url, headers = headers, start = start, end = end) + } + } + + private fun parseForm(arr: ReadableArray): List { + if (arr.size() == 0) invalid("form must be a non-empty array") + return (0 until arr.size()).map { i -> + val p = arr.getMap(i) ?: invalid("form[$i] must be an object") + val name = p.string("name") ?: invalid("form[$i].name is required") + val contentType = p.string("contentType") ?: invalid("form[$i].contentType is required") + val string = p.string("string") + val path = p.string("path")?.let { stripFileScheme(it) } + if ((string == null) == (path == null)) invalid("form[$i] must set exactly one of string, path") + FormPart(name, contentType, string, path, p.string("fileName")) + } + } + + private fun parseRetry(r: ReadableMap): RetryOverride { + val backoff = r.map("backoff") + val exempt = r.map("terminalHttp")?.array("exempt")?.let { arr -> + (0 until arr.size()).mapNotNull { i -> + if (arr.getType(i) == ReadableType.Number) arr.getDouble(i).toInt() else null + } + } + return RetryOverride( + baseMs = backoff?.number("baseMs")?.toLong(), + maxMs = backoff?.number("maxMs")?.toLong(), + jitter = backoff?.number("jitter"), + exempt = exempt, + ) + } + + internal fun parseAcceptRules(arr: ReadableArray?): List { + if (arr == null) return listOf() + return (0 until arr.size()).mapNotNull { i -> + val rule = arr.getMap(i) ?: return@mapNotNull null + val status = rule.number("status") ?: return@mapNotNull null + UploadOutcome.AcceptRule(status.toInt(), rule.string("bodyIncludes")) + } + } + + /** Header values keep their text. A number is written as JSON would write it. */ + internal fun parseHeaderMap(map: ReadableMap?): Map { + if (map == null) return mapOf() + val out = LinkedHashMap() + JsonBridge.fromReadable(map).forEach { (k, v) -> + when (v) { + null -> Unit + is Double -> out[k] = JsonBridge.numberText(v) + else -> out[k] = v.toString() + } + } + return out + } + + private fun requireHttpUrl(url: String, where: String) { + if (url.toHttpUrlOrNull() == null) invalid("$where is not an http(s) url: $url") + } + + /** + * OkHttp throws on a header name or value it can not send. Check it here, + * so the error is an enqueue rejection and not a failure at attempt time. + * The message names the header and the offset only: a value can be a + * credential, and OkHttp's own message would print it. + */ + internal fun requireValidHeaders(headers: Map, where: String) { + headers.forEach { (name, value) -> + // OkHttp's rules: a name is 1+ chars in 0x21..0x7e; a value is tab or 0x20..0x7e. + if (name.isEmpty()) invalid("$where: a header name is empty") + name.indexOfFirst { it !in '\u0021'..'\u007e' }.takeIf { it >= 0 }?.let { i -> + // Only the valid part before the offset: the rest could be a value + // pasted into the name. + invalid("$where: the header name that starts '${name.take(i)}' has an invalid character at offset $i") + } + value.indexOfFirst { it != '\t' && it !in '\u0020'..'\u007e' }.takeIf { it >= 0 }?.let { i -> + invalid("$where: the value of header '$name' has an invalid character at offset $i") + } + // A backstop for any rule OkHttp adds later. Its message is not used. + try { + Headers.Builder().add(name, value) + } catch (e: IllegalArgumentException) { + invalid("$where: the HTTP client does not accept header '$name'") + } + } + } + + private fun invalid(message: String): Nothing = throw InvalidEntryException(message) + + private fun ReadableMap.isSet(key: String) = hasKey(key) && getType(key) != ReadableType.Null + + private fun ReadableMap.string(key: String): String? = + if (hasKey(key) && getType(key) == ReadableType.String) getString(key) else null + + private fun ReadableMap.number(key: String): Double? = + if (hasKey(key) && getType(key) == ReadableType.Number) getDouble(key) else null + + private fun ReadableMap.bool(key: String): Boolean? = + if (hasKey(key) && getType(key) == ReadableType.Boolean) getBoolean(key) else null + + private fun ReadableMap.map(key: String): ReadableMap? = + if (hasKey(key) && getType(key) == ReadableType.Map) getMap(key) else null + + private fun ReadableMap.array(key: String): ReadableArray? = + if (hasKey(key) && getType(key) == ReadableType.Array) getArray(key) else null +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/EntryRun.kt b/android/src/main/java/ai/openspace/backgroundupload/EntryRun.kt new file mode 100644 index 00000000..d9b5ffbe --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/EntryRun.kt @@ -0,0 +1,325 @@ +package ai.openspace.backgroundupload + +import kotlinx.coroutines.CancellationException +import okhttp3.RequestBody +import java.io.IOException +import java.util.UUID +import kotlin.math.max +import kotlin.math.min + +/** + * The run of one queue entry, and the one attempt step both transfers use. + * No Android types: [EntryWorker] is the shell that holds the per-id gate + * and is the real [TransferHost]. + * + * The run: take the entry (queued → running), run the transfer, then + * settle, park, or release. The body kind at run time picks the transfer, + * [SimpleTransfer] or [ChunkedTransfer], so a kind change under a queued run + * is safe. The entry is read from the store at start and again before every + * attempt, so fresh headers and a new expiresAt reach a running run. + */ +internal class EntryRun( + val entryId: String, + val store: QueueStore, + val ops: WorkerOps, + val host: TransferHost, +) { + /** A 401/403 under the headers of [headerGeneration] (the entry's value the attempt sent). */ + class ParkException(val headerGeneration: Int) : Exception("awaiting auth") + + /** A backoff too long to wait here. */ + class BackoffException(val nextAttemptAt: Long, val streak: Int) : Exception("released for backoff") + + class ExpiredException : Exception("expired before completion") + + /** The queue was paused between the module's pause and the work cancel reaching us. */ + class PausedException : Exception("queue paused") + + /** How one attempt ended, after the retry table. */ + sealed class AttemptResult { + data class Accepted(val response: UploadResponse) : AttemptResult() + + /** A 401/403. [reissue]: newer headers arrived while it was in flight, so send again now. */ + data class Auth(val headerGeneration: Int, val reissue: Boolean) : AttemptResult() + + object Transient : AttemptResult() + + data class Terminal(val errorKind: String, val message: String, val response: UploadResponse?) : AttemptResult() + } + + /** One attempt: the entry it ran under, where it went, the retry policy, and how it ended. */ + class Attempt( + val entry: QueueEntry, + val url: String, + val policy: RetryClassifier.Policy, + val result: AttemptResult, + ) { + val method: String get() = entry.descriptor!!.method + } + + companion object { + /** The poll while the network is unusable (offline, or waiting for wifi). */ + const val CONNECTIVITY_POLL_MS = 10_000L + + /** The poll while a short backoff remainder runs out before the run starts. */ + private const val WAIT_POLL_MS = 10_000L + + /** The descriptor's headers, the part's over them, and X-Request-Id over all. */ + fun headersFor(d: Descriptor, part: Part?, requestId: String): Map { + val withPart = if (part == null) d.headers else HeaderMap.merge(d.headers, part.headers) + return HeaderMap.merge(withPart, mapOf("X-Request-Id" to requestId)) + } + } + + var generation = 0 + private set + + /** The bytes of the current attempt, as last reported. A failed simple entry settles with them. */ + @Volatile + var liveBytes = 0L + private set + + /** + * Returns when the run is over. Throws a CancellationException after it + * handled a stop, or an IOException after a store write failed (the + * caller returns retry; nothing settled). + */ + suspend fun run() { + val initial = store.load(entryId) ?: return // forgotten while queued + if (initial.legacy) return + if (initial.state == EntryState.AWAITING_AUTH) { + // The expiry wake of a parked entry. No attempt ran here, so the + // stored bytes stand. + if (RetryClassifier.isExpired(host.now(), initial.expiresAt)) { + ops.settle(entryId, initial.generation, expired(initial).copy(bytesSent = null)) + } + return + } + if (initial.state != EntryState.QUEUED && initial.state != EntryState.RUNNING) return + if (ops.settings().paused) return + if (!waitUntilDue(initial)) return + val entry = ops.begin(entryId) ?: return + generation = entry.generation + + var current = entry + var first = true + while (true) { + try { + if (!first) current = ops.latest(entryId, generation) + first = false + host.foreground(current) + val settlement = transfer(current) + endProgress(completed = settlement is Settlement.Completed) + ops.settle(entryId, generation, settlement) + return + } catch (park: ParkException) { + endProgress(completed = false) + // REISSUE: updateHeaders() landed while this attempt was in flight. + if (ops.park(entryId, generation, park.headerGeneration) != WorkerOps.ParkResult.REISSUE) return + } catch (backoff: BackoffException) { + endProgress(completed = false) + ops.release(entryId, generation, backoff.nextAttemptAt, backoff.streak) + return + } catch (error: ExpiredException) { + endProgress(completed = false) + ops.settle(entryId, generation, expired(current)) + return + } catch (error: NotOwnedException) { + endProgress(completed = false) + return + } catch (error: PausedException) { + endProgress(completed = false) + return + } catch (error: CancellationException) { + // A system stop moves a running entry back to queued. A pause or a + // cancel already moved it; then this does nothing. + endProgress(completed = false) + if (host.stoppedByTimeout()) releaseAfterTimeout() else ops.stopped(entryId, generation) + throw error + } catch (error: IOException) { + // A store write failed (disk full, directory briefly unwritable). + // The attempt step classifies every network IOException itself, so + // one that lands here is storage: transient, no outcome. Back to + // queued; the caller returns retry. + endProgress(completed = false) + ops.stopped(entryId, generation) + throw error + } catch (error: Throwable) { + endProgress(completed = false) + val d = current.descriptor + ops.settle( + entryId, + generation, + Settlement.Failed( + errorKind = "unknown", + message = error.message ?: error.javaClass.simpleName, + response = null, + partIndex = null, + url = d?.reportUrl ?: "", + method = d?.method ?: "POST", + bytesSent = liveBytesOf(current), + ), + ) + return + } + } + } + + private suspend fun transfer(entry: QueueEntry): Settlement = + if (entry.body?.kind == StagedBody.CHUNKED) ChunkedTransfer(this).run(entry) + else SimpleTransfer(this).run(entry) + + private fun liveBytesOf(entry: QueueEntry): Long? = + if (entry.body?.kind == StagedBody.CHUNKED) null else liveBytes + + private fun expired(entry: QueueEntry) = Settlement.Failed( + errorKind = "expired", + message = "expired before completion", + response = null, + partIndex = null, + url = entry.descriptor?.reportUrl ?: "", + method = entry.descriptor?.method ?: "POST", + bytesSent = liveBytesOf(entry), + ) + + /** + * The system stopped the run at its time limit. That happens to a + * headless run whose foreground start was denied (API 31+), after about + * 10 minutes. Starting again at once would send the body from byte 0 on + * every run, so this is one more transient failure: the next backoff + * step, then a wake. + */ + private fun releaseAfterTimeout() { + runCatching { + val e = store.load(entryId) + if (!EntryTransitions.isOwnedRun(e, generation)) return + val streak = e!!.backoffStreak + 1 + val now = host.now() + val backoff = RetryClassifier.backoffMs(policy(e), streak) + ops.release(entryId, generation, RetryClassifier.nextAttemptAt(now, backoff, e.expiresAt), streak) + }.onFailure { Diag.error("could not release '$entryId' after a timeout stop", it) } + } + + /** + * Sleeps out a short backoff remainder. False when the wait is long (the + * wake run comes back for it) or the entry is no longer queued. + */ + private suspend fun waitUntilDue(initial: QueueEntry): Boolean { + var e = initial + while (true) { + val at = e.nextAttemptAt ?: return true + val remaining = at - host.now() + if (remaining <= 0) return true + if (remaining > RetryClassifier.IN_WORKER_BACKOFF_MAX_MS) return false + host.sleep(min(remaining, WAIT_POLL_MS)) + e = store.load(entryId) ?: return false + if (e.state != EntryState.QUEUED && e.state != EntryState.RUNNING) return false + } + } + + // MARK: - helpers for the transfers + + /** + * Waits until the network fits the queue's wifi-only setting. Re-reads + * the settings and the entry at every poll. + */ + suspend fun waitForNetwork() { + while (true) { + val s = ops.settings() + if (s.paused) throw PausedException() + val entry = ops.latest(entryId, generation) + if (RetryClassifier.isExpired(host.now(), entry.expiresAt)) throw ExpiredException() + if (host.connectivity(s.wifiOnly) == Connectivity.Ok) return + host.sleep(CONNECTIVITY_POLL_MS) + } + } + + fun policy(entry: QueueEntry) = + RetryClassifier.policy(ops.settings().retry, entry.descriptor?.retry) + + /** + * One HTTP attempt, the step both transfers share: write the attempt + * ahead (attempts + 1, its X-Request-Id), send, emit the attempt event, + * and classify. [partIndex] is null for a simple entry. [body] builds the + * request body from the entry the attempt runs under. [fileExists] tells a + * missing payload from a network failure. + */ + suspend fun attempt( + partIndex: Int?, + body: (Descriptor, Part?) -> RequestBody?, + onProgress: (Long) -> Unit, + fileExists: () -> Boolean, + ): Attempt { + val requestId = UUID.randomUUID().toString() + val entry = ops.recordAttempt(entryId, generation, requestId) + // The generation of the headers this attempt sends: both come from the same entry. + val headerGeneration = entry.headerGeneration + val d = entry.descriptor!! + val part = partIndex?.let { d.parts!![it] } + val url = part?.url ?: d.url!! + val policy = policy(entry) + + val response = try { + host.send(TransferRequest(url, d.method, headersFor(d, part, requestId), body(d, part)), onProgress) + } catch (error: CancellationException) { + throw error + } catch (error: Throwable) { + // A failed probe reads as present, so a flaky check is a retryable + // network error and not a terminal "file gone". + val verdict = RetryClassifier.classifyFailure(error, runCatching(fileExists).getOrDefault(true)) + host.attempt( + AttemptEvent.ofFailure( + entry, requestId, url, partIndex, RetryClassifier.failureKind(verdict), + error.message ?: error.javaClass.simpleName, host.now(), + ), + ) + val result = if (verdict is RetryClassifier.Verdict.Terminal) { + AttemptResult.Terminal(verdict.errorKind, verdict.message, null) + } else AttemptResult.Transient + return Attempt(entry, url, policy, result) + } + + val verdict = RetryClassifier.classifyResponse(response.code, response.body, d.accept, policy.exempt) + host.attempt( + AttemptEvent.ofResponse( + entry, requestId, url, partIndex, response, verdict == RetryClassifier.Verdict.Accepted, host.now(), + ), + ) + val result = when (verdict) { + RetryClassifier.Verdict.Accepted -> AttemptResult.Accepted(response) + RetryClassifier.Verdict.Auth -> + AttemptResult.Auth(headerGeneration, ops.hasNewerHeaders(entryId, generation, headerGeneration)) + RetryClassifier.Verdict.Transient -> AttemptResult.Transient + is RetryClassifier.Verdict.Terminal -> AttemptResult.Terminal("http", verdict.message, response) + } + return Attempt(entry, url, policy, result) + } + + /** + * A short backoff waits here, with the row still running and showing + * nextAttemptAt; a long one throws [BackoffException] to release the run. + */ + suspend fun backoffOrRelease(policy: RetryClassifier.Policy, streak: Int, expiresAt: Long) { + val backoff = RetryClassifier.backoffMs(policy, streak) + val now = host.now() + if (backoff <= RetryClassifier.IN_WORKER_BACKOFF_MAX_MS) { + val wait = min(backoff, max(0L, expiresAt - now)) + ops.backingOff(entryId, generation, now + wait) + host.sleep(wait) + return + } + throw BackoffException(RetryClassifier.nextAttemptAt(now, backoff, expiresAt), streak) + } + + fun progressStarted(total: Long, sent: Long) { + liveBytes = sent + host.progressStarted(entryId, total, sent) + } + + fun reportProgress(sent: Long, total: Long) { + liveBytes = sent + host.progress(entryId, sent, total) + } + + private fun endProgress(completed: Boolean) = host.progressEnded(entryId, completed) +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/EntryTransitions.kt b/android/src/main/java/ai/openspace/backgroundupload/EntryTransitions.kt new file mode 100644 index 00000000..d099d8df --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/EntryTransitions.kt @@ -0,0 +1,114 @@ +package ai.openspace.backgroundupload + +/** + * The state changes of one entry, as pure functions. [QueueController] and + * [WorkerOps] apply them inside `QueueStore.compute`, so each one is atomic + * against the other side. + */ +object EntryTransitions { + + /** A worker takes a queued entry. */ + fun toRunning(e: QueueEntry, now: Long) = + e.copy(state = EntryState.RUNNING, nextAttemptAt = null, updatedAt = now) + + /** A 401/403 under the current header generation. No backoff. */ + fun toParked(e: QueueEntry, headerGeneration: Int, now: Long) = e.copy( + state = EntryState.AWAITING_AUTH, + parkedGeneration = headerGeneration, + backoffStreak = 0, + updatedAt = now, + ) + + /** + * A short backoff that the worker waits out itself. The row stays running + * and shows when the next attempt is due. + */ + fun toBackingOff(e: QueueEntry, nextAttemptAt: Long, now: Long) = + e.copy(nextAttemptAt = nextAttemptAt, updatedAt = now) + + /** One attempt starts: attempts + 1, its X-Request-Id, and no pending backoff. */ + fun toAttempt(e: QueueEntry, requestId: String, now: Long) = e.copy( + attempts = e.attempts + 1, + lastRequestId = requestId, + nextAttemptAt = null, + updatedAt = now, + ) + + /** A backoff too long to wait inside the worker. The streak is kept so the next wait keeps growing. */ + fun toReleased(e: QueueEntry, nextAttemptAt: Long, streak: Int, now: Long) = e.copy( + state = EntryState.QUEUED, + nextAttemptAt = nextAttemptAt, + backoffStreak = streak, + updatedAt = now, + ) + + fun toSettled(e: QueueEntry, state: EntryState, eventId: String, bytesSent: Long, now: Long) = e.copy( + state = state, + settledEventId = eventId, + bytesSent = bytesSent, + nextAttemptAt = null, + parkedGeneration = null, + updatedAt = now, + ) + + /** + * pause(). parkedGeneration is kept so resume() can return the entry to + * awaiting-auth. nextAttemptAt is cleared: resume() retries at once. + */ + fun toPaused(e: QueueEntry, now: Long) = + e.copy(state = EntryState.PAUSED, nextAttemptAt = null, updatedAt = now) + + /** resume(): back to awaiting-auth only when no updateHeaders() came in between. */ + fun toResumed(e: QueueEntry, headerGeneration: Int, now: Long): QueueEntry = + if (e.parkedGeneration != null && e.parkedGeneration == headerGeneration) { + e.copy(state = EntryState.AWAITING_AUTH, updatedAt = now) + } else { + e.copy(state = EntryState.QUEUED, parkedGeneration = null, updatedAt = now) + } + + /** A system stop of a running worker. WorkManager runs the row again. No outcome. */ + fun toStopped(e: QueueEntry, now: Long) = e.copy(state = EntryState.QUEUED, updatedAt = now) + + /** updateHeaders() on a parked entry. */ + fun toUnparked(e: QueueEntry, paused: Boolean, now: Long) = e.copy( + state = if (paused) EntryState.PAUSED else EntryState.QUEUED, + parkedGeneration = null, + updatedAt = now, + ) + + /** + * Whether a worker of [generation] may still settle [e]. Not after a + * cancel, a replace, or a settle. Under pause only an [accepted] response + * settles: the server already took it. A failure waits for resume, which + * runs the entry again. + */ + fun canSettle(e: QueueEntry?, generation: Int, accepted: Boolean) = + e != null && e.generation == generation && e.isLive && (accepted || e.state != EntryState.PAUSED) + + /** The entry state a journal record puts its entry in. */ + fun stateOf(record: EventJournal.SettledRecord): EntryState = + EntryState.values().firstOrNull { it.wire == record.state } ?: EntryState.ERROR + + /** A settled entry, and the other records of its life to ack. */ + data class Journaled(val entry: QueueEntry, val extraEventIds: List) + + /** + * A live entry with a journal record of its own generation: the settle + * journaled, then its store write was lost (a process death, a failed + * save). Apply the newest record; do not run the request again. Null when + * there is no such record. The boot sweep and a worker's begin share it. + */ + fun journaledSettle(e: QueueEntry, records: List, now: Long): Journaled? { + if (!e.isLive || e.legacy) return null + val own = records.filter { it.id == e.id && it.generation == e.generation } + val latest = own.maxByOrNull { it.at } ?: return null + return Journaled( + toSettled(e, stateOf(latest), latest.eventId, latest.bytesSent, now), + own.filter { it !== latest }.map { it.eventId }, + ) + } + + /** Whether a worker of [generation] still owns the running entry. */ + fun isOwnedRun(e: QueueEntry?, generation: Int) = + e != null && e.generation == generation && e.state == EntryState.RUNNING +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/EntryWorker.kt b/android/src/main/java/ai/openspace/backgroundupload/EntryWorker.kt new file mode 100644 index 00000000..d967862b --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/EntryWorker.kt @@ -0,0 +1,147 @@ +package ai.openspace.backgroundupload + +import android.app.NotificationManager +import android.content.Context +import androidx.work.CoroutineWorker +import androidx.work.ForegroundInfo +import androidx.work.WorkInfo +import androidx.work.WorkerParameters +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.delay +import kotlinx.coroutines.sync.withPermit +import kotlinx.coroutines.withContext + +/** + * The WorkManager shell of one entry's run. The input data holds only the + * entry id. The worker acquires the per-id gate, then [EntryRun] does the + * run; this class is its [TransferHost]: OkHttp, the clock, progress, and + * the notification. + * + * [UploadWorker] and [ChunkedUploadWorker] are the two class names + * WorkManager knows; both run this same code. Every run returns success + * (see [WorkManagerScheduler] for why), except a store failure before the + * run could settle, which returns retry. A v9 row, which has no entry id, + * exits at once in silence. + */ +open class EntryWorker(protected val context: Context, params: WorkerParameters) : + CoroutineWorker(context, params) { + + companion object { + private const val GATE_POLL_MS = 100L + } + + private val store by lazy { QueueStore.get(context) } + private val ops by lazy { + WorkerOps( + store, + EventJournal.get(context), + QueueSettingsStore.get(context), + EventReporter, + WorkManagerScheduler(context), + ) + } + private val config by lazy { NotificationConfig.load(context) } + private val notificationManager = + context.getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager + + @Volatile + private var connectivity = Connectivity.Ok + + @Volatile + private var showsNotification = false + + final override suspend fun doWork(): Result = withContext(Dispatchers.IO) { + val id = inputData.getString(WorkManagerScheduler.ENTRY_ID_KEY) ?: return@withContext Result.success() + // Acquire before the first store read: a cancel-then-enqueue can start + // this run while the old one still winds down. + while (!WorkerGate.tryAcquire(id, this@EntryWorker)) delay(GATE_POLL_MS) + try { + EntryRun(id, store, ops, host).run() + Result.success() + } catch (error: CancellationException) { + throw error + } catch (error: Throwable) { + // A store failure (disk full at start or during an attempt). Nothing + // was settled; try the run again later. + Diag.error("run of '$id' failed before it could settle; retrying", error) + Result.retry() + } finally { + WorkerGate.release(id, this@EntryWorker) + } + } + + private val host = object : TransferHost { + override fun now() = System.currentTimeMillis() + + override suspend fun sleep(ms: Long) = delay(ms) + + override suspend fun send(request: TransferRequest, onProgress: (Long) -> Unit): UploadResponse = + transferSemaphore.withPermit { okhttpSend(uploadHttpClient, request, onProgress) } + + override fun connectivity(wifiOnly: Boolean): Connectivity { + connectivity = validateConnectivity(context, wifiOnly) + updateNotification() + return connectivity + } + + override suspend fun foreground(entry: QueueEntry) { + showsNotification = entry.descriptor?.noNotification == false + startForeground() + } + + override fun progressStarted(id: String, total: Long, sent: Long) { + UploadProgress.add(id, total) + UploadProgress.set(id, sent) + } + + override fun progress(id: String, sent: Long, total: Long) { + UploadProgress.set(id, sent) + EventReporter.progress(id, sent, total) + updateNotification() + } + + override fun progressEnded(id: String, completed: Boolean) { + if (completed) UploadProgress.complete(id) else UploadProgress.remove(id) + EventReporter.flushProgress(id) + EventReporter.dropProgress(id) + } + + override fun attempt(event: AttemptEvent) = EventReporter.attempt(event) + + override fun stoppedByTimeout() = stopReason == WorkInfo.STOP_REASON_TIMEOUT + } + + // MARK: - notification + + // v9 rules. A suppressed notification means no foreground mode. A denied + // foreground start (API 31+, app in the background: the usual case for a + // WorkManager relaunch) is not a failure; the transfer runs without + // foreground priority, under JobScheduler's time limit (see + // [EntryRun.releaseAfterTimeout]). Any other failure is logged and the run + // goes on. + private suspend fun startForeground() { + if (!showsNotification) return + try { + ensureNotificationChannel(notificationManager, config) + setForeground(getForegroundInfo()) + } catch (error: CancellationException) { + throw error + } catch (error: Throwable) { + if (!isForegroundStartDenied(error)) Diag.warn("foreground start failed; running without it", error) + } + } + + private fun updateNotification() { + if (!showsNotification) return + runCatching { + notificationManager.notify( + config.systemNotificationId, + buildUploadNotification(context, config, connectivity), + ) + } + } + + override suspend fun getForegroundInfo(): ForegroundInfo = + uploadForegroundInfo(config, buildUploadNotification(context, config, connectivity)) +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/EventJournal.kt b/android/src/main/java/ai/openspace/backgroundupload/EventJournal.kt index 2dd718b3..3f560947 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/EventJournal.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/EventJournal.kt @@ -1,82 +1,153 @@ package ai.openspace.backgroundupload import android.content.Context +import com.facebook.react.bridge.WritableMap import com.google.gson.Gson import java.io.File +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit -// Durable record of terminal upload events (completed / error / cancelled). -// Written BEFORE the event is emitted to JS; deleted only when JS acknowledges. -// One JSON file per event named .json — tmp+rename keeps each write -// self-contained so a crash mid-append can never corrupt other entries. -// -// `maxEntries` is a runaway guard: the design assumes JS drains the journal via -// ack() on every boot, but if that loop breaks (or a consumer hasn't adopted it -// yet) the directory would grow without bound. When exceeded we drop the OLDEST -// entries. Set high enough that legitimate heavy offline use won't hit it — this -// only fires in the pathological "nothing ever acks" case. +/** + * The durable record of settled outcomes (completed, error, cancelled). A + * record is written BEFORE the outcome is emitted to JS and deleted only when + * JS acknowledges it. One JSON file per record, `.json`, written + * with tmp + fsync + rename, so a crash mid-write can not corrupt another + * record. + * + * [maxEntries] is a runaway guard: if nothing ever acknowledges, the oldest + * records are dropped, except those a row still names. It only fires in + * that broken case. + */ class EventJournal( private val dir: File, private val maxEntries: Int = MAX_ENTRIES, + /** Runs a task after a delay. Tests pass a manual one. */ + private val retryLater: (delayMs: Long, task: () -> Unit) -> Unit = ::onTimer, ) { - data class Entry( + /** RawResponse. [status] is null for a chunked completion. */ + data class Response( + val status: Int?, + val headers: Map?, + val body: String?, + val bodyTruncated: Boolean, + ) { + fun toMap(): Map = LinkedHashMap().apply { + status?.let { put("status", it.toDouble()) } + headers?.let { put("headers", it) } + body?.let { put("body", it) } + put("bodyTruncated", bodyTruncated) + } + + companion object { + fun of(response: UploadResponse): Response { + val (body, cut) = BodyCap.cap(response.body, BodyCap.SETTLED_MAX_BYTES) + return Response(response.code, response.headers, body, response.truncated || cut) + } + + /** A chunked completion: N parts, no one response. */ + val NONE = Response(null, null, null, false) + } + } + + /** One settled outcome, in the SettledEvent shape plus [generation]. */ + data class SettledRecord( val eventId: String, - val uploadId: String, - val type: String, // completed | error | cancelled - val timestamp: Long, - val responseCode: Int? = null, - val responseBody: String? = null, - val responseBodyTruncated: Boolean = false, - val responseHeaders: Map? = null, - val error: String? = null, - val errorKind: String? = null, // http | network | file | expired | unknown - val cancelReason: String? = null, // user | system - // Chunked uploads: the index of the failing part, when one part's response - // caused the error. - val partIndex: Int? = null, + val id: String, + val key: String, + val varsJson: String, + val at: Long, + val attempts: Int, + val requestId: String?, + /** + * How many times this outcome reached a JS listener: 1 after a live + * emit, 0 when it was journaled with no listener; +1 per later delivery + * (replay, re-emit). [append] sets it. + */ + val deliveries: Int, + /** The entry state this outcome puts it in. */ + val state: String, + val bytesSent: Long, + val totalBytes: Long, + val url: String, + val method: String, + val partIndex: Int?, + /** completed | error | cancelled */ + val kind: String, + /** completed; also error with errorKind http. */ + val response: Response?, + val errorKind: String?, + val message: String?, + val cancelReason: String?, + /** The entry life this belongs to (same-id rule 6). */ + val generation: Int, ) { - fun toWritableMap(): com.facebook.react.bridge.WritableMap = - com.facebook.react.bridge.Arguments.createMap().apply { - putString("eventId", eventId) - putString("id", uploadId) - putString("type", type) - putDouble("timestamp", timestamp.toDouble()) - responseCode?.let { putInt("responseCode", it) } - responseBody?.let { putString("responseBody", it) } - if (responseBodyTruncated) putBoolean("responseBodyTruncated", true) - responseHeaders?.let { - putMap("responseHeaders", com.facebook.react.bridge.Arguments.makeNativeMap(it)) - } - error?.let { putString("error", it) } - errorKind?.let { putString("errorKind", it) } - cancelReason?.let { putString("cancelReason", it) } - partIndex?.let { putInt("partIndex", it) } + fun toMap(): Map = LinkedHashMap().apply { + put("eventId", eventId) + put("id", id) + put("key", key) + put("vars", runCatching { JsonBridge.parse(varsJson) }.getOrNull()) + put("at", at.toDouble()) + put("attempts", attempts.toDouble()) + requestId?.let { put("requestId", it) } + put("deliveries", deliveries.toDouble()) + put("state", state) + put("bytesSent", bytesSent.toDouble()) + put("totalBytes", totalBytes.toDouble()) + put("url", url) + put("method", method) + partIndex?.let { put("partIndex", it.toDouble()) } + put("kind", kind) + when (kind) { + KIND_COMPLETED -> put("response", (response ?: Response.NONE).toMap()) + KIND_ERROR -> put("error", LinkedHashMap().apply { + put("errorKind", errorKind ?: "unknown") + put("message", message ?: "") + response?.let { put("response", it.toMap()) } + partIndex?.let { put("partIndex", it.toDouble()) } + }) + KIND_CANCELLED -> put("cancelReason", cancelReason ?: "user") } + } + + fun toWritableMap(): WritableMap = JsonBridge.toWritableMap(toMap()) } companion object { - const val MAX_BODY_CHARS = 64 * 1024 + const val KIND_COMPLETED = "completed" + const val KIND_ERROR = "error" + const val KIND_CANCELLED = "cancelled" + const val MAX_ENTRIES = 1000 + + /** The first wait before a held record is written again. It doubles up to [RETRY_MAX_MS]. */ + const val RETRY_MS = 5_000L + const val RETRY_MAX_MS = 600_000L private val gson = Gson() - // Char-count cap (not byte-accurate: splitting on a byte boundary risks - // cutting a surrogate pair; a slightly loose cap is fine as a safety limit). - // Returns the (possibly truncated) body and whether truncation occurred. - // Single source of truth so the journaled copy and the live-emitted copy match. - fun capBody(body: String?): Pair = - if (body != null && body.length > MAX_BODY_CHARS) - body.substring(0, MAX_BODY_CHARS) to true - else body to false + // Event ids are UUIDs that native mints. ackEvents takes ids from JS, and + // an id is a file name here, so anything else is ignored. + private val EVENT_ID = Regex("^[A-Za-z0-9-]{1,64}$") + + fun isValidEventId(id: String) = EVENT_ID.matches(id) + + private val timer by lazy { + Executors.newSingleThreadScheduledExecutor { r -> + Thread(r, "RNFileUploader.journal").apply { isDaemon = true } + } + } + + private fun onTimer(delayMs: Long, task: () -> Unit) { + timer.schedule(task, delayMs, TimeUnit.MILLISECONDS) + } @Volatile private var instance: EventJournal? = null - // The worker may run in a process where React never initialized, so the - // journal must be reachable from a bare Context, not the module. + /** v10 records live in `rnbgupload-settled`. The v9 `rnbgupload-events` is read once by [LegacyImport]. */ fun get(context: Context): EventJournal = instance ?: synchronized(this) { - instance - ?: EventJournal(File(context.filesDir, "rnbgupload-events")).also { instance = it } + instance ?: EventJournal(File(context.filesDir, "rnbgupload-settled")).also { instance = it } } } @@ -84,61 +155,206 @@ class EventJournal( dir.mkdirs() } + /** + * The JS listener, set by [drain] and cleared by [stopListening]. While it + * is set, a new record starts at 1 delivery and the caller emits it live. + * While it is null, a record starts at 0 and the next drain delivers it. + * Both decisions take this object's lock, so a record is either in the + * drain or emitted live, never both and never neither. + */ + private var listener: Any? = null + + /** + * Records whose file write failed, by eventId. They count as journaled in + * every read and ack, and a timer writes them once the disk allows. They + * live only in memory: a process death loses them. + */ + private val held = LinkedHashMap() + private var retryScheduled = false + + private fun fileFor(eventId: String) = File(dir, "$eventId.json") + + /** + * Writes [record] with deliveries 1 when a listener is set, else 0, and + * returns it as written. The caller emits it only when deliveries > 0. + * Throws IOException when the write failed; nothing is kept then. + * + * [keep] names the records the prune must not delete (every eventId a row + * names). It runs only when the journal is over its cap, under this lock. + * Callers hold the store lock (lock order: store, then journal). + */ + @Synchronized + fun append(record: SettledRecord, keep: () -> Set = { emptySet() }): SettledRecord { + val stamped = stamp(record) + AtomicFiles.writeText(fileFor(stamped.eventId), gson.toJson(stamped)) + pruneToMax(keep) + return stamped + } + + /** + * As [append], but never throws. A failed write holds the record in + * memory and a timer tries it again. For a worker's settle: the request + * already ran, and a thrown error would send it again. + */ @Synchronized - fun append(entry: Entry) { - // Defensive cap in case a caller didn't pre-cap; idempotent when it did. - val (body, truncated) = capBody(entry.responseBody) - val bounded = - if (truncated) entry.copy(responseBody = body, responseBodyTruncated = true) else entry - // A journal write must NEVER throw into the caller. The worker calls this - // right after a successful upload; a propagated IOException (e.g. disk full) - // would be classified as a retryable error and re-run the upload, sending - // duplicate data to the server. Losing one journal entry is the lesser evil. + fun appendOrHold(record: SettledRecord, keep: () -> Set = { emptySet() }): SettledRecord = try { - val tmp = File(dir, "${entry.eventId}.tmp") - tmp.writeText(gson.toJson(bounded)) - tmp.renameTo(File(dir, "${entry.eventId}.json")) + append(record, keep) } catch (t: Throwable) { - t.printStackTrace() - return + Diag.error("journal append failed for ${record.eventId}; held in memory", t) + val stamped = stamp(record) + held[stamped.eventId] = stamped + scheduleRetry(RETRY_MS) + stamped + } + + private fun stamp(record: SettledRecord): SettledRecord { + val response = record.response?.let { r -> + val (body, cut) = BodyCap.cap(r.body, BodyCap.SETTLED_MAX_BYTES) + if (cut) r.copy(body = body, bodyTruncated = true) else r + } + return record.copy(deliveries = if (listener != null) 1 else 0, response = response) + } + + /** Whether [eventId] is held in memory, not yet on disk. */ + @Synchronized + fun isHeld(eventId: String) = eventId in held + + /** Writes every held record. Returns true when none is left. */ + @Synchronized + fun writeHeld(): Boolean { + val written = held.values.filter { r -> + runCatching { AtomicFiles.writeText(fileFor(r.eventId), gson.toJson(r)) }.isSuccess + } + written.forEach { held.remove(it.eventId) } + return held.isEmpty() + } + + private fun scheduleRetry(delayMs: Long) { + if (retryScheduled) return + retryScheduled = true + retryLater(delayMs) { + synchronized(this) { + retryScheduled = false + if (!writeHeld()) scheduleRetry(minOf(delayMs * 2, RETRY_MAX_MS)) + } } - pruneToMax() } - // Keep the directory bounded. Prune by file modification time (no parsing) - // rather than the entry's own timestamp — cheaper, and close enough since a - // file's mtime is when it was journaled. Guarded: a prune failure must not - // propagate for the same reason append() must not. - private fun pruneToMax() { + // Prunes by file time (no parsing), sparing the records rows name. Never + // throws: it only runs in the broken case where nothing acknowledges. + private fun pruneToMax(keep: () -> Set) { try { - // Sweep orphaned .tmp files (writeText succeeded but rename failed). - dir.listFiles { f -> f.extension == "tmp" }?.forEach { it.delete() } + dir.listFiles { f -> f.name.endsWith(AtomicFiles.TMP_SUFFIX) }?.forEach { it.delete() } val files = dir.listFiles { f -> f.extension == "json" } ?: return if (files.size <= maxEntries) return - files.sortedBy { it.lastModified() } + val named = keep() + files.filter { it.nameWithoutExtension !in named } + .sortedBy { it.lastModified() } .take(files.size - maxEntries) .forEach { it.delete() } } catch (t: Throwable) { - t.printStackTrace() + Diag.error("journal prune failed", t) } } + /** + * getUnacknowledgedEvents(): sets [owner] as the listener and returns + * every record, oldest first, each counted as one more delivery. One lock + * spans both, see [listener]. + */ @Synchronized - @Suppress("SENSELESS_COMPARISON") // Gson can inject null into a non-null field - fun unacknowledged(): List = - (dir.listFiles { f -> f.extension == "json" } ?: emptyArray()) - .mapNotNull { f -> - runCatching { gson.fromJson(f.readText(), Entry::class.java) }.getOrNull() - } - // Gson bypasses the constructor, so a file missing a field yields null - // despite the non-null Kotlin type. Check every field JS relies on being - // present, not just eventId — an entry reaching JS with a null `type` - // would fall silently through a `switch (event.type)`. - .filter { it.eventId != null && it.uploadId != null && it.type != null } - .sortedBy { it.timestamp } + fun drain(owner: Any, isActive: () -> Boolean = { true }): List { + // [isActive] is read under this lock. A module torn down before its + // queued drain runs does not become the listener, and counts nothing. + if (!isActive()) return emptyList() + listener = owner + return unacknowledged().mapNotNull { incrementDeliveries(it.eventId) } + } + /** Clears the listener, only when [owner] set it: a reload builds the next module before it tears down this one. */ + @Synchronized + fun stopListening(owner: Any) { + if (listener === owner) listener = null + } + + @Synchronized + fun isListening() = listener != null + + /** The listener a live settled event goes to: the module whose JS drained last. */ + @Synchronized + fun listener(): Any? = listener + + /** + * A re-emit of a journaled outcome (same-id rule 7): one more delivery, + * returned for a live emit. Null when no listener is set (the next drain + * delivers it) or the record is gone. + */ + @Synchronized + fun redeliver(eventId: String): SettledRecord? = + if (listener == null) null else incrementDeliveries(eventId) + + /** Every record, oldest first, the held ones included. Corrupt files are skipped. */ + @Synchronized + fun unacknowledged(): List = + ((dir.listFiles { f -> f.extension == "json" } ?: emptyArray()).mapNotNull { read(it) } + held.values) + .sortedWith(compareBy { it.at }.thenBy { it.eventId }) + + @Synchronized + fun find(eventId: String): SettledRecord? = + if (isValidEventId(eventId)) held[eventId] ?: read(fileFor(eventId)) else null + + @Synchronized + fun forEntry(id: String): List = unacknowledged().filter { it.id == id } + + /** + * One more delivery of [eventId]: rewrites the record with deliveries + 1 + * and returns it. Null when the record is gone. When the rewrite fails the + * incremented record is still returned, so the delivery goes ahead. + */ + @Synchronized + fun incrementDeliveries(eventId: String): SettledRecord? { + val record = find(eventId) ?: return null + val next = record.copy(deliveries = record.deliveries + 1) + if (eventId in held) { + held[eventId] = next + return next + } + try { + AtomicFiles.writeText(fileFor(eventId), gson.toJson(next)) + } catch (t: Throwable) { + Diag.error("journal deliveries update failed for $eventId", t) + } + return next + } + + /** Idempotent. Unknown and malformed ids are ignored. */ @Synchronized fun ack(eventIds: List) { - eventIds.forEach { File(dir, "$it.json").delete() } + eventIds.filter { isValidEventId(it) }.forEach { + held.remove(it) + fileFor(it).delete() + } + } + + /** Acks every record of entry [id]: cancel() of a settled entry forgets its outcomes. */ + @Synchronized + fun ackEntry(id: String) { + ack(forEntry(id).map { it.eventId }) + } + + @Suppress("SENSELESS_COMPARISON", "USELESS_ELVIS") + private fun read(file: File): SettledRecord? { + if (!file.exists()) return null + val r = runCatching { gson.fromJson(file.readText(), SettledRecord::class.java) }.getOrNull() + ?: return null + // Gson does not run constructors. Check every field JS relies on. + if (r.eventId == null || r.id == null || r.key == null || r.kind == null || r.state == null) return null + return r.copy( + varsJson = r.varsJson ?: "null", + url = r.url ?: "", + method = r.method ?: "POST", + deliveries = r.deliveries.coerceAtLeast(0), + ) } } diff --git a/android/src/main/java/ai/openspace/backgroundupload/EventReporter.kt b/android/src/main/java/ai/openspace/backgroundupload/EventReporter.kt index 46460e1e..ea3cc62a 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/EventReporter.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/EventReporter.kt @@ -1,48 +1,69 @@ package ai.openspace.backgroundupload -import android.content.Context import com.facebook.react.bridge.Arguments -// Sends live events to JS through the module's codegen event emitters. Terminal -// outcomes are journaled before they reach here, so when JS is absent (headless -// worker, mid-reload) dropping the live event costs nothing — the consumer picks -// it up from getUnacknowledgedEvents instead. -object EventReporter { - - // Journal first, then emit. The journal is the durable record; it survives - // when JS is dead. The live emit is best-effort. The two carry the identical - // payload. Thus a consumer can acknowledge a live event by its eventId. - fun journalAndEmit(context: Context, entry: EventJournal.Entry) { - EventJournal.get(context).append(entry) - emit(entry) +/** The events that queue transitions produce. [EventReporter] sends them to JS; tests record them. */ +interface QueueEvents { + fun state(row: RequestRow) + + /** + * The caller journaled [record] first, and emits it only when its + * deliveries is above 0: [EventJournal] decides that under its lock. + * [listener] is [EventJournal.listener]: the module whose JS drained, so + * the delivery the journal counted goes to that JS. + */ + fun settled(record: EventJournal.SettledRecord, listener: Any) +} + +/** + * Sends live events to JS through the module's codegen emitters. With no + * module (a headless worker, a reload) an event is dropped. Settled outcomes + * are journaled before they reach here, so a dropped one is replayed from + * the journal. + */ +object EventReporter : QueueEvents { + + private val throttle = ProgressThrottle { id, sent, total -> + val module = UploaderModule.instance ?: return@ProgressThrottle + module.emitProgress(Arguments.createMap().apply { + putString("id", id) + putDouble("bytesSent", sent.toDouble()) + putDouble("totalBytes", total.toDouble()) + }) } - // Emit a terminal event from its journal entry, so the live event carries the - // exact same payload (incl. eventId) as the journaled copy — letting a consumer - // ackEvents([eventId]) right after handling a live event, and keeping iOS/Android - // event shapes identical. - fun emit(entry: EventJournal.Entry) { + override fun state(row: RequestRow) { val module = UploaderModule.instance ?: return - val params = entry.toWritableMap() - when (entry.type) { - "completed" -> module.emitCompletedEvent(params) - "cancelled" -> module.emitCancelledEvent(params) - else -> module.emitErrorEvent(params) - } + module.emitState(JsonBridge.toWritableMap(row.toMap())) + } + + // Not UploaderModule.instance: a reload sets that before the new JS + // subscribes, and the journal counted this delivery for the listener. + override fun settled(record: EventJournal.SettledRecord, listener: Any) { + val module = listener as? UploaderModule ?: return + module.emitSettled(record.toWritableMap()) } - fun progress(uploadId: String, bytesSentTotal: Long, contentLength: Long) { + /** Moves the row's bytesSent in memory and emits through the throttle. */ + fun progress(id: String, sent: Long, total: Long) { + RequestIndex.shared.setBytes(id, sent) + throttle.offer(id, sent, total, isForeground()) + } + + fun flushProgress(id: String) = throttle.flush(id) + + fun dropProgress(id: String) = throttle.drop(id) + + fun attempt(event: AttemptEvent) { val module = UploaderModule.instance ?: return - module.emitProgressEvent(Arguments.createMap().apply { - putString("id", uploadId) - // Guard against a zero-byte file (contentLength == 0) producing NaN. - val pct = if (contentLength <= 0) 0.0 else bytesSentTotal.toDouble() * 100 / contentLength - putDouble("progress", pct) // 0-100 - }) + module.emitAttempt(event.toWritableMap()) } fun notification() { val module = UploaderModule.instance ?: return - module.emitNotificationEvent(Arguments.createMap()) + module.emitNotification(Arguments.createMap()) } + + private fun isForeground(): Boolean = + runCatching { UploaderModule.instance?.isForeground() == true }.getOrDefault(false) } diff --git a/android/src/main/java/ai/openspace/backgroundupload/JsonBridge.kt b/android/src/main/java/ai/openspace/backgroundupload/JsonBridge.kt new file mode 100644 index 00000000..fe84fbe7 --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/JsonBridge.kt @@ -0,0 +1,164 @@ +package ai.openspace.backgroundupload + +import com.facebook.react.bridge.Arguments +import com.facebook.react.bridge.ReadableArray +import com.facebook.react.bridge.ReadableMap +import com.facebook.react.bridge.ReadableType +import com.facebook.react.bridge.WritableArray +import com.facebook.react.bridge.WritableMap +import com.google.gson.Gson +import com.google.gson.JsonElement +import com.google.gson.JsonParser +import com.google.gson.JsonPrimitive +import com.google.gson.Strictness +import com.google.gson.stream.JsonReader +import com.google.gson.stream.JsonToken +import java.io.StringReader +import kotlin.math.abs +import kotlin.math.floor + +/** + * Moves values between the bridge (ReadableMap, WritableMap), plain Kotlin + * values (Map, List, String, Double, Boolean, null), and JSON text. `vars` + * and `data` cross from JS as JSON text and are stored as it came; native + * parses them back only to hand objects to JS. + * + * Numbers: RN gives every JS number to Kotlin as a Double. [numberText] + * writes a Double with no fraction as an integer, so a header value 5 is + * "5", as JSON.stringify writes it, not "5.0". + */ +object JsonBridge { + private val gson = Gson() + + // 2^53. Above this a Double can not hold every integer, so it keeps the + // Double form. + private const val MAX_SAFE_INTEGER = 9_007_199_254_740_992.0 + + fun fromReadable(map: ReadableMap): Map { + val out = LinkedHashMap() + val keys = map.keySetIterator() + while (keys.hasNextKey()) { + val key = keys.nextKey() + out[key] = valueOf(map, key) + } + return out + } + + fun fromReadableArray(array: ReadableArray): List = + (0 until array.size()).map { i -> + when (array.getType(i)) { + ReadableType.Null -> null + ReadableType.Boolean -> array.getBoolean(i) + ReadableType.Number -> array.getDouble(i) + ReadableType.String -> array.getString(i) + ReadableType.Map -> array.getMap(i)?.let { fromReadable(it) } + ReadableType.Array -> array.getArray(i)?.let { fromReadableArray(it) } + } + } + + /** The plain value at [key]. Null for an absent key. */ + fun valueOf(map: ReadableMap, key: String): Any? { + if (!map.hasKey(key)) return null + return when (map.getType(key)) { + ReadableType.Null -> null + ReadableType.Boolean -> map.getBoolean(key) + ReadableType.Number -> map.getDouble(key) + ReadableType.String -> map.getString(key) + ReadableType.Map -> map.getMap(key)?.let { fromReadable(it) } + ReadableType.Array -> map.getArray(key)?.let { fromReadableArray(it) } + } + } + + /** JSON text to plain values. Throws on malformed text. Numbers come back as Double. */ + fun parse(json: String): Any? = fromElement(JsonParser.parseString(json)) + + /** + * Whether [text] is one strict JSON value, as JSON.stringify writes it. + * Any top-level value counts, so "null" is valid. Lenient forms (single + * quotes, bare keys, trailing text) are not. + */ + fun isJson(text: String): Boolean = runCatching { + val reader = JsonReader(StringReader(text)).apply { setStrictness(Strictness.STRICT) } + gson.getAdapter(JsonElement::class.java).read(reader) + reader.peek() == JsonToken.END_DOCUMENT + }.getOrDefault(false) + + /** A number as JSON would print it: an integer when it has no fraction. */ + fun numberText(d: Double): String = gson.toJson(number(d)) + + private fun number(d: Double): JsonPrimitive = + if (d.isFinite() && d == floor(d) && abs(d) < MAX_SAFE_INTEGER) JsonPrimitive(d.toLong()) + else JsonPrimitive(d) + + private fun fromElement(element: JsonElement): Any? = when { + element.isJsonNull -> null + element.isJsonObject -> LinkedHashMap().apply { + element.asJsonObject.entrySet().forEach { (k, v) -> put(k, fromElement(v)) } + } + element.isJsonArray -> element.asJsonArray.map { fromElement(it) } + else -> { + val p = element.asJsonPrimitive + when { + p.isBoolean -> p.asBoolean + p.isNumber -> p.asDouble + else -> p.asString + } + } + } + + /** + * Plain values to the bridge. The factories default to the native ones. The + * JVM tests pass JavaOnlyMap and JavaOnlyArray, because the native ones need + * the React Native C++ library. + */ + fun toWritableMap( + map: Map, + newMap: () -> WritableMap = Arguments::createMap, + newArray: () -> WritableArray = Arguments::createArray, + ): WritableMap { + val out = newMap() + map.forEach { (k, v) -> putValue(out, k, v, newMap, newArray) } + return out + } + + fun toWritableArray( + list: List, + newMap: () -> WritableMap = Arguments::createMap, + newArray: () -> WritableArray = Arguments::createArray, + ): WritableArray { + val out = newArray() + list.forEach { v -> + when (v) { + null -> out.pushNull() + is Boolean -> out.pushBoolean(v) + is Number -> out.pushDouble(v.toDouble()) + is String -> out.pushString(v) + is Map<*, *> -> out.pushMap(toWritableMap(stringKeys(v), newMap, newArray)) + is List<*> -> out.pushArray(toWritableArray(v, newMap, newArray)) + else -> out.pushString(v.toString()) + } + } + return out + } + + fun putValue( + map: WritableMap, + key: String, + value: Any?, + newMap: () -> WritableMap = Arguments::createMap, + newArray: () -> WritableArray = Arguments::createArray, + ) { + when (value) { + null -> map.putNull(key) + is Boolean -> map.putBoolean(key, value) + is Number -> map.putDouble(key, value.toDouble()) + is String -> map.putString(key, value) + is Map<*, *> -> map.putMap(key, toWritableMap(stringKeys(value), newMap, newArray)) + is List<*> -> map.putArray(key, toWritableArray(value, newMap, newArray)) + else -> map.putString(key, value.toString()) + } + } + + private fun stringKeys(map: Map<*, *>): Map = + map.entries.associate { (k, v) -> k.toString() to v } +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/LegacyImport.kt b/android/src/main/java/ai/openspace/backgroundupload/LegacyImport.kt new file mode 100644 index 00000000..da5b19ab --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/LegacyImport.kt @@ -0,0 +1,104 @@ +package ai.openspace.backgroundupload + +import android.content.Context +import com.google.gson.Gson +import java.io.File + +/** + * First v10 launch. Each v9 journal entry becomes a read-only settled row + * with key `legacy` and the v9 upload id. Nothing is delivered; the app + * reads the rows with getRequests() and cancels them. v9 chunked manifests + * are left in place: a same-id enqueue adopts them. + * + * The caller cancels the v9 WorkManager rows after this returns true. The + * order is safe: under v10 code a v9 row exits at once (it has no entry + * id), so nothing can write a v9 journal file after the import. + * + * Runs once, guarded by a marker file. A failed row save leaves the marker + * unwritten, so the next launch tries again. + */ +object LegacyImport { + const val MARKER = "v9-imported" + const val KEY = "legacy" + const val V9_JOURNAL_DIR = "rnbgupload-events" + + /** The v9 journal Entry, every field nullable: Gson reads whatever is there. */ + data class V9Entry( + val eventId: String?, + val uploadId: String?, + val type: String?, + val timestamp: Long?, + ) + + private val gson = Gson() + + /** Returns true when the import ran at this launch (no marker yet). */ + fun runOnce(context: Context, store: QueueStore): Boolean = + runOnce(File(QueueStore.rootDir(context), MARKER), File(context.filesDir, V9_JOURNAL_DIR), store) + + /** [runOnce] over plain files, for the JVM tests. */ + internal fun runOnce(marker: File, v9Dir: File, store: QueueStore): Boolean { + if (marker.exists()) return false + val complete = import(v9Dir, store) + if (complete) { + runCatching { AtomicFiles.writeText(marker, "1") } + .onFailure { Diag.error("could not write the v9 import marker", it) } + } + return true + } + + /** + * Imports every v9 journal entry in [v9Dir]. The newest entry per upload + * id wins. Returns false when a row could not be saved (its file is kept). + */ + internal fun import(v9Dir: File, store: QueueStore): Boolean { + val files = v9Dir.listFiles { f -> f.extension == "json" } ?: return true + val read = files.mapNotNull { f -> + val entry = runCatching { gson.fromJson(f.readText(), V9Entry::class.java) }.getOrNull() + if (entry == null) { + Diag.warn("v9 journal file unreadable, skipped: ${f.name}") + f.delete() + null + } else f to entry + } + var complete = true + read.groupBy { it.second.uploadId }.forEach { (uploadId, group) -> + val newest = group.maxByOrNull { it.second.timestamp ?: 0L }!!.second + val row = if (uploadId == null) null else legacyRow(newest) + val saved = when { + row == null -> true + store.load(row.id) != null -> true // a v10 entry already owns the id + else -> runCatching { store.save(row) }.isSuccess + } + if (saved) group.forEach { it.first.delete() } else complete = false + } + return complete + } + + internal fun legacyRow(entry: V9Entry): QueueEntry? { + val id = entry.uploadId ?: return null + val state = when (entry.type) { + "completed" -> EntryState.COMPLETED + "error" -> EntryState.ERROR + "cancelled" -> EntryState.CANCELLED + else -> return null + } + val at = entry.timestamp ?: 0L + return QueueEntry( + id = id, + key = KEY, + varsJson = "null", + descriptor = null, + body = null, + state = state, + attempts = 0, + bytesSent = 0, + totalBytes = 0, + expiresAt = at, + createdAt = at, + updatedAt = at, + generation = 1, + legacy = true, + ) + } +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/ProgressThrottle.kt b/android/src/main/java/ai/openspace/backgroundupload/ProgressThrottle.kt new file mode 100644 index 00000000..f4032d20 --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/ProgressThrottle.kt @@ -0,0 +1,55 @@ +package ai.openspace.backgroundupload + +/** + * Limits progress events per id: at most one per second while the app is in + * the foreground, one per 10 minutes in the background. A value held back is + * kept as pending; [flush] sends it (the trailing edge on settle, park, + * release, and stop). + */ +class ProgressThrottle( + private val clock: () -> Long = System::currentTimeMillis, + private val emit: (id: String, sent: Long, total: Long) -> Unit, +) { + companion object { + const val FOREGROUND_MS = 1_000L + const val BACKGROUND_MS = 600_000L + } + + private class Slot(var lastEmitAt: Long?, var pending: Pair?) + + private val slots = HashMap() + + fun offer(id: String, sent: Long, total: Long, foreground: Boolean) { + val interval = if (foreground) FOREGROUND_MS else BACKGROUND_MS + val now = clock() + val send = synchronized(slots) { + val slot = slots.getOrPut(id) { Slot(null, null) } + val last = slot.lastEmitAt + if (last == null || now - last >= interval) { + slot.lastEmitAt = now + slot.pending = null + true + } else { + slot.pending = sent to total + false + } + } + if (send) emit(id, sent, total) + } + + /** Sends the held-back value once, if there is one. */ + fun flush(id: String) { + val pending = synchronized(slots) { + val slot = slots[id] ?: return + val p = slot.pending ?: return + slot.pending = null + slot.lastEmitAt = clock() + p + } + emit(id, pending.first, pending.second) + } + + fun drop(id: String) { + synchronized(slots) { slots.remove(id) } + } +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/QueueController.kt b/android/src/main/java/ai/openspace/backgroundupload/QueueController.kt new file mode 100644 index 00000000..c72cb0fe --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/QueueController.kt @@ -0,0 +1,458 @@ +package ai.openspace.backgroundupload + +import java.io.IOException +import java.util.UUID + +/** + * Every transition that JS causes: enqueue, pause, resume, cancel, + * setWifiOnly, updateHeaders, ack, and the boot sweep. [UploaderModule] calls + * it from one single-thread executor, so these calls never overlap each other. + * Workers change entries at the same time; every change here is inside the + * store lock, so each one is atomic against them. + * + * Order of a change: journal (when there is an outcome), store, work + * schedule, then events. Events go out after the store lock is released. + */ +class QueueController( + private val store: QueueStore, + private val journal: EventJournal, + private val settings: QueueSettingsStore, + private val events: QueueEvents, + private val scheduler: WorkScheduler, + private val isWorkerRunning: (id: String) -> Boolean = WorkerGate::isRunning, + private val clock: () -> Long = System::currentTimeMillis, +) { + + // MARK: - enqueue + + private class Enqueued(val entry: QueueEntry?, val reEmit: EventJournal.SettledRecord?) + + /** A body staged before the store lock was taken, and the generation it was staged for. */ + private class PreStaged(val generation: Int, val staged: BodyStaging.Staged) + + /** Test seam: runs between the staging outside the lock and the commit under it. */ + internal var afterPreStage: () -> Unit = {} + + /** + * Persists the entry and every staged byte, then schedules it. Returns the + * id. Throws [QueueException] with E_RUNNING, E_FILE_MISSING, E_STORAGE, or + * E_INVALID. + * + * A copied body is staged before the store lock when the decision can not + * change meanwhile (see [EnqueueRules.preStageGeneration]), so a large + * copy does not block every worker transition. Under the lock the + * decision is made again; a staged body that no longer fits is deleted + * and the body is staged again under the lock. + */ + fun enqueue(p: EntryParsing.Parsed): String { + val pre = preStage(p) + var preUsed = false + fun preFor(generation: Int): BodyStaging.Staged? = + pre?.takeIf { it.generation == generation }?.staged?.also { preUsed = true } + val result = try { + store.locked { decideAndCommit(p, ::preFor) } + } finally { + if (pre != null && !preUsed) discard(p.id, pre) + } + result.reEmit?.let { record -> journal.listener()?.let { events.settled(record, it) } } + result.entry?.let { entry -> + scheduleRun(entry) + events.state(entry.toRow()) + } + return p.id + } + + private fun preStage(p: EntryParsing.Parsed): PreStaged? { + val generation = store.locked { + val existing = store.load(p.id) + val v9 = v9ManifestFor(existing, p.id) + val action = EnqueueRules.decide(existing, v9, p.descriptor) { journal.find(it) != null } + EnqueueRules.preStageGeneration(existing, action, p.descriptor) + } ?: return null + val staged = stageOrThrow(p.descriptor, store.entryDir(p.id), generation) + afterPreStage() + return PreStaged(generation, staged) + } + + /** Deletes a pre-staged body that the commit did not use, unless the stored entry points at it. */ + private fun discard(id: String, pre: PreStaged) { + val file = pre.staged.body.fileName?.let { java.io.File(store.entryDir(id), it) } ?: return + val current = store.load(id) + if (current == null || store.bodyFile(current) != file) file.delete() + } + + /** Runs under the store lock. [preStaged] returns the body staged outside the lock for a generation, if any. */ + private fun decideAndCommit(p: EntryParsing.Parsed, preStaged: (generation: Int) -> BodyStaging.Staged?): Enqueued { + val existing = store.load(p.id) + val v9 = v9ManifestFor(existing, p.id) + val s = settings.load() + val now = clock() + val dir = store.entryDir(p.id) + return when (val action = EnqueueRules.decide(existing, v9, p.descriptor) { journal.find(it) != null }) { + EnqueueRules.Action.RejectRunning -> throw QueueException( + QueueException.E_RUNNING, + "entry '${p.id}' is running; a different body is accepted once it stops", + ) + // With no listener yet, the next drain delivers it. + is EnqueueRules.Action.ReEmit -> Enqueued(null, journal.redeliver(action.eventId)) + EnqueueRules.Action.Resume -> { + val next = EnqueueRules.resumed(existing!!, p, s.paused, s.headerGeneration, now) + saveOrThrow(next) + Enqueued(next, null) + } + EnqueueRules.Action.Create -> { + val staged = preStaged(1) ?: stageOrThrow(p.descriptor, dir, 1) + commit(EnqueueRules.created(p, staged, null, s.paused, s.headerGeneration, now)) + } + is EnqueueRules.Action.AdoptV9 -> { + val incoming = p.descriptor.parts + val parts = incoming?.let { EnqueueRules.adoptedParts(action.manifest, it) } + // The same parts resume over the v9 blob, as a same-body enqueue does. + val keepOwned = incoming != null && ChunkedParts.sameParts(action.manifest.parts, incoming) + val staged = stageOrThrow(p.descriptor.copy(parts = parts), dir, action.generation, store.blobFile(p.id), keepOwned) + commit(EnqueueRules.adopted(p, staged, parts, existing, action.generation, s.paused, s.headerGeneration, now)) + } + EnqueueRules.Action.Replace -> { + val old = existing!! + // Different parts: a present caller file wins; the old blob is the fallback. + val ownedBlob = if (old.body?.kind == StagedBody.CHUNKED) store.bodyFile(old) else null + val staged = preStaged(old.generation + 1) + ?: stageOrThrow(p.descriptor, dir, old.generation + 1, ownedBlob) + commit(EnqueueRules.replaced(old, p, staged, s.paused, s.headerGeneration, now)) + } + } + } + + /** A v9 manifest counts only where no v10 entry owns the id, or the owner is its legacy row. */ + private fun v9ManifestFor(existing: QueueEntry?, id: String): LegacyManifest? = + if (existing == null || existing.legacy) store.legacyManifest(id) else null + + private fun stageOrThrow( + d: Descriptor, + dir: java.io.File, + generation: Int, + ownedBlob: java.io.File? = null, + keepOwned: Boolean = false, + ): BodyStaging.Staged = + try { + BodyStaging.stage(d, dir, generation, ownedBlob, keepOwned) + } catch (e: QueueException) { + throw e + } catch (e: IOException) { + throw QueueException(QueueException.E_STORAGE, "could not stage the body: ${e.message}") + } + + /** Saves a newly staged entry. On failure the new body file is removed; the old entry stays as it was. */ + private fun commit(next: QueueEntry): Enqueued { + try { + store.save(next) + } catch (e: IOException) { + if (next.body?.kind != StagedBody.CHUNKED) store.bodyFile(next)?.delete() + throw QueueException(QueueException.E_STORAGE, "could not save the entry: ${e.message}") + } + store.pruneUnreferenced(next) + return Enqueued(next, null) + } + + private fun saveOrThrow(next: QueueEntry) { + try { + store.save(next) + } catch (e: IOException) { + throw QueueException(QueueException.E_STORAGE, "could not save the entry: ${e.message}") + } + } + + // MARK: - queue control + + /** Whole-queue pause. Live rows move to paused and their work stops. No outcome. */ + fun pause() { + settings.update { it.copy(paused = true) } + val now = clock() + val paused = transformAll { e -> + if (e.isLive && e.state != EntryState.PAUSED) EntryTransitions.toPaused(e, now) else e + } + paused.forEach { scheduler.cancel(it.id) } + paused.forEach { events.state(it.toRow()) } + } + + fun resume() { + val s = settings.update { it.copy(paused = false) } + val now = clock() + val resumed = transformAll { e -> + if (e.state == EntryState.PAUSED) EntryTransitions.toResumed(e, s.headerGeneration, now) else e + } + resumed.forEach { events.state(it.toRow()) } + // Every queued entry, not only the resumed ones: a run is idempotent. + store.all().forEach { scheduleRun(it) } + } + + /** + * Live: journal a 'cancelled' (user) outcome, then forget after its ack. + * Settled (or legacy): forget now, row, bytes, and its unacknowledged + * outcomes. Unknown: no-op. + * + * A failed journal write rejects E_STORAGE and changes nothing: the entry + * goes on, and JS can call cancel() again. + * + * A failed entry save after the journal write also rejects E_STORAGE, but + * the cancel is durable: the work is stopped and the record is emitted. + * Its ack, the next cancel(), a worker's begin or settle, or the boot + * sweep applies it. A live entry that already has a record of its own + * generation gets that record applied, not a second outcome. + */ + fun cancel(id: String) { + var stop = false + var journaled: EventJournal.SettledRecord? = null + var saved: QueueEntry? = null + try { + store.locked { + stop = true // unknown or settled: stop any stray work, as before + val e = store.load(id) ?: return@locked + if (!e.isLive || e.legacy) { + journal.ackEntry(id) + store.remove(id) + return@locked + } + stop = false // a live entry: only once an outcome is journaled + val now = clock() + EntryTransitions.journaledSettle(e, journal.forEntry(id), now)?.let { + stop = true + saveOrThrow(it.entry) + journal.ack(it.extraEventIds) + saved = it.entry + return@locked + } + val record = cancelledRecord(e, now) + journaled = try { + journal.append(record) { store.referencedEventIds() + record.eventId } + } catch (error: IOException) { + throw QueueException(QueueException.E_STORAGE, "could not journal the cancel: ${error.message}") + } + stop = true + val next = EntryTransitions.toSettled(e, EntryState.CANCELLED, record.eventId, e.bytesSent, now) + saveOrThrow(next) + saved = next + } + } finally { + // Once an outcome is journaled, the worker must stop even when the + // save failed: a running request would settle a second outcome. + if (stop) scheduler.cancel(id) + journaled?.let { record -> + if (record.deliveries > 0) journal.listener()?.let { events.settled(record, it) } + } + saved?.let { events.state(it.toRow()) } + } + } + + private fun cancelledRecord(e: QueueEntry, now: Long) = EventJournal.SettledRecord( + eventId = UUID.randomUUID().toString(), + id = e.id, + key = e.key, + varsJson = e.varsJson, + at = now, + attempts = e.attempts, + requestId = e.lastRequestId, + deliveries = 0, // the journal sets it + state = EntryState.CANCELLED.wire, + bytesSent = e.bytesSent, + totalBytes = e.totalBytes, + url = e.descriptor?.reportUrl ?: "", + method = e.descriptor?.method ?: "POST", + partIndex = null, + kind = EventJournal.KIND_CANCELLED, + response = null, + errorKind = null, + message = null, + cancelReason = "user", + generation = e.generation, + ) + + fun setWifiOnly(enabled: Boolean) { + settings.update { it.copy(wifiOnly = enabled) } + } + + fun configureRetry(defaults: RetryDefaults) { + settings.update { it.copy(retry = defaults) } + } + + /** + * Bumps the header generation, merges [patch] into every entry not yet + * forgotten, and requeues the parked ones. Workers compare each entry's + * own headerGeneration, which changes here under the store lock together + * with its headers. So a worker that gets a 401 either sees the patched + * entry and re-issues, or parks first and is requeued here. + */ + fun updateHeaders(patch: Map) { + val s = settings.update { it.copy(headerGeneration = it.headerGeneration + 1) } + val now = clock() + val unparkedIds = mutableSetOf() + val changed = transformAll { e -> + val patched = e.withHeadersPatched(patch, s.headerGeneration) + if (patched.state != EntryState.AWAITING_AUTH) return@transformAll patched + unparkedIds += e.id + EntryTransitions.toUnparked(patched, s.paused, now) + } + changed.filter { it.id in unparkedIds }.forEach { entry -> + scheduleRun(entry) + events.state(entry.toRow()) + } + } + + // MARK: - journal + + /** + * getUnacknowledgedEvents(): [listener] becomes the JS listener, and every + * unacknowledged outcome is returned, each counted as one more delivery. + */ + fun unacknowledged(listener: Any, isActive: () -> Boolean = { true }): List = + journal.drain(listener, isActive) + + /** + * Removes the records. An acked completed or cancelled outcome of the + * entry's current life forgets the entry: row and bytes. An error keeps + * the row until cancel() or a same-id enqueue. Unknown ids are ignored. + * + * A record of the entry's current life on a live entry means the settle + * journaled and emitted, but its store write failed. The record is applied + * first, as the boot sweep would; without that, the sweep finds no record + * after this ack and runs the finished request again. When that save fails + * too, the record stays unacked so the sweep can apply it later. + */ + fun ack(eventIds: List) { + val forgotten = mutableListOf() + val repaired = mutableListOf() + store.locked { + eventIds.forEach { eventId -> + val record = journal.find(eventId) ?: return@forEach + var e = store.load(record.id) + if (e != null && e.isLive && !e.legacy && e.generation == record.generation) { + val next = EntryTransitions.toSettled(e, EntryTransitions.stateOf(record), record.eventId, record.bytesSent, clock()) + if (!trySave(next)) return@forEach + repaired += next + e = next + } + journal.ack(listOf(eventId)) + if (record.kind == EventJournal.KIND_ERROR || e == null) return@forEach + if (e.generation == record.generation && e.settledEventId == record.eventId) { + store.remove(record.id) + forgotten += record.id + } + } + } + forgotten.forEach { scheduler.cancel(it) } + repaired.filter { it.id !in forgotten }.forEach { events.state(it.toRow()) } + } + + // MARK: - boot sweep + + /** + * Repairs what a process death can leave, then schedules queued work. An + * entry whose worker runs in this process is skipped: that worker finishes + * its own transition. Run at module init, after [LegacyImport]. + * + * 1. A live entry with a journal record of its own generation: the process + * died between the journal append and the store transition. Apply it. + * 2. A running entry with no worker: a process death mid-run. Queue it. + * A live row that disagrees with the queue's paused setting (a death + * partway through pause() or resume()): make it agree. + * 3. A settled entry with records of its generation other than its own: + * orphans from a cancel race. Ack them. + * 4. A completed or cancelled entry whose own record is gone: the ack + * landed but the forget did not. Forget it. + * 5. Schedule every queued entry, and the expiry wake of every parked one. + */ + fun sweep() { + val now = clock() + val changed = mutableListOf() + val toForget = mutableListOf() + val toSchedule = mutableListOf() + val paused = settings.load().paused + store.locked { + val records = journal.unacknowledged().groupBy { it.id } + for (e in store.all()) { + if (e.legacy || isWorkerRunning(e.id)) continue + val own = records[e.id].orEmpty().filter { it.generation == e.generation } + if (e.isLive) { + val journaled = EntryTransitions.journaledSettle(e, own, now) + if (journaled != null) { + if (trySave(journaled.entry)) { + journal.ack(journaled.extraEventIds) + changed += journaled.entry + } + continue + } + var cur = e + if (e.state == EntryState.RUNNING) { + cur = EntryTransitions.toStopped(e, now) + } + // A process death partway through pause() or resume() leaves rows + // that disagree with the queue setting. + if (paused && cur.state != EntryState.PAUSED) { + cur = EntryTransitions.toPaused(cur, now) + } else if (!paused && cur.state == EntryState.PAUSED) { + cur = EntryTransitions.toResumed(cur, settings.load().headerGeneration, now) + } + if (cur !== e) { + if (trySave(cur)) changed += cur else continue + } + if (!paused || cur.state == EntryState.AWAITING_AUTH) toSchedule += cur + } else { + val orphans = own.filter { it.eventId != e.settledEventId } + if (orphans.isNotEmpty()) journal.ack(orphans.map { it.eventId }) + val forgettable = e.state == EntryState.COMPLETED || e.state == EntryState.CANCELLED + if (forgettable && own.none { it.eventId == e.settledEventId }) { + store.remove(e.id) + toForget += e.id + } + } + } + } + toForget.forEach { scheduler.cancel(it) } + toSchedule.forEach { scheduleRun(it, keepWake = true) } + changed.forEach { events.state(it.toRow()) } + } + + private fun trySave(entry: QueueEntry): Boolean = try { + store.save(entry) + true + } catch (e: IOException) { + Diag.error("could not save '${entry.id}'", e) + false + } + + // MARK: - helpers + + /** + * Runs [entry] when it is queued. A backoff longer than a worker waits goes + * to the wake; a parked entry gets a wake at its expiry, so it settles + * 'expired' on time. + */ + private fun scheduleRun(entry: QueueEntry, keepWake: Boolean = false) { + val now = clock() + when (entry.state) { + EntryState.QUEUED -> { + val at = entry.nextAttemptAt + if (at != null && at - now > RetryClassifier.IN_WORKER_BACKOFF_MAX_MS) { + scheduler.scheduleWake(entry, at, replace = !keepWake) + } else { + scheduler.schedule(entry) + } + } + EntryState.AWAITING_AUTH -> scheduler.scheduleWake(entry, entry.expiresAt, replace = !keepWake) + else -> Unit + } + } + + /** Applies [transform] to every non-legacy entry under the store lock. Returns the ones it changed. */ + private fun transformAll(transform: (QueueEntry) -> QueueEntry): List { + val changed = mutableListOf() + store.locked { + store.all().filter { !it.legacy }.forEach { e -> + store.compute(e.id) { cur -> + if (cur == null) null else transform(cur).also { if (it !== cur) changed += it } + } + } + } + return changed + } +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/QueueEntry.kt b/android/src/main/java/ai/openspace/backgroundupload/QueueEntry.kt new file mode 100644 index 00000000..7a5e7f2f --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/QueueEntry.kt @@ -0,0 +1,228 @@ +package ai.openspace.backgroundupload + +import com.google.gson.annotations.SerializedName + +/** The row states. [wire] is the RequestState string JS sees. */ +enum class EntryState(val wire: String) { + @SerializedName("queued") QUEUED("queued"), + @SerializedName("running") RUNNING("running"), + @SerializedName("awaiting-auth") AWAITING_AUTH("awaiting-auth"), + @SerializedName("paused") PAUSED("paused"), + @SerializedName("completed") COMPLETED("completed"), + @SerializedName("error") ERROR("error"), + @SerializedName("cancelled") CANCELLED("cancelled"); + + val isLive get() = this == QUEUED || this == RUNNING || this == AWAITING_AUTH || this == PAUSED +} + +/** One multipart field. [path] is the caller's file, read only at staging. */ +data class FormPart( + val name: String, + val contentType: String, + val string: String?, + val path: String?, + val fileName: String?, +) + +/** A request's `retry`, as a partial override of the configure() defaults. */ +data class RetryOverride( + val baseMs: Long?, + val maxMs: Long?, + val jitter: Double?, + val exempt: List?, +) + +/** What request(vars) returned, as native needs it to run. */ +data class Descriptor( + /** Null only with parts. */ + val url: String?, + val method: String, + /** The merged headers, plus the Content-Type that staging sets. */ + val headers: Map, + /** JSON text of `data`. It is also the bytes of the staged body. */ + val dataJson: String?, + val form: List?, + /** The caller's path, with any file:// prefix removed. */ + val file: String?, + /** The accepted flags live here. */ + val parts: List?, + val accept: List, + val retry: RetryOverride?, + val noNotification: Boolean, +) { + val bodyKind: String + get() = when { + parts != null -> StagedBody.CHUNKED + file != null -> StagedBody.FILE + form != null -> StagedBody.MULTIPART + dataJson != null -> StagedBody.JSON + else -> StagedBody.NONE + } + + /** The url to report for this request: the descriptor's, or the last part's. */ + val reportUrl: String get() = url ?: parts?.lastOrNull()?.url ?: "" +} + +/** + * Where the request body is on disk. [fileName] is relative to the entry + * directory, so a moved app data directory does not break it. + */ +data class StagedBody( + val kind: String, + val fileName: String?, + val boundary: String?, + val totalBytes: Long, +) { + companion object { + const val NONE = "none" + const val JSON = "json" + const val MULTIPART = "multipart" + const val FILE = "file" + const val CHUNKED = "chunked" + } +} + +/** One queue entry. [QueueStore] persists it as `entry.json` with Gson. */ +data class QueueEntry( + val id: String, + val key: String, + /** "null" for null vars. */ + val varsJson: String, + /** Null only for a legacy row. */ + val descriptor: Descriptor?, + /** Null only for a legacy row. */ + val body: StagedBody?, + val state: EntryState, + /** Attempts in this life. Chunked: across every part. */ + val attempts: Int, + val bytesSent: Long, + val totalBytes: Long, + val expiresAt: Long, + val createdAt: Long, + val updatedAt: Long, + /** Set while the entry waits out a backoff: queued for a long one, running for a short one. */ + val nextAttemptAt: Long? = null, + /** The consecutive transient failures before the last release. The next backoff continues from it. */ + val backoffStreak: Int = 0, + /** The settings header generation the headers were last merged at. */ + val headerGeneration: Int = 0, + /** Set while awaiting-auth (and kept under pause): the header generation it parked under. */ + val parkedGeneration: Int? = null, + /** +1 each time a settled entry reopens, and on a different-body replace. Journal records carry it. */ + val generation: Int = 1, + /** The journal record of this life's outcome. */ + val settledEventId: String? = null, + /** The X-Request-Id of the last attempt. */ + val lastRequestId: String? = null, + val legacy: Boolean = false, +) { + val isLive get() = state.isLive + val isSettled get() = !state.isLive + + fun toRow() = RequestRow( + id = id, + key = key, + varsJson = varsJson, + state = state.wire, + bytesSent = bytesSent, + totalBytes = totalBytes, + attempts = attempts, + updatedAt = updatedAt, + nextAttemptAt = nextAttemptAt, + createdAt = createdAt, + ) + + /** + * Whether [incoming] carries the same body. A different body kind, a + * different url or method, or different content is a different body. + * Chunked compares the parts (the path is ignored: the owned blob is the + * truth, as in v9). + */ + fun sameBodyAs(incoming: Descriptor): Boolean { + val stored = descriptor ?: return false + if (stored.bodyKind != incoming.bodyKind) return false + if (stored.method != incoming.method) return false + return when (stored.bodyKind) { + StagedBody.CHUNKED -> ChunkedParts.sameParts(stored.parts!!, incoming.parts!!) + StagedBody.FILE -> stored.url == incoming.url && stored.file == incoming.file + StagedBody.MULTIPART -> stored.url == incoming.url && stored.form == incoming.form + StagedBody.JSON -> stored.url == incoming.url && stored.dataJson == incoming.dataJson + else -> stored.url == incoming.url + } + } + + /** + * updateHeaders(): the patch replaces same-named headers (any case) and adds + * the rest. A part that carries its own copy of a patched header gets the + * new value too, because a stale per-part Authorization would shadow the + * fresh one. + */ + fun withHeadersPatched(patch: Map, generation: Int): QueueEntry { + val d = descriptor ?: return copy(headerGeneration = generation) + return copy( + descriptor = d.copy( + headers = HeaderMap.merge(d.headers, patch), + parts = d.parts?.map { part -> + val shared = patch.filterKeys { name -> HeaderMap.contains(part.headers, name) } + if (shared.isEmpty()) part else part.copy(headers = HeaderMap.merge(part.headers, shared)) + }, + ), + headerGeneration = generation, + ) + } +} + +/** One row of getRequests() and of a state event. */ +class RequestRow( + val id: String, + val key: String, + val varsJson: String, + val state: String, + val bytesSent: Long, + val totalBytes: Long, + val attempts: Int, + val updatedAt: Long, + val nextAttemptAt: Long?, + /** Sort order only; not sent to JS. */ + val createdAt: Long, +) { + /** vars parsed back to an object, once. A malformed text reads as null. */ + val vars: Any? by lazy { runCatching { JsonBridge.parse(varsJson) }.getOrNull() } + + fun withBytes(sent: Long) = RequestRow( + id, key, varsJson, state, sent, totalBytes, attempts, updatedAt, nextAttemptAt, createdAt, + ) + + /** The RequestRow shape. nextAttemptAt only when set. */ + fun toMap(): Map = LinkedHashMap().apply { + put("id", id) + put("key", key) + put("vars", vars) + put("state", state) + put("bytesSent", bytesSent.toDouble()) + put("totalBytes", totalBytes.toDouble()) + put("attempts", attempts.toDouble()) + put("updatedAt", updatedAt.toDouble()) + nextAttemptAt?.let { put("nextAttemptAt", it.toDouble()) } + } +} + +/** Header maps whose names match without regard to case. */ +object HeaderMap { + fun contains(headers: Map, name: String) = + headers.keys.any { it.equals(name, ignoreCase = true) } + + fun get(headers: Map, name: String): String? = + headers.entries.firstOrNull { it.key.equals(name, ignoreCase = true) }?.value + + /** [over] replaces same-named entries of [base] (any case); its spelling is kept. */ + fun merge(base: Map, over: Map): Map { + val out = LinkedHashMap() + base.forEach { (k, v) -> if (!contains(over, k)) out[k] = v } + out.putAll(over) + return out + } + + fun without(headers: Map, name: String): Map = + headers.filterKeys { !it.equals(name, ignoreCase = true) } +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/QueueSettings.kt b/android/src/main/java/ai/openspace/backgroundupload/QueueSettings.kt new file mode 100644 index 00000000..1a364526 --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/QueueSettings.kt @@ -0,0 +1,97 @@ +package ai.openspace.backgroundupload + +import android.content.Context +import com.google.gson.Gson +import java.io.File + +/** The configure() retry defaults. A request's `retry` overrides them field by field. */ +data class RetryDefaults( + val baseMs: Long = 1_000, + val maxMs: Long = 7_200_000, + val jitter: Double = 0.2, + val exempt: List = listOf(404), +) + +/** Queue-wide settings. They live next to the entries, in `settings.json`. */ +data class QueueSettings( + val wifiOnly: Boolean = false, + val paused: Boolean = false, + /** +1 per updateHeaders(). A 401 from an attempt sent under an older value re-issues at once. */ + val headerGeneration: Int = 0, + val retry: RetryDefaults = RetryDefaults(), +) + +/** + * Reads and writes [QueueSettings]. The value is cached after the first read. + * Workers read it before every attempt, so the cache matters. A corrupt file + * reads as the defaults. + */ +class QueueSettingsStore(private val file: File) { + + companion object { + private val gson = Gson() + + @Volatile + private var instance: QueueSettingsStore? = null + + fun get(context: Context): QueueSettingsStore = + instance ?: synchronized(this) { + instance ?: QueueSettingsStore(File(QueueStore.rootDir(context), "settings.json")) + .also { instance = it } + } + + /** configure().retry → defaults. Absent fields keep the library defaults. */ + fun retryDefaults(retry: Map?): RetryDefaults { + val d = RetryDefaults() + if (retry == null) return d + val backoff = retry["backoff"] as? Map<*, *> + val terminal = retry["terminalHttp"] as? Map<*, *> + return RetryDefaults( + baseMs = (backoff?.get("baseMs") as? Number)?.toLong() ?: d.baseMs, + maxMs = (backoff?.get("maxMs") as? Number)?.toLong() ?: d.maxMs, + jitter = (backoff?.get("jitter") as? Number)?.toDouble() ?: d.jitter, + exempt = (terminal?.get("exempt") as? List<*>)?.mapNotNull { (it as? Number)?.toInt() } + ?: d.exempt, + ) + } + } + + private var cached: QueueSettings? = null + + @Synchronized + fun load(): QueueSettings { + cached?.let { return it } + val read = if (file.exists()) { + runCatching { gson.fromJson(file.readText(), QueueSettings::class.java) }.getOrNull() + } else null + return validated(read).also { cached = it } + } + + /** Throws IOException when the write fails. The cache then keeps the old value. */ + @Synchronized + fun update(transform: (QueueSettings) -> QueueSettings): QueueSettings { + val next = transform(load()) + AtomicFiles.writeText(file, gson.toJson(next)) + cached = next + return next + } + + // Gson does not run constructors, so absent fields read as null or 0. + @Suppress("SENSELESS_COMPARISON", "USELESS_ELVIS") + private fun validated(s: QueueSettings?): QueueSettings { + if (s == null) return QueueSettings() + val d = RetryDefaults() + val r = s.retry + return QueueSettings( + wifiOnly = s.wifiOnly, + paused = s.paused, + headerGeneration = s.headerGeneration, + retry = if (r == null) d else RetryDefaults( + baseMs = if (r.baseMs > 0) r.baseMs else d.baseMs, + maxMs = if (r.maxMs > 0) r.maxMs else d.maxMs, + jitter = r.jitter, + exempt = r.exempt ?: d.exempt, + ), + ) + } +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/QueueStore.kt b/android/src/main/java/ai/openspace/backgroundupload/QueueStore.kt new file mode 100644 index 00000000..3db76532 --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/QueueStore.kt @@ -0,0 +1,221 @@ +package ai.openspace.backgroundupload + +import android.content.Context +import com.google.gson.Gson +import java.io.File +import java.io.IOException +import java.util.Base64 + +/** + * The durable queue: one directory per entry id. This is the v9 chunked + * manifest store, generalized in place. The root directory keeps its v9 name + * (`rnbgupload-chunked`) so v9 blobs and manifests are found without a move. + * + * A v10 directory holds `entry.json` and at most one staged body file (see + * [StagedBody.fileName]). A v9 directory holds `manifest.json` and `blob` + * until a same-id enqueue adopts it. + * + * Every write is tmp + fsync + rename ([AtomicFiles]). Bodies are staged + * before `entry.json` is saved, so an `entry.json` on disk means its body is + * durable. The store keeps [RequestIndex] current on every save and remove. + * It is reachable from a bare Context, because a worker can run in a process + * where React never started. + */ +class QueueStore(private val dir: File, private val index: RequestIndex = RequestIndex()) { + + companion object { + const val ENTRY_FILE = "entry.json" + const val V9_MANIFEST_FILE = "manifest.json" + const val BLOB_FILE = "blob" + + private val gson = Gson() + + @Volatile + private var instance: QueueStore? = null + + fun rootDir(context: Context) = File(context.filesDir, "rnbgupload-chunked") + + /** The process-wide store. The first call loads every row into [RequestIndex.shared]. */ + fun get(context: Context): QueueStore = + instance ?: synchronized(this) { + instance ?: QueueStore(rootDir(context), RequestIndex.shared) + .also { it.loadIndex() } + .also { instance = it } + } + } + + init { + dir.mkdirs() + } + + /** Rebuilds the index from disk. */ + @Synchronized + fun loadIndex() { + index.replaceAll(all().map { it.toRow() }) + } + + // Ids come from the caller and can hold path separators, so the directory + // name is an encoding of the id. The id is read back from the file. + fun entryDir(id: String) = + File(dir, Base64.getUrlEncoder().withoutPadding().encodeToString(id.toByteArray())) + + fun blobFile(id: String) = File(entryDir(id), BLOB_FILE) + + /** The staged body file of [entry], or null for a bodiless request. */ + fun bodyFile(entry: QueueEntry): File? = + entry.body?.fileName?.let { File(entryDir(entry.id), it) } + + private fun entryFile(id: String) = File(entryDir(id), ENTRY_FILE) + + /** Runs [block] under the store lock. For work that spans several store calls. */ + fun locked(block: () -> T): T = synchronized(this) { block() } + + @Synchronized + fun load(id: String): QueueEntry? = read(entryFile(id)) + + /** Throws IOException when the entry did not persist. */ + @Synchronized + fun save(entry: QueueEntry) { + AtomicFiles.writeText(entryFile(entry.id), gson.toJson(entry)) + index.put(entry.toRow()) + } + + /** + * An atomic read-modify-write. The lock spans load, [transform], and save, + * so nothing can write between them and be erased. A result that is the + * same object as the input writes nothing. A null result writes nothing: + * forgetting an entry is always an explicit [remove]. A throwing + * transform or a failed write propagates. + */ + @Synchronized + fun compute(id: String, transform: (QueueEntry?) -> QueueEntry?): QueueEntry? { + val current = load(id) + val next = transform(current) + if (next != null && next !== current) save(next) + return next + } + + /** Best effort, for a worker that can go on from memory: null when the entry is gone or the write failed. */ + @Synchronized + fun update(id: String, transform: (QueueEntry) -> QueueEntry): QueueEntry? = + runCatching { + val current = load(id) ?: return null + val next = transform(current) + if (next !== current) save(next) + next + }.getOrNull() + + /** + * Forgets the entry: row and bytes. `entry.json` goes first, so a partial + * delete never leaves a row that points at missing bytes. + */ + @Synchronized + fun remove(id: String) { + entryFile(id).delete() + entryDir(id).deleteRecursively() + index.remove(id) + } + + /** Every v10 entry. A directory with only a v9 manifest is not a row. */ + @Synchronized + fun all(): List = + (dir.listFiles { f -> f.isDirectory } ?: emptyArray()) + .mapNotNull { d -> File(d, ENTRY_FILE).takeIf { it.exists() }?.let { read(it) } } + + /** Every eventId a row names. The journal prune spares them. */ + @Synchronized + fun referencedEventIds(): Set = all().mapNotNull { it.settledEventId }.toSet() + + /** + * The v9 chunked manifest in [id]'s directory. The caller asks only when + * there is no v10 entry or the entry is a legacy row: a v10 entry + * prunes the manifest when it adopts it. + */ + @Synchronized + fun legacyManifest(id: String): LegacyManifest? { + val file = File(entryDir(id), V9_MANIFEST_FILE) + if (!file.exists()) return null + val parsed = runCatching { gson.fromJson(file.readText(), LegacyManifest::class.java) }.getOrNull() + return LegacyManifest.validated(parsed) + } + + /** + * Deletes every file in the entry directory that [entry] does not use: + * an older body, a v9 manifest it adopted, tmp files from a crash. + */ + @Synchronized + fun pruneUnreferenced(entry: QueueEntry) { + val keep = setOfNotNull(ENTRY_FILE, entry.body?.fileName) + entryDir(entry.id).listFiles()?.forEach { f -> + if (f.name !in keep) f.deleteRecursively() + } + } + + private fun read(file: File): QueueEntry? { + if (!file.exists()) return null + val parsed = try { + gson.fromJson(file.readText(), QueueEntry::class.java) + } catch (error: Throwable) { + Diag.warn("queue entry unreadable, skipped: ${file.parentFile?.name}", error) + return null + } + return validated(parsed).also { + if (it == null) Diag.warn("queue entry incomplete, skipped: ${file.parentFile?.name}") + } + } + + // Gson does not run constructors. A corrupt file, or one from an older + // build, can hold null in a non-null field. Reject what the engine relies + // on; normalize what has a safe default. + @Suppress("SENSELESS_COMPARISON", "USELESS_ELVIS") + private fun validated(e: QueueEntry?): QueueEntry? { + if (e == null || e.id == null || e.key == null || e.state == null) return null + if (!e.legacy && (e.descriptor == null || e.body == null || e.body.kind == null)) return null + val d = e.descriptor?.let { d -> + if (d.parts != null && d.parts.any { it == null || it.url == null }) return null + d.copy( + method = d.method ?: "POST", + headers = d.headers ?: emptyMap(), + accept = d.accept ?: emptyList(), + parts = d.parts?.map { if (it.headers == null) it.copy(headers = emptyMap()) else it }, + ) + } + return e.copy( + varsJson = e.varsJson ?: "null", + descriptor = d, + generation = if (e.generation <= 0) 1 else e.generation, + ) + } +} + +/** + * A v9 chunked manifest, only the fields v10 reads. The field names are the + * v9 ones; Gson skips the rest of the file. + */ +data class LegacyManifest( + val id: String, + val parts: List, + val accept: List, +) { + companion object { + @Suppress("SENSELESS_COMPARISON") + fun validated(m: LegacyManifest?): LegacyManifest? { + if (m == null || m.id == null || m.parts == null || m.parts.isEmpty()) return null + if (m.parts.any { it == null || it.url == null }) return null + return m.copy( + parts = m.parts.map { if (it.headers == null) it.copy(headers = emptyMap()) else it }, + accept = m.accept ?: emptyList(), + ) + } + } +} + +/** A rejection that reaches JS as `promise.reject(code, message)`. */ +class QueueException(val code: String, message: String) : IOException(message) { + companion object { + const val E_RUNNING = "E_RUNNING" + const val E_FILE_MISSING = "E_FILE_MISSING" + const val E_STORAGE = "E_STORAGE" + const val E_INVALID = "E_INVALID" + } +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/RequestIndex.kt b/android/src/main/java/ai/openspace/backgroundupload/RequestIndex.kt new file mode 100644 index 00000000..be3b7b0d --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/RequestIndex.kt @@ -0,0 +1,47 @@ +package ai.openspace.backgroundupload + +import java.util.concurrent.ConcurrentHashMap + +/** + * The in-memory rows behind the synchronous getRequests(). [QueueStore] + * keeps it current: it calls [put] after every save and [remove] after every + * remove. Progress ticks move [RequestRow.bytesSent] here only. + */ +class RequestIndex { + companion object { + val shared = RequestIndex() + } + + private val rows = ConcurrentHashMap() + + fun replaceAll(all: Collection) { + rows.clear() + all.forEach { rows[it.id] = it } + } + + /** + * A save of a running entry keeps the larger bytesSent. The stored value + * lags the in-memory progress, and a save for an attempt must not move the + * row backwards. + */ + fun put(row: RequestRow) { + rows.compute(row.id) { _, old -> + if (old != null && old.state == row.state && row.state == EntryState.RUNNING.wire && + old.bytesSent > row.bytesSent && old.bytesSent <= row.totalBytes + ) row.withBytes(old.bytesSent) else row + } + } + + fun remove(id: String) { + rows.remove(id) + } + + /** Oldest first, then by id. */ + fun snapshot(): List = + rows.values.sortedWith(compareBy { it.createdAt }.thenBy { it.id }) + + /** A progress tick. A missing id is ignored. */ + fun setBytes(id: String, bytesSent: Long) { + rows.computeIfPresent(id) { _, row -> row.withBytes(bytesSent) } + } +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/RetryClassifier.kt b/android/src/main/java/ai/openspace/backgroundupload/RetryClassifier.kt new file mode 100644 index 00000000..d570ec61 --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/RetryClassifier.kt @@ -0,0 +1,85 @@ +package ai.openspace.backgroundupload + +import java.io.IOException +import kotlin.math.min +import kotlin.random.Random + +/** + * The retry table (plan section 6.1), as pure functions. + * + * | Response or failure | Verdict | + * | 2xx, or an accept rule matches | Accepted | + * | 401, 403 | Auth (park) | + * | 408, 429, 5xx | Transient | + * | other 4xx in `exempt` (default [404]) | Transient | + * | other 4xx | Terminal http | + * | any other status (1xx, a final 3xx) | Terminal http | + * | IOException, payload file missing | Terminal file | + * | IOException | Transient | + * | anything else | Terminal unknown | + */ +object RetryClassifier { + + sealed class Verdict { + object Accepted : Verdict() + object Transient : Verdict() + object Auth : Verdict() + data class Terminal(val errorKind: String, val message: String) : Verdict() + } + + data class Policy(val baseMs: Long, val maxMs: Long, val jitter: Double, val exempt: List) + + /** A wait up to this long happens inside the worker. A longer one releases the worker. */ + const val IN_WORKER_BACKOFF_MAX_MS = 30_000L + + fun policy(defaults: RetryDefaults, override: RetryOverride?): Policy = Policy( + baseMs = override?.baseMs ?: defaults.baseMs, + maxMs = override?.maxMs ?: defaults.maxMs, + jitter = override?.jitter ?: defaults.jitter, + exempt = override?.exempt ?: defaults.exempt, + ) + + fun classifyResponse( + code: Int, + body: String?, + accept: List, + exempt: List, + ): Verdict = when { + UploadOutcome.isAccepted(code, body, accept) -> Verdict.Accepted + code == 401 || code == 403 -> Verdict.Auth + code == 408 || code == 429 || code in 500..599 -> Verdict.Transient + code in 400..499 && code in exempt -> Verdict.Transient + else -> Verdict.Terminal("http", "HTTP $code") + } + + /** A CancellationException is never classified; the caller rethrows it first. */ + fun classifyFailure(error: Throwable, fileExists: Boolean): Verdict = when { + error is IOException && !fileExists -> + Verdict.Terminal("file", "request body file is missing: ${error.message ?: error.javaClass.simpleName}") + error is IOException -> Verdict.Transient + else -> Verdict.Terminal("unknown", error.message ?: error.javaClass.simpleName) + } + + /** + * The live attempt's errorKind for a transport failure, from its + * [classifyFailure] verdict: file, unknown, or network for a transient one. + */ + fun failureKind(verdict: Verdict): String = (verdict as? Verdict.Terminal)?.errorKind ?: "network" + + fun isExpired(now: Long, expiresAt: Long) = now >= expiresAt + + /** + * base * 2^(streak-1), capped at maxMs, then spread by ± jitter and capped + * again. streak 1 is baseMs. + */ + fun backoffMs(policy: Policy, streak: Int, random: Random = Random.Default): Long { + val exponent = (streak.coerceAtLeast(1) - 1).coerceAtMost(40) + val raw = min(policy.baseMs.toDouble() * Math.pow(2.0, exponent.toDouble()), policy.maxMs.toDouble()) + val jitter = policy.jitter.coerceIn(0.0, 1.0) + val spread = raw * (1.0 + jitter * (2.0 * random.nextDouble() - 1.0)) + return min(spread, policy.maxMs.toDouble()).toLong().coerceAtLeast(0L) + } + + /** The wake time, never later than expiresAt, so an entry expires on time. */ + fun nextAttemptAt(now: Long, backoffMs: Long, expiresAt: Long): Long = min(now + backoffMs, expiresAt) +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/Scheduler.kt b/android/src/main/java/ai/openspace/backgroundupload/Scheduler.kt new file mode 100644 index 00000000..872dd2a3 --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/Scheduler.kt @@ -0,0 +1,111 @@ +package ai.openspace.backgroundupload + +import android.content.Context +import androidx.work.ExistingWorkPolicy +import androidx.work.OneTimeWorkRequest +import androidx.work.OneTimeWorkRequestBuilder +import androidx.work.WorkInfo +import androidx.work.WorkManager +import androidx.work.workDataOf +import java.util.concurrent.TimeUnit + +/** + * Starts and stops worker runs for entries. A run only carries the entry id; + * the worker reads everything else from the store. So an extra run is always + * harmless: it finds nothing to do and exits. + */ +interface WorkScheduler { + /** A run as soon as possible, on the entry's main chain. */ + fun schedule(entry: QueueEntry) + + /** + * A run at [at] under a second unique name. Used for long backoffs and for + * the expiry of a parked entry. [replace] false keeps a wake that already + * exists (the boot sweep). + */ + fun scheduleWake(entry: QueueEntry, at: Long, replace: Boolean) + + /** Cancels the main chain and the wake. */ + fun cancel(id: String) +} + +/** + * WorkManager unique work per entry id, APPEND_OR_REPLACE, as v9. + * + * Why APPEND_OR_REPLACE: a worker settles before doWork returns, so a same-id + * enqueue can arrive while the row is still RUNNING. KEEP would drop it and + * REPLACE would kill the running worker. APPEND runs it after; OR_REPLACE + * starts a fresh chain after a CANCELLED or FAILED one. Workers always return + * success, because WorkManager fails the dependents of a FAILED row without a + * run. + * + * Long waits do not go on the main chain. A delayed row there would hold + * back every later "run now" appended behind it. They use the wake name + * (`#wake`) with an initial delay instead. + * + * No WorkManager Constraints: connectivity and wifi-only are checked inside + * the worker, as v9, because the constraint path was unreliable. + */ +class WorkManagerScheduler(context: Context) : WorkScheduler { + companion object { + /** v9 rows carry the tag "RNFileUploader"; this one is new so the v9 cancel does not touch v10 work. */ + const val WORK_TAG = "RNFileUploader.v10" + const val ID_TAG_PREFIX = "RNFileUploaderId:" + /** A string literal, because WorkManager persists it across builds. */ + const val ENTRY_ID_KEY = "entryId" + const val V9_WORK_TAG = "RNFileUploader" + + fun wakeName(id: String) = "$id#wake" + + /** Milliseconds from [now] until [at]; never negative. */ + fun initialDelayMs(at: Long?, now: Long): Long = if (at == null) 0L else (at - now).coerceAtLeast(0L) + + /** An unfinished row that is not RUNNING: a queued run that did not start yet. */ + fun hasQueuedSuccessor(states: List): Boolean = + states.any { !it.isFinished && it != WorkInfo.State.RUNNING } + } + + private val workManager = WorkManager.getInstance(context) + + override fun schedule(entry: QueueEntry) { + // A queued successor already guarantees a run after the current one. + val states = workManager.getWorkInfosForUniqueWork(entry.id).get().map { it.state } + if (hasQueuedSuccessor(states)) return + workManager + .beginUniqueWork(entry.id, ExistingWorkPolicy.APPEND_OR_REPLACE, request(entry, 0L)) + .enqueue() + } + + override fun scheduleWake(entry: QueueEntry, at: Long, replace: Boolean) { + val delay = initialDelayMs(at, System.currentTimeMillis()) + workManager.enqueueUniqueWork( + wakeName(entry.id), + if (replace) ExistingWorkPolicy.REPLACE else ExistingWorkPolicy.KEEP, + request(entry, delay), + ) + } + + override fun cancel(id: String) { + workManager.cancelUniqueWork(id) + workManager.cancelUniqueWork(wakeName(id)) + } + + /** First v10 launch: the v9 rows. Their workers are gone. */ + fun cancelV9Work() { + workManager.cancelAllWorkByTag(V9_WORK_TAG) + } + + private fun request(entry: QueueEntry, delayMs: Long): OneTimeWorkRequest { + val builder = if (entry.body?.kind == StagedBody.CHUNKED) { + OneTimeWorkRequestBuilder() + } else { + OneTimeWorkRequestBuilder() + } + return builder + .addTag(WORK_TAG) + .addTag(ID_TAG_PREFIX + entry.id) + .setInputData(workDataOf(ENTRY_ID_KEY to entry.id)) + .setInitialDelay(delayMs, TimeUnit.MILLISECONDS) + .build() + } +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/TransferHost.kt b/android/src/main/java/ai/openspace/backgroundupload/TransferHost.kt new file mode 100644 index 00000000..d7556a3d --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/TransferHost.kt @@ -0,0 +1,34 @@ +package ai.openspace.backgroundupload + +/** + * What a run needs from the platform: time, the network, progress, the + * notification, and live attempt events. [EntryWorker] is the real one. The + * JVM tests pass a fake with a scripted [send] and a manual clock, so every + * branch of [EntryRun] runs with no device. + */ +internal interface TransferHost { + fun now(): Long + + suspend fun sleep(ms: Long) + + /** One request through the library-wide cap of 4. Throws on a transport failure. */ + suspend fun send(request: TransferRequest, onProgress: (Long) -> Unit): UploadResponse + + /** The network state for the queue's wifi-only setting. The notification shows it. */ + fun connectivity(wifiOnly: Boolean): Connectivity + + /** Foreground mode for an entry that shows the notification. Never throws. */ + suspend fun foreground(entry: QueueEntry) + + fun progressStarted(id: String, total: Long, sent: Long) + + fun progress(id: String, sent: Long, total: Long) + + /** The trailing progress event, then the progress state is dropped. */ + fun progressEnded(id: String, completed: Boolean) + + fun attempt(event: AttemptEvent) + + /** Whether the system stopped this run at its time limit (JobScheduler, about 10 minutes). */ + fun stoppedByTimeout(): Boolean +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/Upload.kt b/android/src/main/java/ai/openspace/backgroundupload/Upload.kt deleted file mode 100644 index 3df469b0..00000000 --- a/android/src/main/java/ai/openspace/backgroundupload/Upload.kt +++ /dev/null @@ -1,107 +0,0 @@ -package ai.openspace.backgroundupload - -import com.facebook.react.bridge.ReadableArray -import com.facebook.react.bridge.ReadableMap -import java.util.UUID - -// Data model of a single upload -// Can be created from RN's ReadableMap -// Can be used for JSON deserialization -data class Upload( - val id: String, - val url: String, - val path: String, - val method: String, - val wifiOnly: Boolean, - // Non-2xx responses to treat as a successful completion (for example, a 409 - // whose body marks an expected duplicate). Every other non-2xx response is a - // terminal http error. The list is empty by default. - val accept: List, - val headers: Map, - /** - * Suppresses the progress notification for this upload. - * - * The notification is not decoration: posting one is what lets the worker run - * in foreground mode, which is how a long-running worker survives Doze and - * memory pressure. A suppressed upload is an ordinary background worker, so - * the OS may defer it or stop it mid-flight for WorkManager to re-run later. - * Suppress only payloads small enough that a restart costs nothing. - * - * An opt-out rather than an opt-in so that absence means "notify": this model - * is serialized into WorkManager's database, and a job enqueued by a build - * that predates the option can be replayed by a build that has it. - */ - val noNotification: Boolean, -) { - // v8 persisted `acceptStatus: List` where v9 persists `accept`. This is - // not a constructor parameter. It exists only so Gson can surface the legacy - // field to [normalized]. It is null, and thus never serialized, for every - // upload that this build creates. - private val acceptStatus: List? = null - - val showsNotification get() = !noNotification - - /** - * Gson does not use the constructor. Thus a WorkManager job that an older - * build enqueued can give this worker an object whose non-null fields are - * null. A v8 job carries `acceptStatus` and no `accept`. That NPEs the first - * time the worker touches [accept], after the file has fully transmitted, - * and the re-runs then re-send the whole file. This is the same - * normalize-after-fromJson pattern as ChunkedManifestStore.validated(): map - * the legacy statuses to rules, default what is absent, and give the worker - * an object that is safe to use. - */ - @Suppress("SENSELESS_COMPARISON", "USELESS_ELVIS") - fun normalized(): Upload = Upload( - id = id, - url = url, - path = path, - method = method ?: "POST", - wifiOnly = wifiOnly, - accept = accept - ?: acceptStatus?.map { UploadOutcome.AcceptRule(it) } - ?: emptyList(), - headers = headers ?: emptyMap(), - noNotification = noNotification, - ) - - class MissingOptionException(optionName: String) : - IllegalArgumentException("Missing '$optionName'") - - companion object { - fun fromReadableMap(map: ReadableMap) = Upload( - id = map.getString(Upload::id.name) ?: UUID.randomUUID().toString(), - url = map.getString(Upload::url.name) ?: throw MissingOptionException(Upload::url.name), - path = map.getString(Upload::path.name) ?: throw MissingOptionException(Upload::path.name), - method = map.getString(Upload::method.name) ?: "POST", - wifiOnly = if (map.hasKey(Upload::wifiOnly.name)) map.getBoolean(Upload::wifiOnly.name) else false, - accept = parseAcceptRules(map.getArray(Upload::accept.name)), - headers = parseHeaderMap(map.getMap(Upload::headers.name)), - // The notification text and identity are not per-upload options. The - // worker reads them from the NotificationConfig that configure() saved. - noNotification = if (map.hasKey(Upload::noNotification.name)) - map.getBoolean(Upload::noNotification.name) else false, - ) - } -} - -// Upload and ChunkedManifest share this: one accept-rules shape, one parser. -internal fun parseAcceptRules(arr: ReadableArray?): List { - if (arr == null) return listOf() - return (0 until arr.size()).mapNotNull { i -> - val rule = arr.getMap(i) ?: return@mapNotNull null - UploadOutcome.AcceptRule( - status = rule.getInt("status"), - bodyIncludes = if (rule.hasKey("bodyIncludes")) rule.getString("bodyIncludes") else null, - ) - } -} - -internal fun parseHeaderMap(headers: ReadableMap?): Map { - if (headers == null) return mapOf() - val map = mutableMapOf() - for (entry in headers.entryIterator) { - map[entry.key] = entry.value.toString() - } - return map -} diff --git a/android/src/main/java/ai/openspace/backgroundupload/UploadOutcome.kt b/android/src/main/java/ai/openspace/backgroundupload/UploadOutcome.kt index f4fb2763..f1edc40a 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/UploadOutcome.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/UploadOutcome.kt @@ -1,7 +1,5 @@ package ai.openspace.backgroundupload -import java.io.IOException - // Pure classification of terminal upload outcomes. Kept free of Android/React // types so it can be unit-tested on a plain JVM — this is the highest-consequence // logic in the uploader (it decides success vs failure), so it's covered directly. @@ -11,8 +9,7 @@ object UploadOutcome { * A non-2xx response to treat as success. `bodyIncludes` narrows the rule by * a response-body substring. This is necessary when one status has several * meanings, and only the message shows the difference (our backend's 409). - * Gson persists it inside [Upload] and [ChunkedManifest]; see - * consumer-rules.pro. + * Gson persists it inside [Descriptor]; see consumer-rules.pro. */ data class AcceptRule( val status: Int, @@ -27,14 +24,4 @@ object UploadOutcome { rule.status == code && (rule.bodyIncludes == null || body?.contains(rule.bodyIncludes) == true) } - - // Classify a thrown error into a stable kind for the JS layer. `fileExists` - // is passed in (not read here) to keep this pure; callers should default it to - // true when the existence check itself fails, so a flaky file probe reads as a - // retryable network error rather than a terminal "file gone". - fun errorKind(error: Throwable, fileExists: Boolean): String = when { - error is IOException && !fileExists -> "file" - error is IOException -> "network" - else -> "unknown" - } } diff --git a/android/src/main/java/ai/openspace/backgroundupload/UploadUtils.kt b/android/src/main/java/ai/openspace/backgroundupload/UploadUtils.kt index 0e48da98..154cac08 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/UploadUtils.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/UploadUtils.kt @@ -9,6 +9,7 @@ import okhttp3.OkHttpClient import okhttp3.Request import okhttp3.RequestBody import okhttp3.RequestBody.Companion.asRequestBody +import okhttp3.RequestBody.Companion.toRequestBody import okhttp3.Response import okio.Buffer import okio.BufferedSink @@ -19,49 +20,79 @@ import java.io.IOException import java.io.RandomAccessFile import kotlin.coroutines.resumeWithException -// Throttling interval of progress reports +// Throttling interval of the raw progress callback. ProgressThrottle limits +// the JS events above this. private const val PROGRESS_INTERVAL = 500 // milliseconds private const val RANGE_COPY_BUFFER = 64 * 1024 +/** [truncated] when the body passed [BodyCap.SETTLED_MAX_BYTES] and the rest was not read. */ data class UploadResponse( val code: Int, val body: String, - val headers: Map + val headers: Map, + val truncated: Boolean = false, ) -// make an upload request using okhttp -suspend fun okhttpUpload( +/** One request as the worker sends it. [body] is null only for GET and DELETE with no body. */ +data class TransferRequest( + val url: String, + val method: String, + val headers: Map, + val body: RequestBody?, +) + +/** Sends one request and reports bytes written. The headers are sent as they are. */ +suspend fun okhttpSend( client: OkHttpClient, - upload: Upload, - file: File, - onProgress: (Long) -> Unit + request: TransferRequest, + onProgress: (Long) -> Unit, ): UploadResponse { - val request = Request.Builder() - .url(upload.url) - .headers(upload.headers.toHeaders()) - .method(upload.method, withProgressListener(file.asRequestBody(), throttled(onProgress))) + val body = request.body?.let { withProgressListener(it, throttled(onProgress)) } + val built = Request.Builder() + .url(request.url) + .headers(request.headers.toHeaders()) + .method(request.method, body) .build() - return awaitResponse(client, request) + return awaitResponse(client, built) } +// Every body has a null content type, so OkHttp does not invent a +// Content-Type. The header on the request (the caller's, or the one staging +// set) is sent unchanged. + +/** A whole staged file. */ +fun fileBody(file: File): RequestBody = file.asRequestBody(null as MediaType?) + +/** A zero-length body for a POST, PUT, or PATCH with no body. OkHttp requires one. */ +fun emptyBody(): RequestBody = ByteArray(0).toRequestBody(null) + /** - * PUTs one byte range of the source file: a chunked part. It streams straight - * from disk, with no temporary chunk file. The headers are the consumer's, - * unchanged. The library adds nothing, per the design's protocol-as-data rule. + * The file bytes [start, end) as a request body: a chunked part. It streams + * from disk with no temporary chunk file. A RandomAccessFile is opened fresh + * on every writeTo, because OkHttp can replay a body (a connection-level + * retry), and a one-shot stream would then send truncated data. */ -suspend fun okhttpUploadPart( - client: OkHttpClient, - part: ChunkedManifest.Part, - file: File, - onProgress: (Long) -> Unit -): UploadResponse { - val request = Request.Builder() - .url(part.url) - .headers(part.headers.toHeaders()) - .put(withProgressListener(rangeRequestBody(file, part.start, part.end), throttled(onProgress))) - .build() - return awaitResponse(client, request) +fun rangeRequestBody(file: File, start: Long, end: Long): RequestBody = object : RequestBody() { + override fun contentType(): MediaType? = null + + override fun contentLength() = end - start + + override fun writeTo(sink: BufferedSink) { + RandomAccessFile(file, "r").use { raf -> + raf.seek(start) + val buffer = ByteArray(RANGE_COPY_BUFFER) + var remaining = end - start + while (remaining > 0L) { + val read = raf.read(buffer, 0, minOf(remaining, buffer.size.toLong()).toInt()) + if (read < 0) throw IOException( + "source file ended before part range [$start, $end): ${file.path}", + ) + sink.write(buffer, 0, read) + remaining -= read + } + } + } } private suspend fun awaitResponse(client: OkHttpClient, request: Request): UploadResponse = @@ -73,18 +104,25 @@ private suspend fun awaitResponse(client: OkHttpClient, request: Request): Uploa continuation.resumeWithException(e) override fun onResponse(call: Call, response: Response) { - val result = response.use { res -> // close the response asap - UploadResponse( - res.code, - // The body, unchanged: an empty body stays empty. A substituted - // HTTP reason phrase would make accept `bodyIncludes` rules match - // text that the server never sent. iOS also reports the body - // as-is. - res.body?.string().orEmpty(), - res.headers.toMultimap().mapValues { it.value.joinToString(", ") } - ) + val result = try { + response.use { res -> // close the response asap + // The body unchanged: an empty body stays empty. A substituted + // reason phrase would make accept `bodyIncludes` rules match text + // the server never sent. The cap applies while it streams in. + val body = res.body?.let { + BodyCap.read(it.source(), BodyCap.SETTLED_MAX_BYTES, it.contentType()?.charset() ?: Charsets.UTF_8) + } + UploadResponse( + res.code, + body?.text.orEmpty(), + res.headers.toMultimap().mapValues { it.value.joinToString(", ") }, + body?.truncated ?: false, + ) + } + } catch (e: IOException) { + continuation.resumeWithException(e) + return } - continuation.resumeWith(Result.success(result)) } }) @@ -101,38 +139,7 @@ private fun throttled(onProgress: (Long) -> Unit): (Long) -> Unit { } } -/** - * Streams the file bytes [start, end) as a request body. A RandomAccessFile - * backs it, opened fresh on every writeTo call. OkHttp can replay a body (for - * example, after a connection-level retry), and a one-shot stream would then - * send truncated data silently. - */ -private fun rangeRequestBody(file: File, start: Long, end: Long) = object : RequestBody() { - // Null, so no Content-Type is invented. The consumer's header is already on - // the request, unchanged. - override fun contentType(): MediaType? = null - - override fun contentLength() = end - start - - override fun writeTo(sink: BufferedSink) { - RandomAccessFile(file, "r").use { raf -> - raf.seek(start) - val buffer = ByteArray(RANGE_COPY_BUFFER) - var remaining = end - start - while (remaining > 0L) { - val read = raf.read(buffer, 0, minOf(remaining, buffer.size.toLong()).toInt()) - if (read < 0) throw IOException( - "source file ended before part range [$start, $end): ${file.path}", - ) - sink.write(buffer, 0, read) - remaining -= read - } - } - } -} - -// create a request body that allows us to listen to progress. -// okhttp has no built-in way of reporting progress +// OkHttp has no built-in progress report, so the body counts bytes as it writes. private fun withProgressListener( body: RequestBody, onProgress: (Long) -> Unit diff --git a/android/src/main/java/ai/openspace/backgroundupload/UploadWorker.kt b/android/src/main/java/ai/openspace/backgroundupload/UploadWorker.kt index 1f0c63e8..a49f7548 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/UploadWorker.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/UploadWorker.kt @@ -1,286 +1,64 @@ package ai.openspace.backgroundupload -import android.app.NotificationManager import android.content.Context -import androidx.work.CoroutineWorker -import androidx.work.ForegroundInfo import androidx.work.WorkerParameters -import com.google.gson.Gson -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.delay -import kotlinx.coroutines.withContext +import okhttp3.RequestBody import java.io.File -import java.io.IOException -import java.net.UnknownHostException -import java.util.UUID -import java.util.concurrent.TimeUnit -// Retry delay -private val RETRY_DELAY = TimeUnit.SECONDS.toMillis(10L) - -// The retry budget for errors that count (see checkRetry). A connectivity gap -// or flaky-network IO resets the budget. The retry policy is internal to the -// library. It is not an option. -private const val MAX_RETRIES = 5 - -class UploadWorker(private val context: Context, params: WorkerParameters) : - CoroutineWorker(context, params) { - - companion object { - /** - * Key for the serialized [Upload] in the worker's input data. - * - * A string literal on purpose. This key is persisted in WorkManager's - * database, so the build that runs a job may not be the build that enqueued - * it — a key derived from a symbol name (an enum constant, a property) breaks - * the moment R8 renames it or someone refactors, and the failure looks like - * "No Params" on a job that was queued perfectly well by the previous version. - */ - const val PARAMS_KEY = "params" - } - - private lateinit var upload: Upload - // configure() saved this. The worker can read it when WorkManager relaunched - // the worker with no JS. It is lazy, so the SharedPreferences read occurs on - // the worker's IO dispatcher, not at construction. - private val config by lazy { NotificationConfig.load(context) } - private var retries = 0 - private var connectivity = Connectivity.Ok - private val notificationManager = - context.getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager - - override suspend fun doWork(): Result = withContext(Dispatchers.IO) { - // Retrieve the upload. If this throws errors, error reporting won't work. - // However, the only way it has errors is the implementation is incorrect, - // which can be caught in development - val paramsJson = inputData.getString(PARAMS_KEY) ?: throw Throwable("No Params") - // normalized(): an older build can have enqueued this job, and its JSON - // shape can make non-null fields null (Gson does not use the constructor). - // See Upload.normalized. - upload = Gson().fromJson(paramsJson, Upload::class.java).normalized() - - // initialization, errors thrown here won't be retried - try { - // An upload that suppresses its notification cannot enter foreground mode, - // since the notification is the foreground service's own notification. - if (upload.showsNotification) { - // The foreground notification needs a channel to exist first, or posting - // it silently fails and setForeground can crash on newer Android. - ensureNotificationChannel(notificationManager, config) - // `setForeground` is recommended for long-running workers. - // Foreground mode helps prioritize the worker, reducing the risk - // of it being killed during low memory or Doze/App Standby situations. - // ⚠️ This should be called in the foreground - setForeground(getForegroundInfo()) - } - } catch (error: Throwable) { - if (!isForegroundStartDenied(error)) { - if (!checkAndHandleCancellation()) handleError(error) - throw error - } - // The app is in the background on API 31+ (see isForegroundStartDenied). - // Continue the upload without foreground priority. Do not fail an upload - // that can run. +/** The WorkManager class for a single-body entry. The run is [EntryWorker]'s. */ +class UploadWorker(context: Context, params: WorkerParameters) : EntryWorker(context, params) + +/** + * One request with one body: none, JSON, multipart, or a copied file. One + * attempt at a time ([EntryRun.attempt]), until a verdict ends it: + * accepted → Completed; auth → re-issue (newer headers) or park; + * transient → back off (short: here; long: release); terminal → Failed. + */ +internal class SimpleTransfer(private val run: EntryRun) { + + suspend fun run(start: QueueEntry): Settlement { + val d0 = start.descriptor!! + val file = run.store.bodyFile(start) + // The payload probe: a staged body that is gone can never be sent. + if (file != null && !file.exists()) { + return Settlement.Failed("file", "the staged request body is missing", null, null, d0.reportUrl, d0.method) } + val total = start.body?.totalBytes ?: 0L + run.progressStarted(total, 0L) + var streak = start.backoffStreak - - // Complex work, errors thrown below here trigger retry. - // We don't let WorkManager manage retries and network constraints as it's very buggy. - // i.e. we'd occasionally get BackgroundServiceStartNotAllowedException, - // or ForegroundServiceStartNotAllowedException, or "isStopped" gets set to "true" - // for no reason - var isRetried = false while (true) { - try { - // - "delay" should be within the "try" block to account for worker cancellation, - // which cancels the delay immediately and throws CancellationException. - // - Linear backoff instead of exponential. One reason for this is we retry on - // invalid connections. Exponential will take too long. - // - We retry only transport failures here (no response). An HTTP - // response, 4xx and 5xx included, is terminal at this layer. - // handleResponse classifies it (a 2xx or an accept rule -> completed, - // else an http error), and the worker returns without a retry. A - // response-code retry policy is the JS queue's job. This matches the - // iOS behavior. - if (isRetried) delay(RETRY_DELAY) - isRetried = true - - val response = upload() ?: continue - handleResponse(response) - return@withContext Result.success() - } catch (error: Throwable) { - if (checkAndHandleCancellation()) throw error - if (checkRetry(error)) continue - handleError(error) - throw error - } - } - - // This should never happen. Only here to satisfy the type check - return@withContext Result.failure() - } - - private suspend fun upload(): UploadResponse? { - val file = File(upload.path) - val size = file.length() - - // Register progress asap so the total progress is accurate - // This needs to happen before the semaphore wait - UploadProgress.add(upload.id, size) - - // Don't bother to run on an invalid network - if (!validateAndReportConnectivity()) return null - - // wait for its turn to run - transferSemaphore.acquire() - - try { - return okhttpUpload(uploadHttpClient, upload, file) { progress -> - handleProgress(progress, size) - } - } catch (error: Throwable) { - // reset progress on error - UploadProgress.set(upload.id, 0L) - // pass the error to upper layer for retry decision - throw error - } finally { - transferSemaphore.release() - } - } - - private fun handleProgress(bytesSentTotal: Long, fileSize: Long) { - UploadProgress.set(upload.id, bytesSentTotal) - EventReporter.progress(upload.id, bytesSentTotal, fileSize) - updateNotification() - } - - // Redraws the progress notification. A no-op for a suppressed upload — the - // worker never posted one, and `notify` would create it outside foreground mode. - private fun updateNotification() { - if (!upload.showsNotification) return - notificationManager.notify( - config.systemNotificationId, - buildUploadNotification(context, config, connectivity), - ) - } - - // An HTTP response came back. It is "completed" only for a 2xx or a matching - // accept rule (axios validateStatus semantics: a 400 is an error, not a - // completion). Every other response is a terminal http error that carries the - // full response. In both cases the request finished, so the worker does not - // retry. - private fun handleResponse(response: UploadResponse) { - UploadProgress.complete(upload.id) - val accepted = UploadOutcome.isAccepted(response.code, response.body, upload.accept) - val (body, truncated) = EventJournal.capBody(response.body) - journalAndEmit( - EventJournal.Entry( - eventId = UUID.randomUUID().toString(), - uploadId = upload.id, - type = if (accepted) "completed" else "error", - timestamp = System.currentTimeMillis(), - responseCode = response.code, - responseBody = body, - responseBodyTruncated = truncated, - responseHeaders = response.headers, - errorKind = if (accepted) null else "http", - error = if (accepted) null else "HTTP ${response.code}", + val latest = run.ops.latest(run.entryId, run.generation) + if (RetryClassifier.isExpired(run.host.now(), latest.expiresAt)) throw EntryRun.ExpiredException() + run.waitForNetwork() + + val a = run.attempt( + partIndex = null, + body = { d, _ -> requestBody(file, d.method) }, + onProgress = { sent -> run.reportProgress(sent, total) }, + fileExists = { file == null || file.exists() }, ) - ) - } - - private fun handleError(error: Throwable) { - UploadProgress.remove(upload.id) - // Default fileExists=true so a failed existence probe reads as network, not file. - val fileExists = runCatching { File(upload.path).exists() }.getOrDefault(true) - journalAndEmit( - EventJournal.Entry( - eventId = UUID.randomUUID().toString(), - uploadId = upload.id, - type = "error", - timestamp = System.currentTimeMillis(), - error = error.message ?: "Unknown exception", - errorKind = UploadOutcome.errorKind(error, fileExists), - ) - ) - } - - // Check if cancelled by user or new worker with same ID - // Worker won't rerun, perform teardown - private fun checkAndHandleCancellation(): Boolean { - if (!isStopped) return false - - UploadProgress.remove(upload.id) - - // Only a user cancel is terminal, so only a user cancel is journaled. - // - // WorkManager decides whether to reschedule BEFORE it stops the worker, and - // it ignores the Result we return. cancelUniqueWork marks the row CANCELLED - // first, so a user cancel is genuinely the end. A system stop — a - // foreground-service timeout, quota, or memory pressure — leaves the row - // RUNNING and WorkManager re-runs this same upload. Journaling a terminal - // `cancelled` there would durably tell JS the upload was dead while it was in - // fact about to be retried, so the consumer would settle the transfer and the - // retry would land as a duplicate on the server. - // - // Emitting nothing is the honest answer for a system stop: the upload is - // still in flight as far as anyone should be concerned. If WorkManager ever - // declines to reschedule, `getAllUploads()` is how a consumer notices. - if (!UserCancellations.consume(upload.id)) return true - - journalAndEmit( - EventJournal.Entry( - eventId = UUID.randomUUID().toString(), - uploadId = upload.id, - type = "cancelled", - timestamp = System.currentTimeMillis(), - cancelReason = "user", - ) - ) - return true - } - - private fun journalAndEmit(entry: EventJournal.Entry) = - EventReporter.journalAndEmit(context, entry) - - /** @return whether to retry */ - private fun checkRetry(error: Throwable): Boolean { - var unlimitedRetry = false - - // Error was thrown due to unmet network preferences. - // Also happens every time you switch from one network to any other - if (!validateAndReportConnectivity()) unlimitedRetry = true - // Due to the flaky nature of networking, sometimes the network is - // valid but the URL is still inaccessible, so keep waiting until - // the URL is accessible - else if (error is UnknownHostException) unlimitedRetry = true - // There are many IOExceptions that only differ by messages, - // so we can't check using class, but theoretically, - // only the one caused by file not existing should stop the retry. - // The rest should be related to flaky network or flaky file I/O, - // where we can retry without limit. - else if (error is IOException) { - try { - if (!File(upload.path).exists()) return false - unlimitedRetry = true - } catch (_: Throwable) { - // read file error, can't do anything but retry - unlimitedRetry = false + when (val r = a.result) { + is EntryRun.AttemptResult.Accepted -> return Settlement.Completed(r.response, a.url, a.method) + is EntryRun.AttemptResult.Auth -> { + if (!r.reissue) throw EntryRun.ParkException(r.headerGeneration) + streak = 0 // updateHeaders() landed while this attempt was in flight: re-issue now. + } + EntryRun.AttemptResult.Transient -> { + run.reportProgress(0L, total) + streak++ + run.backoffOrRelease(a.policy, streak, a.entry.expiresAt) + } + is EntryRun.AttemptResult.Terminal -> + return Settlement.Failed(r.errorKind, r.message, r.response, null, a.url, a.method, bytesSent = run.liveBytes) } } - - retries = if (unlimitedRetry) 0 else retries + 1 - return retries <= MAX_RETRIES } - // Checks connection and alerts connection issues - private fun validateAndReportConnectivity(): Boolean { - this.connectivity = validateConnectivity(context, upload.wifiOnly) - // alert connectivity mode - updateNotification() - return this.connectivity == Connectivity.Ok + // OkHttp needs a body for POST, PUT, and PATCH, and forbids one for GET. + private fun requestBody(file: File?, method: String): RequestBody? = when { + file != null -> fileBody(file) + method == "GET" || method == "DELETE" -> null + else -> emptyBody() } - - override suspend fun getForegroundInfo(): ForegroundInfo = - uploadForegroundInfo(config, buildUploadNotification(context, config, connectivity)) } diff --git a/android/src/main/java/ai/openspace/backgroundupload/UploaderModule.kt b/android/src/main/java/ai/openspace/backgroundupload/UploaderModule.kt index cbd26a85..83ad4505 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/UploaderModule.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/UploaderModule.kt @@ -1,11 +1,5 @@ package ai.openspace.backgroundupload -import android.util.Log -import androidx.work.ExistingWorkPolicy -import androidx.work.OneTimeWorkRequestBuilder -import androidx.work.WorkInfo -import androidx.work.WorkManager -import androidx.work.workDataOf import com.facebook.react.bridge.Arguments import com.facebook.react.bridge.Promise import com.facebook.react.bridge.ReactApplicationContext @@ -13,16 +7,21 @@ import com.facebook.react.bridge.ReadableArray import com.facebook.react.bridge.ReadableMap import com.facebook.react.bridge.WritableArray import com.facebook.react.bridge.WritableMap -import com.google.gson.Gson -import java.io.File -import java.nio.file.Files -import java.nio.file.StandardCopyOption - +import com.facebook.react.common.LifecycleState +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.Future +import java.util.concurrent.TimeUnit /** * TurboModule (New Architecture). [NativeRNFileUploaderSpec] is generated by * codegen from `src/NativeRNFileUploader.ts` into this same package (see - * `codegenConfig.android.javaPackageName` in package.json), so it needs no import. + * `codegenConfig.android.javaPackageName` in package.json). + * + * A thin shell over [QueueController]. Every promise method runs on one + * single-thread executor ([queueExecutor]), because enqueue copies files and + * every method touches the disk. The promise resolves from that executor. + * getRequests() is synchronous and reads the in-memory index only. */ class UploaderModule(context: ReactApplicationContext) : NativeRNFileUploaderSpec(context) { @@ -30,359 +29,186 @@ class UploaderModule(context: ReactApplicationContext) : companion object { const val NAME = "RNFileUploader" const val TAG = "RNFileUploader.UploaderModule" - const val WORKER_TAG = "RNFileUploader" - // WorkInfo exposes tags but not the unique-work name, so the upload id is - // also stored as a prefixed tag to recover it from a WorkInfo row. - const val ID_TAG_PREFIX = "RNFileUploaderId:" - // v10 slice 1 ships the JS layer alone. Every queue method rejects with - // this code until slice 2 builds the Android queue and executor. - const val E_NOT_IMPLEMENTED = "E_NOT_IMPLEMENTED" - - // The live module, so EventReporter can reach the codegen emitters — they are - // protected on the generated spec, so only this class may call them. Null - // whenever JS is absent (headless worker, mid-reload); terminal outcomes are - // journaled before being emitted, so a dropped live event is never lost. - // - // Volatile: written on the module-creation thread and read from the - // WorkManager worker, OkHttp callbacks and main, with no other barrier. + + /** The longest getRequests() waits for the first-launch import. */ + private const val IMPORT_WAIT_MS = 2_000L + + // The live module, so EventReporter can reach the codegen emitters: they + // are protected on the generated spec. Null whenever JS is absent + // (headless worker, mid-reload); outcomes are journaled before they are + // emitted, so a dropped live event is never lost. @Volatile var instance: UploaderModule? = null private set + + /** One thread for every module-side disk operation. It outlives a JS reload. */ + val queueExecutor: ExecutorService = Executors.newSingleThreadExecutor { r -> + Thread(r, "RNFileUploader.queue") + } } - private val workManager = WorkManager.getInstance(context) + private val store = QueueStore.get(context) + private val journal = EventJournal.get(context) + private val settings = QueueSettingsStore.get(context) + private val scheduler = WorkManagerScheduler(context) + private val controller = QueueController(store, journal, settings, EventReporter, scheduler) + + // The v9 import, then the v9 work cancel, then the boot sweep. + // getRequests() waits for the import only (file reads and row saves), so + // the first call after an upgrade already shows the legacy rows. The + // cancel opens the WorkManager database, so it runs after that wait. + private val importDone: Future init { instance = this + importDone = queueExecutor.submit { + runCatching { LegacyImport.runOnce(context, store) } + .onFailure { Diag.error("v9 import failed", it) } + .getOrDefault(true) // a failed import still cancels the v9 work + } + queueExecutor.execute { + if (runCatching { importDone.get() }.getOrDefault(false)) { + runCatching { scheduler.cancelV9Work() }.onFailure { Diag.error("v9 work cancel failed", it) } + } + runCatching { controller.sweep() }.onFailure { Diag.error("boot sweep failed", it) } + } } + // Set by invalidate(). A drain still queued on the executor then does not + // make this dead module the journal's listener. + @Volatile + private var invalidated = false + override fun invalidate() { - // A reload constructs the replacement before tearing this one down, so only - // clear the pointer when it still refers to us. + // A reload constructs the replacement before tearing this one down, so + // only clear the pointer (and the listener) when it still refers to us. + // The flag goes first: the drain reads it under the journal lock. + invalidated = true + journal.stopListening(this) if (instance === this) instance = null super.invalidate() } override fun getName(): String = NAME + /** Picks the progress throttle interval: 1 s in the foreground, 10 min otherwise. */ + fun isForeground(): Boolean = reactApplicationContext.lifecycleState == LifecycleState.RESUMED // MARK: - Event emission (called by EventReporter) - // The v9 workers still report through these. The v10 spec has no per-outcome - // emitters and a different progress shape ({ id, bytesSent, totalBytes }), so - // until slice 2 rewires the workers to onState/onProgress/onSettled, the live - // v9 payloads are dropped here. Terminal outcomes are journaled first, so - // nothing durable is lost. - fun emitProgressEvent(@Suppress("UNUSED_PARAMETER") params: WritableMap) = Unit + fun emitState(params: WritableMap) = safeEmit { emitOnState(params) } - fun emitCompletedEvent(@Suppress("UNUSED_PARAMETER") params: WritableMap) = Unit + fun emitProgress(params: WritableMap) = safeEmit { emitOnProgress(params) } - fun emitErrorEvent(@Suppress("UNUSED_PARAMETER") params: WritableMap) = Unit + fun emitAttempt(params: WritableMap) = safeEmit { emitOnAttempt(params) } - fun emitCancelledEvent(@Suppress("UNUSED_PARAMETER") params: WritableMap) = Unit + fun emitSettled(params: WritableMap) = safeEmit { emitOnSettled(params) } - fun emitNotificationEvent(params: WritableMap) = safeEmit { emitOnNotification(params) } + fun emitNotification(params: WritableMap) = safeEmit { emitOnNotification(params) } private inline fun safeEmit(emit: () -> Unit) { try { emit() } catch (exc: NullPointerException) { - // The generated spec's emitter callback is only installed when the C++ - // TurboModule is constructed, and is gone once the runtime tears down, so a - // null callback is expected in both gaps. It is ALSO null for the whole - // process on the old architecture, where this module still registers and its - // methods work but no event can ever be delivered — hence warn, not debug, - // so that case is diagnosable instead of silent. - Log.w(TAG, "live event dropped (no event emitter — New Architecture required)") + // The emitter callback exists only while the C++ TurboModule does, so a + // null callback is expected before setup and after teardown. It is also + // null for the whole process on the old architecture, so warn. + Diag.warn("live event dropped (no event emitter; New Architecture required)") } catch (exc: Throwable) { - // Anything else is a real bridging or payload failure worth seeing. - Log.e(TAG, "failed to emit live event", exc) + Diag.error("failed to emit live event", exc) } } - - /** - * Returns terminal events (completed/error/cancelled) that JS has not yet - * acknowledged, including ones that fired while JS was dead. Read these on - * startup, process them, then call ackEvents to remove them. - */ - override fun getUnacknowledgedEvents(promise: Promise) { - try { - val events = EventJournal.get(reactApplicationContext).unacknowledged() - val arr = Arguments.createArray() - events.forEach { arr.pushMap(it.toWritableMap()) } - promise.resolve(arr) - } catch (exc: Throwable) { - Log.e(TAG, exc.message, exc) - promise.reject(exc) - } - } - - - /** - * Removes journaled events by eventId once JS has processed them. Resolves - * void. Idempotent: an unknown id is ignored. - */ - override fun ackEvents(ids: ReadableArray, promise: Promise) { - try { - val eventIds = (0 until ids.size()).mapNotNull { ids.getString(it) } - val journal = EventJournal.get(reactApplicationContext) - // An acknowledged 'completed' is the ONE moment when a chunked upload's - // manifest and moved bytes may be deleted. Every other terminal keeps - // them for a resume. Resolve which uploads those are before the entries - // are removed. - val completedUploadIds = journal.unacknowledged() - .filter { it.type == "completed" && eventIds.contains(it.eventId) } - .map { it.uploadId } - journal.ack(eventIds) - releaseAckedCompletions( - completedUploadIds, - ChunkedManifestStore.get(reactApplicationContext), - ) { id -> workManager.cancelUniqueWork(id) } - promise.resolve(null) - } catch (exc: Throwable) { - Log.e(TAG, exc.message, exc) - promise.reject(exc) + // MARK: - Promise methods + + /** Runs [block] on the queue executor and settles [promise] with its value or a coded rejection. */ + private fun onQueue(promise: Promise, block: () -> Any?) { + queueExecutor.execute { + try { + promise.resolve(block()) + } catch (e: QueueException) { + promise.reject(e.code, e.message, e) + } catch (e: EntryParsing.InvalidEntryException) { + promise.reject(QueueException.E_INVALID, e.message, e) + } catch (e: Throwable) { + Diag.error("queue operation failed", e) + promise.reject(QueueException.E_STORAGE, e.message ?: e.javaClass.simpleName, e) + } } } - - /** - * Synchronous. The live rows of the v10 queue. Slice 2 serializes them from - * the in-memory index; until then the queue is empty. - */ - override fun getRequests(): WritableArray = Arguments.createArray() - - /** - * Saves the notification configuration (see [NotificationConfig]). Thus a - * worker that WorkManager relaunches with no JS can read it. Each call - * replaces the full configuration. An omitted field goes back to the library - * default. The v10 `lifetimeMs` and `retry` fields ride along in the same - * map; slice 2 persists them next to the queue. + * Saves the notification configuration (read by headless workers) and the + * retry defaults. Each call replaces the full configuration. lifetimeMs is + * not stored: JS already applied it to expiresAt. */ override fun configure(options: ReadableMap) { NotificationConfig.save(reactApplicationContext, NotificationConfig.fromReadableMap(options)) + @Suppress("UNCHECKED_CAST") + val retry = JsonBridge.valueOf(options, "retry") as? Map + val defaults = QueueSettingsStore.retryDefaults(retry) + queueExecutor.execute { + runCatching { controller.configureRetry(defaults) } + .onFailure { Diag.error("could not save the retry defaults", it) } + } } + override fun enqueue(entry: ReadableMap, promise: Promise) { + // Parse on the calling thread: the ReadableMap belongs to the bridge call. + val parsed = try { + EntryParsing.parse(entry) + } catch (e: EntryParsing.InvalidEntryException) { + promise.reject(QueueException.E_INVALID, e.message, e) + return + } + onQueue(promise) { controller.enqueue(parsed) } + } - // MARK: - v10 queue (stubs until slice 2) - - private fun notImplemented(promise: Promise, method: String) = - promise.reject(E_NOT_IMPLEMENTED, "RNFileUploader.$method: the Android queue is not built yet") - - /** Persists { id, key, vars, descriptor } and schedules it. Slice 2. */ - override fun enqueue(entry: ReadableMap, promise: Promise) = notImplemented(promise, "enqueue") - - override fun pause(promise: Promise) = notImplemented(promise, "pause") - - override fun resume(promise: Promise) = notImplemented(promise, "resume") - - override fun cancel(id: String, promise: Promise) = notImplemented(promise, "cancel") - - override fun setWifiOnly(enabled: Boolean, promise: Promise) = notImplemented(promise, "setWifiOnly") + override fun pause(promise: Promise) = onQueue(promise) { controller.pause(); null } - override fun updateHeaders(patch: ReadableMap, promise: Promise) = notImplemented(promise, "updateHeaders") + override fun resume(promise: Promise) = onQueue(promise) { controller.resume(); null } + override fun cancel(id: String, promise: Promise) = onQueue(promise) { controller.cancel(id); null } - // MARK: - v9 enqueue paths, kept for slice 2 to wire behind enqueue() + override fun setWifiOnly(enabled: Boolean, promise: Promise) = + onQueue(promise) { controller.setWifiOnly(enabled); null } - /** - * @return the id of the enqueued upload - */ - @Suppress("unused") - private fun enqueueUpload(options: ReadableMap): String { - val upload = Upload.fromReadableMap(options) - val data = Gson().toJson(upload) - - // Clear any stale user-cancel mark for this (possibly reused) id - // from a prior life, so a later system stop of this fresh upload isn't - // misreported as a user cancel. Done here (before enqueue), never in the - // worker, so a real cancel arriving as the worker starts can't be erased. - UserCancellations.consume(upload.id) - - val request = OneTimeWorkRequestBuilder() - .addTag(WORKER_TAG) - .addTag(ID_TAG_PREFIX + upload.id) - .setInputData(workDataOf(UploadWorker.PARAMS_KEY to data)) - .build() - - workManager - // Using KEEP policy to prevent it from cancelling the work if it's already running. - // Otherwise, it will emit "cancelled" and then go on to emit "progress" events, - // which is confusing and quite difficult to manage. "cancelled" should be reserved for - // when the user explicitly cancels the upload. - .beginUniqueWork(upload.id, ExistingWorkPolicy.KEEP, request) - .enqueue() - - return upload.id + override fun updateHeaders(patch: ReadableMap, promise: Promise) { + val headers = try { + EntryParsing.headerPatch(patch) + } catch (e: EntryParsing.InvalidEntryException) { + promise.reject(QueueException.E_INVALID, e.message, e) + return + } + onQueue(promise) { controller.updateHeaders(headers); null } } + /** Synchronous. From the in-memory index, never from disk. */ + override fun getRequests(): WritableArray { + runCatching { importDone.get(IMPORT_WAIT_MS, TimeUnit.MILLISECONDS) } + val out = Arguments.createArray() + RequestIndex.shared.snapshot().forEach { out.pushMap(JsonBridge.toWritableMap(it.toMap())) } + return out + } /** - * Starts, or resumes, a chunked upload. It is idempotent against the durable - * [ChunkedManifest]. A first call takes ownership of the source file (an - * O(1) rename into the library's directory) and persists the manifest. A - * re-call with the same id reconciles instead: identical parts are required, - * the stored headers are replaced, and the accepted parts are skipped. Crash - * recovery, a resume after a stop, and a resume with fresh auth are all this - * same call. + * Every unacknowledged outcome, each counted as one more delivery. JS + * subscribes to onSettled, then calls this at once, so this call makes + * the module the journal's listener. The flip and the scan share the + * journal lock: an outcome settled before it is in this drain (journaled + * at 0, returned at 1); one settled after is emitted live at 1. */ - @Suppress("unused") - private fun enqueueChunkedUpload(options: ReadableMap): String { - val store = ChunkedManifestStore.get(reactApplicationContext) - val id = options.getString("id") - ?: throw Upload.MissingOptionException("id") - val blob = store.blobFile(id) - val incoming = ChunkedManifest.fromReadableMap( - options, - sourcePath = blob.absolutePath, - createdAt = System.currentTimeMillis(), - ) - - // One atomic store operation, persisted BEFORE the work is enqueued. The - // manifest is what a worker relaunched with no JS runs from. The store - // lock spans load, reconcile, and save. Thus a running worker's - // markAccepted can never land between them and be erased. The running flag - // inside the lock is race-free too. A worker acquires ChunkedWorkerGate - // before its first manifest read. Thus it either registers first (and the - // recreate is rejected), or it reads the manifest that this call saved. - store.compute(id) { existing -> - if (existing == null) { - val path = options.getString("path") ?: throw Upload.MissingOptionException("path") - takeOwnership(File(path), blob) - incoming - } else { - // `path` is deliberately ignored here. When a manifest exists, the - // owned bytes are the source of truth. - existing.reconcile( - incoming, - running = ChunkedWorkerGate.isRunning(id), - blobSize = File(existing.sourcePath).length(), - ) - } + override fun getUnacknowledgedEvents(promise: Promise) { + onQueue(promise) { + val out = Arguments.createArray() + controller.unacknowledged(this) { !invalidated }.forEach { out.pushMap(it.toWritableMap()) } + out } - - // The stale-mark reasoning is the same as in enqueueUpload. - UserCancellations.consume(id) - - // A queued successor (an unfinished row that is not RUNNING) already - // guarantees a run after the current one finishes. An appended second run - // would only stack duplicate no-op runs. The manifest reconcile above - // still landed. That is how this call's fresh headers reach the queued - // run. - val states = workManager.getWorkInfosForUniqueWork(id).get().map { it.state } - if (hasQueuedSuccessor(states)) return id - - val request = OneTimeWorkRequestBuilder() - .addTag(WORKER_TAG) - .addTag(ID_TAG_PREFIX + id) - .setInputData(workDataOf(ChunkedUploadWorker.ID_KEY to id)) - .build() - - // APPEND_OR_REPLACE, not KEEP. A worker journals its terminal error before - // doWork returns. Thus a consumer that resumes from the error handler can - // arrive while that run's row is still RUNNING. KEEP would silently drop - // the resume, and nothing would ever run it. An append keeps the runs - // strictly sequential, and a trailing run over an already-settled manifest - // is a clean no-op (see ChunkedEngine.startAction). Appended work is a - // chain DEPENDENT: WorkManager marks the dependents of a failed - // prerequisite FAILED without a run. That is why ChunkedUploadWorker - // always returns Result.success(), even after it journals a terminal error - // (see terminalErrorResult). The OR_REPLACE half only rescues enqueues - // that arrive AFTER the chain already settled failed or cancelled: it - // starts a fresh sequence. A re-call while the worker runs still never - // restarts it. The running worker re-reads the stored manifest before - // every part attempt, so a resume's fresh headers reach it. - workManager - .beginUniqueWork(id, ExistingWorkPolicy.APPEND_OR_REPLACE, request) - .enqueue() - - return id } - @Suppress("unused") - private fun takeOwnership(source: File, blob: File) { - if (!source.exists()) { - // A crash between the rename and the manifest save leaves the bytes at - // the blob path with no manifest. Adopt them. Do not fail the retry. - if (blob.exists()) return - throw IllegalArgumentException("chunked source file does not exist: ${source.path}") - } - blob.parentFile?.mkdirs() - if (blob.exists()) blob.delete() - if (source.renameTo(blob)) return - // renameTo cannot cross filesystems. Files.move falls back to copy+delete. - Files.move(source.toPath(), blob.toPath(), StandardCopyOption.REPLACE_EXISTING) - } -} - -/** - * Releases the uploads whose 'completed' events were just acknowledged. That - * is the ONE moment when a chunked upload's manifest and moved bytes may be - * deleted. The allAccepted guard protects a recreate: the id may have been - * RECREATED (a different parts array under the same id) and run again over - * these bytes. An ack of the old life's completion must not cancel that work, - * and it must not delete the blob under it. Simple uploads have no manifest - * and fall through untouched. A cancel of their unique work could kill an - * unrelated new upload that reuses the id. - */ -internal fun releaseAckedCompletions( - uploadIds: List, - store: ChunkedManifestStore, - cancelWork: (String) -> Unit, -) { - uploadIds.forEach { id -> - val manifest = store.load(id) ?: return@forEach - if (!manifest.allAccepted) return@forEach - // Cancel a still-enqueued trailing run BEFORE the delete. A worker that - // starts after the delete finds nothing. It exits silently, but there is - // no reason to run it at all. - cancelWork(id) - store.remove(id) + /** Resolves void. Idempotent; unknown ids are ignored. */ + override fun ackEvents(ids: ReadableArray, promise: Promise) { + val eventIds = (0 until ids.size()).mapNotNull { runCatching { ids.getString(it) }.getOrNull() } + onQueue(promise) { controller.ack(eventIds); null } } } - -/** - * Whether cancelUpload must journal and emit the 'cancelled' event itself. - * That is the case only when NO row is RUNNING. A never-started row (ENQUEUED, - * or BLOCKED as an appended chain's dependent) has no worker to run a stop - * handler. A RUNNING worker's stop handler owns the report, including a worker - * that still waits on the ChunkedWorkerGate. - */ -internal fun cancelReportsFromModule(unfinishedStates: List): Boolean = - unfinishedStates.isNotEmpty() && unfinishedStates.none { it == WorkInfo.State.RUNNING } - -/** An unfinished row that is not RUNNING: a queued run that did not start yet. */ -internal fun hasQueuedSuccessor(states: List): Boolean = - states.any { !it.isFinished && it != WorkInfo.State.RUNNING } - -/** - * One state for a chunked upload id, from all its WorkInfo rows plus the - * durable manifest. A live row wins. With no live row, the manifest speaks. - * The state is never "cancelled". iOS getAllUploads has no lingering cancelled - * rows (a cancelled task leaves the session). And on Android, a cancelled - * chunked upload keeps its manifest. Its truthful state is - * stalled-awaiting-resume, that is, "error". - */ -internal fun chunkedUploadState(states: List, allAccepted: Boolean): String = - when { - WorkInfo.State.RUNNING in states -> "running" - states.any { !it.isFinished } -> "pending" - allAccepted -> "completed" - else -> "error" - } - -/** - * One state for a simple upload id, from all its WorkInfo rows. An id can have - * a lingering finished chain next to a live one. A live row wins. Otherwise - * the most conclusive finished state wins. - */ -internal fun simpleUploadState(states: List): String = - when { - WorkInfo.State.RUNNING in states -> "running" - states.any { !it.isFinished } -> "pending" - WorkInfo.State.SUCCEEDED in states -> "completed" - WorkInfo.State.FAILED in states -> "error" - else -> "cancelled" - } diff --git a/android/src/main/java/ai/openspace/backgroundupload/UserCancellations.kt b/android/src/main/java/ai/openspace/backgroundupload/UserCancellations.kt deleted file mode 100644 index 5b19cea6..00000000 --- a/android/src/main/java/ai/openspace/backgroundupload/UserCancellations.kt +++ /dev/null @@ -1,17 +0,0 @@ -package ai.openspace.backgroundupload - -// Upload ids the JS side explicitly cancelled. Consulted by the worker to -// distinguish user cancels from system kills (WorkManager 2.8.1 has no -// getStopReason). Same-process only: a user cancel always originates from live -// JS, so the set never needs to persist across process death. -object UserCancellations { - private val ids = mutableSetOf() - - @Synchronized - fun mark(id: String) { - ids.add(id) - } - - @Synchronized - fun consume(id: String): Boolean = ids.remove(id) -} diff --git a/android/src/main/java/ai/openspace/backgroundupload/WorkerGate.kt b/android/src/main/java/ai/openspace/backgroundupload/WorkerGate.kt new file mode 100644 index 00000000..d1313aeb --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/WorkerGate.kt @@ -0,0 +1,32 @@ +package ai.openspace.backgroundupload + +import java.util.concurrent.ConcurrentHashMap + +/** + * At most one worker EXECUTES per entry id, process-wide (renamed from the + * v9 ChunkedWorkerGate; both workers use it now). + * + * The unique-work chain almost guarantees this, but not across a cancel: + * cancelUniqueWork marks the row CANCELLED at once while the old worker's + * coroutine still winds down, and the wake-up work runs under a second + * unique name. Two concurrent requests for one chunked part are unsafe on + * the server. A starting worker acquires its id here and a second one waits. + * + * Same-process only. A worker in a dead process holds nothing. + */ +object WorkerGate { + private val holders = ConcurrentHashMap() + + /** True when [token] now holds the id, or already held it. False while another token holds it. */ + fun tryAcquire(id: String, token: Any): Boolean { + val current = holders.putIfAbsent(id, token) + return current == null || current === token + } + + /** Releases only when [token] is the holder, so a late release can not evict a successor. */ + fun release(id: String, token: Any) { + holders.remove(id, token) + } + + fun isRunning(id: String): Boolean = holders.containsKey(id) +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/WorkerOps.kt b/android/src/main/java/ai/openspace/backgroundupload/WorkerOps.kt new file mode 100644 index 00000000..0420fada --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/WorkerOps.kt @@ -0,0 +1,305 @@ +package ai.openspace.backgroundupload + +import java.io.IOException +import java.util.UUID + +/** How a run ended. */ +sealed class Settlement { + abstract val url: String + abstract val method: String + + /** [response] is null for a chunked completion. */ + data class Completed( + val response: UploadResponse?, + override val url: String, + override val method: String, + ) : Settlement() + + /** [bytesSent] is the live bytes of a simple entry's last attempt; null keeps the stored value. */ + data class Failed( + val errorKind: String, + val message: String, + val response: UploadResponse?, + val partIndex: Int?, + override val url: String, + override val method: String, + val bytesSent: Long? = null, + ) : Settlement() +} + +/** + * The entry was paused, cancelled, replaced, or forgotten under a running + * worker. The worker stops without a transition; the module owns what + * happened. Not a CancellationException: it must fail a chunked part's + * scope so the sibling parts stop too. + */ +class NotOwnedException(id: String) : Exception("entry '$id' is no longer owned by this run") + +/** + * Every transition the network causes: running, one attempt, part accepted, + * awaiting-auth, queued-with-backoff, a system stop, and the settle. Each is + * a `compute` guarded by the run's [generation], so a cancel, pause, or + * replace that landed first always wins. + */ +class WorkerOps( + private val store: QueueStore, + private val journal: EventJournal, + private val settings: QueueSettingsStore, + private val events: QueueEvents, + private val scheduler: WorkScheduler, + private val clock: () -> Long = System::currentTimeMillis, +) { + enum class ParkResult { PARKED, REISSUE, NOT_OWNED } + + /** + * Takes a queued entry. Null when there is nothing to run. + * + * A journal record of the entry's own generation means a settle was + * journaled and its store write was lost (a process death between the + * two, or a failed save). WorkManager can run the entry again before any + * boot sweep. Then begin applies the record, as the sweep does, and does + * not send the request again. + */ + fun begin(id: String): QueueEntry? { + val now = clock() + var row: QueueEntry? = null + var taken: QueueEntry? = null + store.locked { + val e = store.load(id) ?: return@locked + if (e.legacy || (e.state != EntryState.QUEUED && e.state != EntryState.RUNNING)) return@locked + val journaled = EntryTransitions.journaledSettle(e, journal.forEntry(id), now) + if (journaled != null) { + store.save(journaled.entry) + journal.ack(journaled.extraEventIds) + row = journaled.entry + return@locked + } + val next = EntryTransitions.toRunning(e, now) + store.save(next) + row = next + taken = next + } + row?.let { events.state(it.toRow()) } + return taken + } + + /** The stored entry, while this run still owns it. */ + fun latest(id: String, generation: Int): QueueEntry { + val e = store.load(id) + if (!EntryTransitions.isOwnedRun(e, generation)) throw NotOwnedException(id) + return e!! + } + + /** + * Write-ahead for one attempt: attempts + 1 and the X-Request-Id, persisted + * before the request is sent. Clears a short backoff's nextAttemptAt, and + * emits the row when it did. Returns the fresh entry, whose headers the + * attempt uses. + */ + fun recordAttempt(id: String, generation: Int, requestId: String): QueueEntry { + val now = clock() + var clearedBackoff = false + val next = store.compute(id) { e -> + if (EntryTransitions.isOwnedRun(e, generation)) { + clearedBackoff = e!!.nextAttemptAt != null + EntryTransitions.toAttempt(e, requestId, now) + } else e + } + if (next == null || next.lastRequestId != requestId || !EntryTransitions.isOwnedRun(next, generation)) { + throw NotOwnedException(id) + } + if (clearedBackoff) events.state(next.toRow()) + return next + } + + /** + * A short backoff the worker waits out in place: the row stays running + * and carries [nextAttemptAt]. Best effort; a lost write only hides the + * time. For a chunked entry, a sibling part's next attempt clears it. + */ + fun backingOff(id: String, generation: Int, nextAttemptAt: Long) { + val now = clock() + var applied = false + val next = store.update(id) { e -> + if (EntryTransitions.isOwnedRun(e, generation)) { + applied = true + EntryTransitions.toBackingOff(e, nextAttemptAt, now) + } else e + } + if (applied && next != null) events.state(next.toRow()) + } + + /** A chunked part the server accepted. Best effort, as v9: a lost flag only re-sends that part later. */ + fun markAccepted(id: String, generation: Int, index: Int): QueueEntry? = + store.update(id) { e -> + val parts = e.descriptor?.parts + if (e.generation != generation || parts == null || index !in parts.indices) e + else { + val next = ChunkedParts.withAccepted(parts, index) + e.copy( + descriptor = e.descriptor.copy(parts = next), + bytesSent = ChunkedParts.acceptedBytes(next), + backoffStreak = 0, + ) + } + } + + /** + * Journal, then transition, then emit, all under the store lock, so a + * cancel, pause, or replace lands either before (this run's outcome is + * dropped) or after. Returns whether this run's outcome stands. + * + * A failed journal write holds the record in memory ([EventJournal.appendOrHold]): + * the request already ran, and a retry would send it twice. A failed + * store write leaves the record for the ack, the next [begin], or the + * boot sweep to apply. + * + * A record of this generation already in the journal (a cancel whose + * entry save failed) wins: it is applied, as [begin] does, and this run's + * outcome is dropped. One life has one outcome. + */ + fun settle(id: String, generation: Int, s: Settlement): Boolean { + val now = clock() + val completed = s is Settlement.Completed + val failed = s as? Settlement.Failed + var delivered: EventJournal.SettledRecord? = null + var settled: QueueEntry? = null + store.locked { + val e = store.load(id) + if (!EntryTransitions.canSettle(e, generation, completed)) return@locked + e!! + val journaled = EntryTransitions.journaledSettle(e, journal.forEntry(id), now) + if (journaled != null) { + try { + store.save(journaled.entry) + journal.ack(journaled.extraEventIds) + settled = journaled.entry + } catch (error: IOException) { + Diag.error("settle could not apply the journaled outcome of '$id'; its ack or the boot sweep applies it", error) + } + return@locked + } + val state = if (completed) EntryState.COMPLETED else EntryState.ERROR + // A failed simple entry keeps the live bytes of its last attempt; a + // chunked one keeps its accepted bytes (the stored value). + val bytesSent = if (completed) e.totalBytes else failed?.bytesSent ?: e.bytesSent + val record = EventJournal.SettledRecord( + eventId = UUID.randomUUID().toString(), + id = e.id, + key = e.key, + varsJson = e.varsJson, + at = now, + attempts = e.attempts, + requestId = e.lastRequestId, + deliveries = 0, // the journal sets it + state = state.wire, + bytesSent = bytesSent, + totalBytes = e.totalBytes, + url = s.url, + method = s.method, + partIndex = failed?.partIndex, + kind = if (completed) EventJournal.KIND_COMPLETED else EventJournal.KIND_ERROR, + response = when (s) { + is Settlement.Completed -> s.response?.let { EventJournal.Response.of(it) } ?: EventJournal.Response.NONE + is Settlement.Failed -> s.response?.let { EventJournal.Response.of(it) } + }, + errorKind = failed?.errorKind, + message = failed?.message, + cancelReason = null, + generation = generation, + ) + // 1. The durable outcome. It never throws. + delivered = journal.appendOrHold(record) { store.referencedEventIds() + record.eventId } + // 2. The transition. + val next = EntryTransitions.toSettled(e, state, record.eventId, bytesSent, now) + try { + store.save(next) + settled = next + } catch (error: IOException) { + Diag.error("settle could not save '$id'; its ack, the next run, or the boot sweep applies the record", error) + } + } + val record = delivered + if (record == null) { + settled?.let { events.state(it.toRow()) } + return false + } + // 3 and 4. Best effort. + if (record.deliveries > 0) journal.listener()?.let { events.settled(record, it) } + settled?.let { events.state(it.toRow()) } + return true + } + + /** + * Whether the stored entry holds newer headers than the ones an attempt + * sent. [headerGeneration] is the entry's own value from [recordAttempt], + * so it always belongs to the headers that went out. The settings value + * is not used: updateHeaders() bumps it before it patches the entries. + */ + fun hasNewerHeaders(id: String, generation: Int, headerGeneration: Int): Boolean = + latest(id, generation).headerGeneration > headerGeneration + + /** + * A 401/403. [headerGeneration] is the entry's value from [recordAttempt]. + * When the entry got newer headers since, the attempt re-issues at once + * instead of parking. The check is inside the store lock, and + * updateHeaders() patches entries inside it too, so it either patched + * this entry first (REISSUE) or finds it parked and requeues it. + */ + fun park(id: String, generation: Int, headerGeneration: Int): ParkResult { + val now = clock() + var result = ParkResult.NOT_OWNED + val next = store.compute(id) { e -> + when { + !EntryTransitions.isOwnedRun(e, generation) -> e + e!!.headerGeneration > headerGeneration -> { + result = ParkResult.REISSUE + e + } + else -> { + result = ParkResult.PARKED + EntryTransitions.toParked(e, headerGeneration, now) + } + } + } + if (result == ParkResult.PARKED && next != null) { + events.state(next.toRow()) + // A parked entry still expires on time. + scheduler.scheduleWake(next, next.expiresAt, replace = true) + } + return result + } + + /** A backoff longer than a worker waits: back to queued, woken at [nextAttemptAt]. */ + fun release(id: String, generation: Int, nextAttemptAt: Long, streak: Int): Boolean { + val now = clock() + var applied = false + val next = store.compute(id) { e -> + if (EntryTransitions.isOwnedRun(e, generation)) { + applied = true + EntryTransitions.toReleased(e!!, nextAttemptAt, streak, now) + } else e + } + if (!applied || next == null) return false + events.state(next.toRow()) + scheduler.scheduleWake(next, nextAttemptAt, replace = true) + return true + } + + /** A system stop. A paused or cancelled entry is the module's, so only a running one moves. Never journals. */ + fun stopped(id: String, generation: Int) { + val now = clock() + var applied = false + val next = runCatching { + store.compute(id) { e -> + if (EntryTransitions.isOwnedRun(e, generation)) { + applied = true + EntryTransitions.toStopped(e!!, now) + } else e + } + }.getOrNull() + if (applied && next != null) events.state(next.toRow()) + } + + fun settings(): QueueSettings = settings.load() +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/AckReleaseTest.kt b/android/src/test/java/ai/openspace/backgroundupload/AckReleaseTest.kt deleted file mode 100644 index 0afe1e88..00000000 --- a/android/src/test/java/ai/openspace/backgroundupload/AckReleaseTest.kt +++ /dev/null @@ -1,69 +0,0 @@ -package ai.openspace.backgroundupload - -import org.junit.Assert.assertEquals -import org.junit.Assert.assertNotNull -import org.junit.Assert.assertNull -import org.junit.Assert.assertTrue -import org.junit.Rule -import org.junit.Test -import org.junit.rules.TemporaryFolder - -// An acknowledged 'completed' is the one moment when a chunked upload's stored -// state may be released. But only the completed life's state may go. A recreate -// under the same id can run over the same bytes, and it must survive the old -// life's ack. -class AckReleaseTest { - @get:Rule - val tmp = TemporaryFolder() - - private fun manifest(id: String, accepted: Boolean) = ChunkedManifest( - id = id, - sourcePath = "/data/blob", - parts = listOf( - ChunkedManifest.Part( - url = "https://example.com/1", - headers = emptyMap(), - start = 0, - end = 100, - accepted = accepted, - ), - ), - accept = emptyList(), - expiresAt = 5_000, - wifiOnly = false, - noNotification = false, - createdAt = 1_000, - ) - - @Test - fun `releases a completed upload's manifest and cancels its trailing runs`() { - val store = ChunkedManifestStore(tmp.newFolder()) - store.save(manifest("u1", accepted = true)) - val cancelled = mutableListOf() - releaseAckedCompletions(listOf("u1"), store) { cancelled.add(it) } - assertNull(store.load("u1")) - assertEquals(listOf("u1"), cancelled) - } - - @Test - fun `spares a recreate running under the same id`() { - // The acknowledged completion belongs to the id's PREVIOUS life. The - // manifest now holds a recreate's unaccepted parts, and a worker can be - // mid-transfer. A work cancel or a blob delete here would destroy its - // bytes. - val store = ChunkedManifestStore(tmp.newFolder()) - store.save(manifest("u1", accepted = false)) - val cancelled = mutableListOf() - releaseAckedCompletions(listOf("u1"), store) { cancelled.add(it) } - assertNotNull(store.load("u1")) - assertTrue(cancelled.isEmpty()) - } - - @Test - fun `ignores ids with no manifest (simple uploads)`() { - val store = ChunkedManifestStore(tmp.newFolder()) - val cancelled = mutableListOf() - releaseAckedCompletions(listOf("raw-upload"), store) { cancelled.add(it) } - assertTrue(cancelled.isEmpty()) - } -} diff --git a/android/src/test/java/ai/openspace/backgroundupload/BodyCapTest.kt b/android/src/test/java/ai/openspace/backgroundupload/BodyCapTest.kt new file mode 100644 index 00000000..53a75385 --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/BodyCapTest.kt @@ -0,0 +1,70 @@ +package ai.openspace.backgroundupload + +import okio.Buffer +import okio.BufferedSource +import okio.buffer +import okio.ForwardingSource +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Test + +class BodyCapTest { + /** A source that counts the bytes read from it. */ + private class Counting(bytes: ByteArray) : ForwardingSource(Buffer().write(bytes)) { + var read = 0L + override fun read(sink: Buffer, byteCount: Long): Long = + super.read(sink, byteCount).also { if (it > 0) read += it } + } + + private fun source(bytes: ByteArray): Pair { + val c = Counting(bytes) + return c to c.buffer() + } + + @Test + fun `a body under the cap is read whole`() { + val (_, s) = source("hello".toByteArray()) + assertEquals(BodyCap.Capped("hello", false), BodyCap.read(s, 10)) + } + + @Test + fun `a body at exactly the cap is not truncated`() { + val (_, s) = source("0123456789".toByteArray()) + assertEquals(BodyCap.Capped("0123456789", false), BodyCap.read(s, 10)) + } + + @Test + fun `a huge body stops streaming just past the cap`() { + val (counting, s) = source(ByteArray(5_000_000) { 'x'.code.toByte() }) + val capped = BodyCap.read(s, 1_000) + assertTrue(capped.truncated) + assertEquals(1_000, capped.text.length) + // okio reads in 8 KB segments; nowhere near the 5 MB body. + assertTrue("read ${counting.read}", counting.read < 64 * 1024) + } + + @Test + fun `a cut inside a UTF-8 character backs off to the last whole one`() { + val euros = "€".repeat(4).toByteArray(Charsets.UTF_8) // 12 bytes + val (_, s) = source(euros) + val capped = BodyCap.read(s, 10) + assertEquals("€".repeat(3), capped.text) + assertTrue(capped.truncated) + } + + @Test + fun `cap measures UTF-8 bytes, not characters`() { + assertEquals("ab" to false, BodyCap.cap("ab", 2)) + assertEquals("é" to true, BodyCap.cap("éé", 3)) + assertEquals(null to false, BodyCap.cap(null, 3)) + // A 4-byte character (an emoji) is never split. + assertEquals("a" to true, BodyCap.cap("a😀", 4)) + } + + @Test + fun `bytes that are not UTF-8 are cut at the cap`() { + val bad = ByteArray(8) { 0x80.toByte() } // continuation bytes only + assertEquals(5, BodyCap.utf8Boundary(bad, 5)) + assertEquals(3, BodyCap.utf8Boundary("abc".toByteArray(), 5)) + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/BodyStagingTest.kt b/android/src/test/java/ai/openspace/backgroundupload/BodyStagingTest.kt new file mode 100644 index 00000000..216c69a6 --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/BodyStagingTest.kt @@ -0,0 +1,207 @@ +package ai.openspace.backgroundupload + +import org.junit.Assert.assertArrayEquals +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertThrows +import org.junit.Assert.assertTrue +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import java.io.File + +class BodyStagingTest { + @get:Rule + val tmp = TemporaryFolder() + + private fun source(name: String, bytes: ByteArray) = File(tmp.newFolder(), name).apply { writeBytes(bytes) } + + @Test + fun `json bytes are the data text, with a default content type`() { + val dir = tmp.newFolder() + val staged = BodyStaging.stage(desc(dataJson = """{"n":1,"t":"é"}""", headers = mapOf()), dir, 1) + assertEquals(StagedBody.JSON, staged.body.kind) + assertEquals("body-1.json", staged.body.fileName) + assertEquals("""{"n":1,"t":"é"}""", File(dir, "body-1.json").readText()) + assertEquals(File(dir, "body-1.json").length(), staged.body.totalBytes) + assertEquals(mapOf("Content-Type" to "application/json"), staged.headers) + } + + @Test + fun `a caller content type wins for json, in any case`() { + val staged = BodyStaging.stage(desc(dataJson = "{}", headers = mapOf("content-type" to "application/vnd+json")), tmp.newFolder(), 1) + assertEquals(mapOf("content-type" to "application/vnd+json"), staged.headers) + } + + @Test + fun `multipart bytes follow RFC 7578`() { + val photo = source("photo.jpg", byteArrayOf(1, 2, 3)) + val form = listOf( + FormPart("meta", "application/json", "{\"a\":\"b\"}", null, null), + FormPart("pho\"to", "image/jpeg", null, photo.path, null), + FormPart("named", "image/jpeg", null, photo.path, "new\nname.jpg"), + ) + val target = File(tmp.newFolder(), "body.multipart") + BodyStaging.writeMultipart(form, "BOUND", target) + val expected = ("--BOUND\r\n" + + "Content-Disposition: form-data; name=\"meta\"\r\n" + + "Content-Type: application/json\r\n\r\n" + + "{\"a\":\"b\"}\r\n" + + "--BOUND\r\n" + + "Content-Disposition: form-data; name=\"pho%22to\"; filename=\"photo.jpg\"\r\n" + + "Content-Type: image/jpeg\r\n\r\n").toByteArray() + byteArrayOf(1, 2, 3) + ("\r\n" + + "--BOUND\r\n" + + "Content-Disposition: form-data; name=\"named\"; filename=\"new%0Aname.jpg\"\r\n" + + "Content-Type: image/jpeg\r\n\r\n").toByteArray() + byteArrayOf(1, 2, 3) + ("\r\n" + + "--BOUND--\r\n").toByteArray() + assertArrayEquals(expected, target.readBytes()) + } + + @Test + fun `form staging always sets the library content type with its boundary`() { + val dir = tmp.newFolder() + val staged = BodyStaging.stage( + desc(form = listOf(FormPart("a", "text/plain", "x", null, null)), headers = mapOf("CONTENT-TYPE" to "text/plain")), + dir, 2, + ) + val boundary = staged.body.boundary!! + assertTrue(boundary.startsWith("----RNBGU")) + assertEquals(mapOf("Content-Type" to "multipart/form-data; boundary=$boundary"), staged.headers) + assertTrue(File(dir, "body-2.multipart").readText().startsWith("--$boundary\r\n")) + } + + @Test + fun `a file body is copied and the source stays`() { + val src = source("a.bin", ByteArray(1000) { it.toByte() }) + val dir = tmp.newFolder() + val staged = BodyStaging.stage(desc(file = src.path), dir, 3) + assertTrue(src.exists()) + assertArrayEquals(src.readBytes(), File(dir, "file-3").readBytes()) + assertEquals(1000, staged.body.totalBytes) + assertEquals(mapOf("Authorization" to "Bearer old"), staged.headers) // no content type added + src.delete() + assertTrue(File(dir, "file-3").exists()) + } + + @Test + fun `a chunked file is moved and must tile the parts`() { + val src = source("video.bin", ByteArray(20)) + val dir = tmp.newFolder() + val staged = BodyStaging.stage(desc(url = null, file = src.path, parts = listOf(part(0, 10), part(10, 20))), dir, 1) + assertFalse(src.exists()) + assertEquals(20, File(dir, "blob").length()) + assertEquals(StagedBody.CHUNKED, staged.body.kind) + assertEquals(20, staged.body.totalBytes) + } + + @Test + fun `an orphan blob is adopted when the source is gone`() { + val dir = tmp.newFolder() + File(dir, "blob").writeBytes(ByteArray(20)) + val staged = BodyStaging.stage(desc(url = null, file = "/gone.bin", parts = listOf(part(0, 20))), dir, 1) + assertEquals(StagedBody.CHUNKED, staged.body.kind) + } + + @Test + fun `a tiling mismatch rejects E_INVALID before the move, so the source stays`() { + val src = source("video.bin", ByteArray(20)) + val dir = tmp.newFolder() + val e = assertThrows(QueueException::class.java) { + BodyStaging.stage(desc(url = null, file = src.path, parts = listOf(part(0, 15))), dir, 1) + } + assertEquals(QueueException.E_INVALID, e.code) + assertTrue(src.exists()) + assertFalse(File(dir, "blob").exists()) + } + + @Test + fun `keepOwned runs over the owned blob and ignores the path`() { + val dir = tmp.newFolder() + val owned = File(dir, "blob").apply { writeBytes(ByteArray(20)) } + val other = source("other.bin", ByteArray(5)) + val staged = BodyStaging.stage( + desc(url = null, file = other.path, parts = listOf(part(0, 20))), dir, 2, owned, keepOwned = true, + ) + assertTrue(other.exists()) + assertEquals("blob", staged.body.fileName) + assertEquals(20, owned.length()) + } + + @Test + fun `a present source wins over the owned blob and moves to this generation's name`() { + val dir = tmp.newFolder() + val owned = File(dir, "blob").apply { writeBytes(ByteArray(20)) } + val bytes = ByteArray(20) { (it + 1).toByte() } + val src = source("new.bin", bytes) + val staged = BodyStaging.stage(desc(url = null, file = src.path, parts = listOf(part(0, 5), part(5, 20))), dir, 3, owned) + assertEquals("blob-3", staged.body.fileName) + assertArrayEquals(bytes, File(dir, "blob-3").readBytes()) + assertFalse(src.exists()) + // The old entry's blob is untouched until the new entry is saved and prunes it. + assertArrayEquals(ByteArray(20), owned.readBytes()) + } + + @Test + fun `a present source of another size is checked against its own length`() { + val dir = tmp.newFolder() + val owned = File(dir, "blob").apply { writeBytes(ByteArray(20)) } + val src = source("new.bin", ByteArray(30)) + val staged = BodyStaging.stage(desc(url = null, file = src.path, parts = listOf(part(0, 30))), dir, 2, owned) + assertEquals("blob-2", staged.body.fileName) + assertEquals(30, staged.body.totalBytes) + assertEquals(20, owned.length()) + } + + @Test + fun `with the source gone, the owned blob is the fallback`() { + val dir = tmp.newFolder() + File(dir, "blob").writeBytes(ByteArray(20)) + val staged = BodyStaging.stage( + desc(url = null, file = "/gone.bin", parts = listOf(part(0, 5), part(5, 20))), dir, 2, File(dir, "blob"), + ) + assertEquals("blob", staged.body.fileName) + } + + @Test + fun `with the source gone, this generation's crash leftover wins over the owned blob`() { + val dir = tmp.newFolder() + File(dir, "blob").writeBytes(ByteArray(20)) + File(dir, "blob-2").writeBytes(ByteArray(30)) + val staged = BodyStaging.stage(desc(url = null, file = "/gone.bin", parts = listOf(part(0, 30))), dir, 2, File(dir, "blob")) + assertEquals("blob-2", staged.body.fileName) + } + + @Test + fun `a chunked body with no source and no blob rejects E_FILE_MISSING`() { + val e = assertThrows(QueueException::class.java) { + BodyStaging.stage(desc(url = null, file = "/gone.bin", parts = listOf(part(0, 20))), tmp.newFolder(), 2, null) + } + assertEquals(QueueException.E_FILE_MISSING, e.code) + } + + @Test + fun `a missing source rejects E_FILE_MISSING and writes nothing`() { + val dir = tmp.newFolder() + val form = listOf( + FormPart("a", "text/plain", "x", null, null), + FormPart("b", "image/jpeg", null, "/missing.jpg", null), + ) + val e = assertThrows(QueueException::class.java) { BodyStaging.stage(desc(form = form), dir, 1) } + assertEquals(QueueException.E_FILE_MISSING, e.code) + assertEquals(0, dir.list()!!.size) + val f = assertThrows(QueueException::class.java) { BodyStaging.stage(desc(file = "/missing.bin"), dir, 1) } + assertEquals(QueueException.E_FILE_MISSING, f.code) + val c = assertThrows(QueueException::class.java) { + BodyStaging.stage(desc(url = null, file = "/missing.bin", parts = listOf(part(0, 1))), dir, 1) + } + assertEquals(QueueException.E_FILE_MISSING, c.code) + } + + @Test + fun `no body stages nothing`() { + val dir = tmp.newFolder() + val staged = BodyStaging.stage(desc(method = "DELETE"), dir, 1) + assertEquals(StagedBody(StagedBody.NONE, null, null, 0), staged.body) + assertEquals(0, dir.list()!!.size) + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/ChunkedEngineTest.kt b/android/src/test/java/ai/openspace/backgroundupload/ChunkedEngineTest.kt index 8c909d62..c93f1dff 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/ChunkedEngineTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/ChunkedEngineTest.kt @@ -11,8 +11,8 @@ import java.util.concurrent.ConcurrentHashMap class ChunkedEngineTest { // runBlocking is single-threaded, so the overlap is deterministic. Every - // executor suspends at yield(). Thus all launchable siblings start before any - // executor finishes. + // executor suspends at yield(), so all launchable siblings start before + // any executor finishes. private class Tracker { var inFlight = 0 var maxInFlight = 0 @@ -39,6 +39,7 @@ class ChunkedEngineTest { fun `never more than WINDOW parts in flight`() = runBlocking { val tracker = Tracker() ChunkedEngine.run((0 until 10).toList()) { tracker.execute(it) } + assertEquals(3, ChunkedEngine.WINDOW) assertEquals(ChunkedEngine.WINDOW, tracker.maxInFlight) } @@ -55,7 +56,7 @@ class ChunkedEngineTest { } @Test - fun `a terminal part failure propagates and cancels the remaining parts`() { + fun `a part failure propagates and cancels the remaining parts`() { val tracker = Tracker() val thrown = assertThrows(IllegalStateException::class.java) { runBlocking { @@ -70,112 +71,17 @@ class ChunkedEngineTest { } @Test - fun `backoff grows exponentially and caps`() { - assertEquals(1_000, ChunkedEngine.backoffMs(1)) - assertEquals(2_000, ChunkedEngine.backoffMs(2)) - assertEquals(4_000, ChunkedEngine.backoffMs(3)) - assertEquals(60_000, ChunkedEngine.backoffMs(7)) - assertEquals(60_000, ChunkedEngine.backoffMs(100)) - // Defensive: a nonsense attempt number must not shift into a huge delay. - assertEquals(1_000, ChunkedEngine.backoffMs(0)) - } - - // MARK: - startAction - - private fun manifest(vararg accepted: Boolean) = ChunkedManifest( - id = "u1", - sourcePath = "/data/blob", - parts = accepted.mapIndexed { i, a -> - ChunkedManifest.Part( - url = "https://example.com/part?n=$i", - headers = emptyMap(), - start = i * 100L, - end = (i + 1) * 100L, - accepted = a, - ) - }, - accept = emptyList(), - expiresAt = 5_000, - wifiOnly = false, - noNotification = false, - createdAt = 1_000, - ) - - @Test - fun `no manifest at start is a silent success, never a journaled error`() { - // A completed ack or removeUpload deleted the manifest while this run sat - // in the queue. That is a legitimate end, already settled. - assertEquals(ChunkedEngine.StartAction.NO_MANIFEST, ChunkedEngine.startAction(null)) - } - - @Test - fun `an all-accepted manifest re-reports completion instead of running`() { - assertEquals( - ChunkedEngine.StartAction.ALREADY_COMPLETE, - ChunkedEngine.startAction(manifest(true, true)), - ) - } - - @Test - fun `pending parts run the engine`() { - assertEquals(ChunkedEngine.StartAction.RUN, ChunkedEngine.startAction(manifest(true, false))) - } - - // MARK: - completionReport - - private fun completedEntry(uploadId: String) = EventJournal.Entry( - eventId = "e-$uploadId", - uploadId = uploadId, - type = "completed", - timestamp = 1, - ) - - @Test - fun `an unacked completed entry is re-emitted, never minted twice`() { - assertEquals( - ChunkedEngine.CompletionReport.ReEmit(completedEntry("u1")), - ChunkedEngine.completionReport(listOf(completedEntry("u1")), "u1", freshCompletion = false), - ) - assertEquals( - ChunkedEngine.CompletionReport.ReEmit(completedEntry("u1")), - ChunkedEngine.completionReport(listOf(completedEntry("u1")), "u1", freshCompletion = true), - ) - } - - @Test - fun `a fresh completion with nothing journaled mints a new entry`() { - assertEquals( - ChunkedEngine.CompletionReport.Mint, - ChunkedEngine.completionReport(emptyList(), "u1", freshCompletion = true), - ) - } - - @Test - fun `a trailing run over an acked completion reports nothing`() { - // The trailing run raced ackEvents. The journal entry is already gone, but - // the manifest still exists for a moment. An acknowledged completion means - // that nobody is owed an event. A minted event would be a duplicate - // 'completed' for an upload that the consumer already settled. - assertEquals( - ChunkedEngine.CompletionReport.None, - ChunkedEngine.completionReport(emptyList(), "u1", freshCompletion = false), - ) - } - - @Test - fun `another upload's completed entry does not satisfy the lookup`() { - assertEquals( - ChunkedEngine.CompletionReport.None, - ChunkedEngine.completionReport(listOf(completedEntry("other")), "u1", freshCompletion = false), - ) - } - - @Test - fun `only 5xx responses are transient`() { - assertTrue(ChunkedEngine.isTransientHttp(500)) - assertTrue(ChunkedEngine.isTransientHttp(599)) - for (code in listOf(400, 401, 403, 404, 409, 429, 499, 600)) { - assertEquals("code $code", false, ChunkedEngine.isTransientHttp(code)) + fun `a park from one part stops the siblings with the park itself`() { + // The worker needs the ParkException back, not a CancellationException. + val thrown = assertThrows(EntryRun.ParkException::class.java) { + runBlocking { + ChunkedEngine.run((0 until 6).toList()) { index -> + yield() + if (index == 1) throw EntryRun.ParkException(4) + yield() + } + } } + assertEquals(4, thrown.headerGeneration) } } diff --git a/android/src/test/java/ai/openspace/backgroundupload/ChunkedManifestTest.kt b/android/src/test/java/ai/openspace/backgroundupload/ChunkedManifestTest.kt deleted file mode 100644 index 798852ef..00000000 --- a/android/src/test/java/ai/openspace/backgroundupload/ChunkedManifestTest.kt +++ /dev/null @@ -1,384 +0,0 @@ -package ai.openspace.backgroundupload - -import ai.openspace.backgroundupload.UploadOutcome.AcceptRule -import org.junit.Assert.assertEquals -import org.junit.Assert.assertFalse -import org.junit.Assert.assertNull -import org.junit.Assert.assertThrows -import org.junit.Assert.assertTrue -import org.junit.Rule -import org.junit.Test -import org.junit.rules.TemporaryFolder -import java.io.File -import java.util.concurrent.CountDownLatch -import java.util.concurrent.TimeUnit - -class ChunkedManifestTest { - @get:Rule - val tmp = TemporaryFolder() - - private fun manifest( - id: String = "u1", - parts: List = listOf( - part(0, 100), - part(100, 250), - ), - expiresAt: Long = 5_000, - ) = ChunkedManifest( - id = id, - sourcePath = "/data/blob", - parts = parts, - accept = listOf(AcceptRule(409, "already completed")), - expiresAt = expiresAt, - wifiOnly = false, - noNotification = false, - createdAt = 1_000, - ) - - private fun part(start: Long, end: Long, accepted: Boolean = false) = - ChunkedManifest.Part( - url = "https://example.com/part?start=$start", - headers = mapOf("Authorization" to "Bearer old"), - start = start, - end = end, - accepted = accepted, - ) - - // MARK: - Model - - @Test - fun `byte math is range-based`() { - val m = manifest(parts = listOf(part(0, 100, accepted = true), part(100, 250))) - assertEquals(250, m.totalBytes) - assertEquals(100, m.acceptedBytes) - } - - @Test - fun `completed only when every part is accepted`() { - val none = manifest() - assertFalse(none.allAccepted) - val partial = none.withPartAccepted(0) - assertFalse(partial.allAccepted) - val all = partial.withPartAccepted(1) - assertTrue(all.allAccepted) - assertEquals(emptyList(), all.pendingIndexes()) - assertEquals(listOf(1), partial.pendingIndexes()) - } - - @Test - fun `expiry is inclusive of the deadline`() { - val m = manifest(expiresAt = 5_000) - assertFalse(m.isExpired(4_999)) - assertTrue(m.isExpired(5_000)) - assertTrue(m.isExpired(5_001)) - } - - // MARK: - Reconcile: resume (same parts array) - - private val blobSize = 250L - - @Test - fun `resume replaces headers and deadline, keeps accepted parts and the moved source`() { - val stored = manifest().withPartAccepted(0) - val fresh = manifest(expiresAt = 99_000).copy( - sourcePath = "/ignored/by/reconcile", - createdAt = 42, - accept = listOf(AcceptRule(208)), - wifiOnly = true, - parts = stored.parts.map { it.copy(headers = mapOf("Authorization" to "Bearer new"), accepted = false) }, - ) - - val merged = stored.reconcile(fresh, running = false, blobSize = blobSize) - - assertEquals("Bearer new", merged.parts[0].headers["Authorization"]) - assertEquals(99_000, merged.expiresAt) - assertEquals(listOf(AcceptRule(208)), merged.accept) - assertTrue(merged.wifiOnly) - // Accepted statuses, ownership, and identity survive from the stored copy. - assertTrue(merged.parts[0].accepted) - assertFalse(merged.parts[1].accepted) - assertEquals("/data/blob", merged.sourcePath) - assertEquals(1_000, merged.createdAt) - } - - @Test - fun `resume is allowed while the upload is running`() { - // Fresh auth must reach a running worker's stalled parts. - val stored = manifest().withPartAccepted(0) - val fresh = manifest().copy( - parts = stored.parts.map { it.copy(headers = mapOf("Authorization" to "Bearer new"), accepted = false) }, - ) - val merged = stored.reconcile(fresh, running = true, blobSize = blobSize) - assertTrue(merged.parts[0].accepted) - assertEquals("Bearer new", merged.parts[1].headers["Authorization"]) - } - - @Test - fun `resume matches the same parts authored in a different order`() { - // Identical tiles, reordered, are the SAME upload: a resume, never a - // recreate (running = true would reject a recreate). Accepted flags follow - // the range, not the array index. - val stored = manifest().withPartAccepted(0) - val fresh = manifest().copy( - parts = listOf(stored.parts[1], stored.parts[0]).map { - it.copy(headers = mapOf("Authorization" to "Bearer new"), accepted = false) - }, - ) - val merged = stored.reconcile(fresh, running = true, blobSize = blobSize) - assertTrue(merged.parts.first { it.start == 0L }.accepted) - assertFalse(merged.parts.first { it.start == 100L }.accepted) - assertEquals("Bearer new", merged.parts[0].headers["Authorization"]) - } - - // MARK: - Reconcile: recreate (different parts array) - - @Test - fun `recreate from a stalled upload replaces parts and resets every status`() { - // The consumer re-authored under a fresh server uploadId: new urls, a new - // split, and fresh headers, accept, and expiresAt. The owned bytes stay. - val stored = manifest().withPartAccepted(0) - val fresh = manifest( - parts = listOf( - part(0, 120).copy(url = "https://example.com/v2?part=1"), - part(120, 250).copy(url = "https://example.com/v2?part=2"), - ), - expiresAt = 99_000, - ).copy(sourcePath = "/ignored/by/reconcile", createdAt = 42, accept = listOf(AcceptRule(208))) - - val recreated = stored.reconcile(fresh, running = false, blobSize = blobSize) - - assertTrue(recreated.parts.none { it.accepted }) - assertEquals(listOf("https://example.com/v2?part=1", "https://example.com/v2?part=2"), recreated.parts.map { it.url }) - assertEquals(99_000, recreated.expiresAt) - assertEquals(listOf(AcceptRule(208)), recreated.accept) - // Ownership survives. The blob is reused for the full re-upload. - assertEquals("/data/blob", recreated.sourcePath) - assertEquals(1_000, recreated.createdAt) - } - - @Test - fun `recreate with the same ranges but new urls also resets statuses`() { - // New part urls embed a new server uploadId, even when the split is - // identical. Nothing sent under the old id counts for the new one. - val stored = manifest().withPartAccepted(0) - val fresh = manifest( - parts = listOf( - part(0, 100).copy(url = "https://example.com/v2?part=1"), - part(100, 250).copy(url = "https://example.com/v2?part=2"), - ), - ) - val recreated = stored.reconcile(fresh, running = false, blobSize = blobSize) - assertTrue(recreated.parts.none { it.accepted }) - } - - @Test - fun `recreate is rejected while the upload is running`() { - val stored = manifest() - val fresh = manifest(parts = listOf(part(0, 250).copy(url = "https://example.com/v2"))) - assertThrows(ChunkedManifest.ReconcileException::class.java) { - stored.reconcile(fresh, running = true, blobSize = blobSize) - } - } - - @Test - fun `recreate rejects parts that do not tile the blob exactly`() { - val stored = manifest() - for ( - bad in listOf( - listOf(part(0, 100), part(150, 250)), // gap - listOf(part(0, 150), part(100, 250)), // overlap - listOf(part(50, 250)), // does not start at 0 - listOf(part(0, 200)), // short of the blob size - listOf(part(0, 100), part(100, 251)), // past the blob size - ) - ) { - assertThrows(ChunkedManifest.ReconcileException::class.java) { - stored.reconcile(manifest(parts = bad), running = false, blobSize = blobSize) - } - } - } - - @Test - fun `recreate accepts parts authored in any order`() { - val stored = manifest() - val fresh = manifest(parts = listOf(part(100, 250), part(0, 100)).map { it.copy(url = it.url + "&v=2") }) - val recreated = stored.reconcile(fresh, running = false, blobSize = blobSize) - assertEquals(2, recreated.parts.size) - } - - @Test - fun `tilesExactly covers the edge shapes`() { - assertTrue(ChunkedManifest.tilesExactly(listOf(part(0, 250)), 250)) - assertFalse(ChunkedManifest.tilesExactly(emptyList(), 0)) - assertFalse(ChunkedManifest.tilesExactly(listOf(part(0, 0)), 0)) // empty range - assertFalse(ChunkedManifest.tilesExactly(listOf(part(0, 250)), 300)) - } - - // MARK: - Create validation - - @Test - fun `create accepts parts that tile the blob exactly`() { - val m = manifest() - assertEquals(m, ChunkedManifest.validatedForCreate(m, blobSize)) - } - - @Test - fun `create rejects parts that do not tile the blob`() { - for ( - bad in listOf( - listOf(part(0, 100), part(150, 250)), // gap - listOf(part(0, 150), part(100, 250)), // overlap - listOf(part(50, 250)), // does not start at 0 - listOf(part(0, 200)), // short of the blob size - listOf(part(0, 100), part(100, 251)), // past the blob size - ) - ) { - assertThrows(ChunkedManifest.ReconcileException::class.java) { - ChunkedManifest.validatedForCreate(manifest(parts = bad), blobSize) - } - } - } - - @Test - fun `a rejected create writes no manifest, leaving the blob adoptable`() { - // startUpload validates AFTER takeOwnership moved the bytes. The throw - // propagates out of compute before a save. Thus the blob sits ownerless at - // its path. That is exactly what takeOwnership's orphan branch adopts on - // the corrected retry. - val store = ChunkedManifestStore(tmp.newFolder()) - store.blobFile("u1").apply { parentFile!!.mkdirs() }.writeText("owned bytes") - assertThrows(ChunkedManifest.ReconcileException::class.java) { - store.compute("u1") { ChunkedManifest.validatedForCreate(manifest(), 999L) } - } - assertNull(store.load("u1")) - assertTrue(store.blobFile("u1").exists()) - } - - // MARK: - Store - - @Test - fun `save then load round-trips, across store instances`() { - val dir = tmp.newFolder() - val m = manifest().withPartAccepted(1) - ChunkedManifestStore(dir).save(m) - // A new instance over the same dir is what a process relaunch looks like. - assertEquals(m, ChunkedManifestStore(dir).load("u1")) - } - - @Test - fun `load returns null for an unknown id`() { - assertNull(ChunkedManifestStore(tmp.newFolder()).load("nope")) - } - - @Test - fun `update persists the transformed manifest`() { - val store = ChunkedManifestStore(tmp.newFolder()) - store.save(manifest()) - val updated = store.update("u1") { it.withPartAccepted(0) } - assertTrue(updated!!.parts[0].accepted) - assertTrue(store.load("u1")!!.parts[0].accepted) - } - - @Test - fun `update of a missing manifest returns null`() { - assertNull(ChunkedManifestStore(tmp.newFolder()).update("nope") { it }) - } - - @Test - fun `compute creates when no manifest exists`() { - val store = ChunkedManifestStore(tmp.newFolder()) - val created = store.compute("u1") { existing -> - assertNull(existing) - manifest() - } - assertEquals(created, store.load("u1")) - } - - @Test - fun `compute holds the store lock across load, transform, and save`() { - // The startUpload reconcile and a running worker's markAccepted race. If - // the lock did not span all three steps, the update below could land - // between compute's load and save, and it would be erased from disk. When - // they are serialized, both effects must survive. - val store = ChunkedManifestStore(tmp.newFolder()) - store.save(manifest()) - val inTransform = CountDownLatch(1) - val computing = Thread { - store.compute("u1") { existing -> - inTransform.countDown() - Thread.sleep(300) // hold the lock with load done and save not yet run - existing!!.copy(expiresAt = 99_000) - } - }.apply { start() } - assertTrue(inTransform.await(5, TimeUnit.SECONDS)) - val updating = Thread { store.update("u1") { it.withPartAccepted(0) } }.apply { start() } - computing.join() - updating.join() - val final = store.load("u1")!! - assertEquals(99_000, final.expiresAt) - assertTrue(final.parts[0].accepted) - } - - @Test - fun `a throwing compute transform propagates and writes nothing`() { - val store = ChunkedManifestStore(tmp.newFolder()) - store.save(manifest()) - assertThrows(ChunkedManifest.ReconcileException::class.java) { - store.compute("u1") { throw ChunkedManifest.ReconcileException("rejected") } - } - assertEquals(manifest(), store.load("u1")) - } - - @Test - fun `contains tracks save and remove`() { - val store = ChunkedManifestStore(tmp.newFolder()) - assertFalse(store.contains("u1")) - store.save(manifest()) - assertTrue(store.contains("u1")) - store.remove("u1") - assertFalse(store.contains("u1")) - } - - @Test - fun `remove deletes the manifest and the blob`() { - val store = ChunkedManifestStore(tmp.newFolder()) - store.save(manifest()) - store.blobFile("u1").writeText("bytes") - store.remove("u1") - assertNull(store.load("u1")) - assertFalse(store.blobFile("u1").exists()) - } - - @Test - fun `remove of an unknown id is a no-op`() { - ChunkedManifestStore(tmp.newFolder()).remove("simple-upload-id") - } - - @Test - fun `ids with filesystem-hostile characters round-trip`() { - val store = ChunkedManifestStore(tmp.newFolder()) - val id = "a/b:c dü..\\e" - store.save(manifest(id = id)) - assertEquals(id, store.load(id)!!.id) - store.remove(id) - assertNull(store.load(id)) - } - - @Test - fun `a corrupt manifest reads as absent, not fatal`() { - val dir = tmp.newFolder() - val store = ChunkedManifestStore(dir) - store.save(manifest()) - File(File(dir, dir.list()!!.first()), "manifest.json").writeText("{not json") - assertNull(store.load("u1")) - assertEquals(emptyList(), store.all()) - } - - @Test - fun `all lists every stored manifest`() { - val store = ChunkedManifestStore(tmp.newFolder()) - store.save(manifest(id = "u1")) - store.save(manifest(id = "u2").withPartAccepted(0)) - assertEquals(setOf("u1", "u2"), store.all().map { it.id }.toSet()) - } -} diff --git a/android/src/test/java/ai/openspace/backgroundupload/ChunkedPartsTest.kt b/android/src/test/java/ai/openspace/backgroundupload/ChunkedPartsTest.kt new file mode 100644 index 00000000..e7ba858e --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/ChunkedPartsTest.kt @@ -0,0 +1,56 @@ +package ai.openspace.backgroundupload + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test + +class ChunkedPartsTest { + + @Test + fun `tilesExactly covers the edge shapes`() { + assertTrue(ChunkedParts.tilesExactly(listOf(part(0, 100), part(100, 250)), 250)) + assertTrue(ChunkedParts.tilesExactly(listOf(part(100, 250), part(0, 100)), 250)) // any order + assertFalse(ChunkedParts.tilesExactly(emptyList(), 0)) + assertFalse(ChunkedParts.tilesExactly(listOf(part(0, 0)), 0)) // empty range + assertFalse(ChunkedParts.tilesExactly(listOf(part(0, 100), part(150, 250)), 250)) // gap + assertFalse(ChunkedParts.tilesExactly(listOf(part(0, 150), part(100, 250)), 250)) // overlap + assertFalse(ChunkedParts.tilesExactly(listOf(part(50, 250)), 250)) // not from 0 + assertFalse(ChunkedParts.tilesExactly(listOf(part(0, 200)), 250)) // short + assertFalse(ChunkedParts.tilesExactly(listOf(part(0, 251)), 250)) // past the end + } + + @Test + fun `the same parts in another order are the same upload`() { + val a = listOf(part(0, 100), part(100, 250)) + assertTrue(ChunkedParts.sameParts(a, a.reversed())) + // Headers are not compared: a resume sends fresh ones. + assertTrue(ChunkedParts.sameParts(a, a.map { it.copy(headers = mapOf("X" to "new")) })) + } + + @Test + fun `new urls or a new split are different parts`() { + val a = listOf(part(0, 100), part(100, 250)) + assertFalse(ChunkedParts.sameParts(a, listOf(part(0, 100, url = "https://v2/1"), part(100, 250)))) + assertFalse(ChunkedParts.sameParts(a, listOf(part(0, 120), part(120, 250)))) + assertFalse(ChunkedParts.sameParts(a, listOf(part(0, 250)))) + } + + @Test + fun `accepted flags follow the range, not the index`() { + val stored = listOf(part(0, 100, accepted = true), part(100, 250)) + val incoming = listOf(part(100, 250), part(0, 100)) + val carried = ChunkedParts.carryAccepted(stored, incoming) + assertTrue(carried.first { it.start == 0L }.accepted) + assertFalse(carried.first { it.start == 100L }.accepted) + } + + @Test + fun `byte math and pending indexes`() { + val parts = listOf(part(0, 100, accepted = true), part(100, 250)) + assertEquals(250, ChunkedParts.totalBytes(parts)) + assertEquals(100, ChunkedParts.acceptedBytes(parts)) + assertEquals(listOf(1), ChunkedParts.pendingIndexes(parts)) + assertEquals(emptyList(), ChunkedParts.pendingIndexes(ChunkedParts.withAccepted(parts, 1))) + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/ChunkedWorkerGateTest.kt b/android/src/test/java/ai/openspace/backgroundupload/ChunkedWorkerGateTest.kt deleted file mode 100644 index 0b65d54b..00000000 --- a/android/src/test/java/ai/openspace/backgroundupload/ChunkedWorkerGateTest.kt +++ /dev/null @@ -1,57 +0,0 @@ -package ai.openspace.backgroundupload - -import org.junit.After -import org.junit.Assert.assertFalse -import org.junit.Assert.assertTrue -import org.junit.Test - -class ChunkedWorkerGateTest { - private val a = Any() - private val b = Any() - - @After - fun tearDown() { - // The gate is a process-wide singleton. Leave nothing for other tests. - ChunkedWorkerGate.release("u1", a) - ChunkedWorkerGate.release("u1", b) - ChunkedWorkerGate.release("u2", a) - } - - @Test - fun `a second worker for the same id must wait`() { - assertTrue(ChunkedWorkerGate.tryAcquire("u1", a)) - // The replacement worker after a cancel-then-start: it must not run a part - // PUT while the cancelled worker still holds the id. - assertFalse(ChunkedWorkerGate.tryAcquire("u1", b)) - ChunkedWorkerGate.release("u1", a) - assertTrue(ChunkedWorkerGate.tryAcquire("u1", b)) - } - - @Test - fun `reacquiring with the same token is idempotent`() { - assertTrue(ChunkedWorkerGate.tryAcquire("u1", a)) - assertTrue(ChunkedWorkerGate.tryAcquire("u1", a)) - } - - @Test - fun `a stale release cannot evict a successor`() { - assertTrue(ChunkedWorkerGate.tryAcquire("u1", a)) - ChunkedWorkerGate.release("u1", a) - assertTrue(ChunkedWorkerGate.tryAcquire("u1", b)) - ChunkedWorkerGate.release("u1", a) // the old worker's finally, arriving late - assertTrue(ChunkedWorkerGate.isRunning("u1")) - assertFalse(ChunkedWorkerGate.tryAcquire("u1", a)) - } - - @Test - fun `ids are independent and isRunning tracks the holder`() { - assertFalse(ChunkedWorkerGate.isRunning("u1")) - assertTrue(ChunkedWorkerGate.tryAcquire("u1", a)) - assertTrue(ChunkedWorkerGate.isRunning("u1")) - assertFalse(ChunkedWorkerGate.isRunning("u2")) - assertTrue(ChunkedWorkerGate.tryAcquire("u2", a)) - ChunkedWorkerGate.release("u1", a) - assertFalse(ChunkedWorkerGate.isRunning("u1")) - assertTrue(ChunkedWorkerGate.isRunning("u2")) - } -} diff --git a/android/src/test/java/ai/openspace/backgroundupload/EntryParsingTest.kt b/android/src/test/java/ai/openspace/backgroundupload/EntryParsingTest.kt new file mode 100644 index 00000000..5ed1c534 --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/EntryParsingTest.kt @@ -0,0 +1,188 @@ +package ai.openspace.backgroundupload + +import com.facebook.react.bridge.JavaOnlyArray +import com.facebook.react.bridge.JavaOnlyMap +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertThrows +import org.junit.Test + +class EntryParsingTest { + + private fun entryMap(descriptor: JavaOnlyMap, varsJson: Any? = """{"n":1}""") = + JavaOnlyMap.of("id", "e1", "key", "note", "varsJson", varsJson, "descriptor", descriptor) + + private fun base(vararg extra: Any?) = + JavaOnlyMap.of("url", "https://example.com/items", "expiresAt", 9_000.0, *extra) + + @Test + fun `a JSON POST parses with defaults and keeps the JSON text as JS wrote it`() { + val text = """{"text":"hi","n":1,"status":null}""" + val p = EntryParsing.parse(entryMap(base("dataJson", text), varsJson = """{"b":2,"a":null}""")) + assertEquals("e1", p.id) + assertEquals("note", p.key) + assertEquals("""{"b":2,"a":null}""", p.varsJson) // key order and null values survive + assertEquals(9_000L, p.expiresAt) + assertEquals("POST", p.descriptor.method) + assertEquals(text, p.descriptor.dataJson) + assertEquals(StagedBody.JSON, p.descriptor.bodyKind) + } + + @Test + fun `null vars cross as the text null`() { + assertEquals("null", EntryParsing.parse(entryMap(base(), varsJson = "null")).varsJson) + } + + @Test + fun `a dataJson of null is a real JSON body`() { + val d = EntryParsing.parse(entryMap(base("dataJson", "null"))).descriptor + assertEquals("null", d.dataJson) + assertEquals(StagedBody.JSON, d.bodyKind) + } + + @Test + fun `no dataJson is no body`() { + val d = EntryParsing.parse(entryMap(base())).descriptor + assertNull(d.dataJson) + assertEquals(StagedBody.NONE, d.bodyKind) + } + + @Test + fun `a missing or malformed varsJson or dataJson is rejected`() { + val cases = listOf( + entryMap(base(), varsJson = null), + entryMap(base(), varsJson = JavaOnlyMap.of("n", 1.0)), // the old object form + entryMap(base(), varsJson = "{n:1}"), // lenient JSON + entryMap(base(), varsJson = """{"n":1} trailing"""), + entryMap(base("dataJson", "")), + entryMap(base("dataJson", "{'a':1}")), + entryMap(base("dataJson", JavaOnlyMap.of("a", 1.0))), + entryMap(base("data", JavaOnlyMap.of("a", 1.0))), // the old data form would send no body + ) + cases.forEach { m -> + assertThrows("$m", EntryParsing.InvalidEntryException::class.java) { EntryParsing.parse(m) } + } + } + + @Test + fun `chunked parts, headers, accept, retry, and android parse`() { + val d = JavaOnlyMap.of( + "method", "PUT", + "file", "file:///data/a%20b.bin", + "expiresAt", 9_000.0, + "headers", JavaOnlyMap.of("Content-Type", "video/mp4", "X-N", 5.0), + "parts", JavaOnlyArray.of( + JavaOnlyMap.of("url", "https://s3/1", "range", JavaOnlyMap.of("start", 0.0, "end", 10.0)), + JavaOnlyMap.of( + "url", "https://s3/2", "headers", JavaOnlyMap.of("Content-Range", "10-19"), + "range", JavaOnlyMap.of("start", 10.0, "end", 20.0), + ), + ), + "accept", JavaOnlyArray.of(JavaOnlyMap.of("status", 409.0, "bodyIncludes", "already completed")), + "retry", JavaOnlyMap.of( + "backoff", JavaOnlyMap.of("baseMs", 50.0), + "terminalHttp", JavaOnlyMap.of("exempt", JavaOnlyArray()), + ), + "android", JavaOnlyMap.of("noNotification", true), + ) + val parsed = EntryParsing.parse(entryMap(d)).descriptor + assertEquals("/data/a b.bin", parsed.file) + assertEquals(StagedBody.CHUNKED, parsed.bodyKind) + assertEquals(listOf(Part("https://s3/1", mapOf(), 0, 10), Part("https://s3/2", mapOf("Content-Range" to "10-19"), 10, 20)), parsed.parts) + assertEquals(mapOf("Content-Type" to "video/mp4", "X-N" to "5"), parsed.headers) + assertEquals(listOf(UploadOutcome.AcceptRule(409, "already completed")), parsed.accept) + assertEquals(RetryOverride(50, null, null, emptyList()), parsed.retry) + assertEquals(true, parsed.noNotification) + assertEquals("https://s3/2", parsed.reportUrl) + } + + @Test + fun `form parts parse with exactly one of string or path`() { + val d = base( + "form", JavaOnlyArray.of( + JavaOnlyMap.of("name", "meta", "contentType", "application/json", "string", "{}"), + JavaOnlyMap.of("name", "photo", "contentType", "image/jpeg", "path", "/p.jpg", "fileName", "p.jpg"), + ), + ) + assertEquals( + listOf( + FormPart("meta", "application/json", "{}", null, null), + FormPart("photo", "image/jpeg", null, "/p.jpg", "p.jpg"), + ), + EntryParsing.parse(entryMap(d)).descriptor.form, + ) + val both = base("form", JavaOnlyArray.of(JavaOnlyMap.of("name", "x", "contentType", "t", "string", "s", "path", "/p"))) + assertThrows(EntryParsing.InvalidEntryException::class.java) { EntryParsing.parse(entryMap(both)) } + } + + @Test + fun `what native can not run is rejected`() { + val cases = listOf( + JavaOnlyMap.of("url", "https://example.com"), // no expiresAt + JavaOnlyMap.of("expiresAt", 1.0), // no url and no parts + base("dataJson", "1", "file", "/a"), // two body kinds + base("method", "GET", "dataJson", "1"), // GET with a body + base("method", "GET", "dataJson", "null"), // GET with the JSON body null + base("method", "GET", "file", "/a"), + base("method", "TRACE"), + JavaOnlyMap.of("url", "not a url", "expiresAt", 1.0), + base("headers", JavaOnlyMap.of("Bad\nName", "v")), + base("parts", JavaOnlyArray.of(JavaOnlyMap.of("url", "https://s3/1", "range", JavaOnlyMap.of("start", 0.0, "end", 1.0)))), // parts without file + ) + cases.forEach { d -> + assertThrows("$d", EntryParsing.InvalidEntryException::class.java) { EntryParsing.parse(entryMap(d)) } + } + assertThrows(EntryParsing.InvalidEntryException::class.java) { + EntryParsing.parse(JavaOnlyMap.of("key", "k", "descriptor", base())) + } + } + + @Test + fun `a GET with no body parses`() { + assertEquals("GET", EntryParsing.parse(entryMap(base("method", "GET"))).descriptor.method) + } + + @Test + fun `a bad header value is rejected with its name and offset, never its value`() { + val secret = "Bearer s3cr3t-token" + val e = assertThrows(EntryParsing.InvalidEntryException::class.java) { + EntryParsing.parse(entryMap(base("headers", JavaOnlyMap.of("Authorization", "$secret\n")))) + } + assertEquals("headers: the value of header 'Authorization' has an invalid character at offset ${secret.length}", e.message) + assertFalse(e.message!!.contains("s3cr3t")) + } + + @Test + fun `a bad header name is rejected with the valid part before the offset only`() { + val e = assertThrows(EntryParsing.InvalidEntryException::class.java) { + EntryParsing.requireValidHeaders(mapOf("Authorization: Bearer s3cr3t" to "v"), "headers") + } + assertEquals("headers: the header name that starts 'Authorization:' has an invalid character at offset 14", e.message) + assertFalse(e.message!!.contains("s3cr3t")) + assertThrows(EntryParsing.InvalidEntryException::class.java) { + EntryParsing.requireValidHeaders(mapOf("" to "v"), "headers") + } + } + + @Test + fun `the header check accepts what OkHttp sends`() { + EntryParsing.requireValidHeaders(mapOf("X-Tab" to "a\tb", "X-Tilde" to "~!#", "Content-Range" to "bytes 0-9/10"), "headers") + okhttp3.Headers.Builder().add("X-Tab", "a\tb").add("X-Tilde", "~!#") + } + + @Test + fun `an updateHeaders patch is checked like descriptor headers`() { + assertEquals(mapOf("Authorization" to "Bearer new"), EntryParsing.headerPatch(JavaOnlyMap.of("Authorization", "Bearer new"))) + val e = assertThrows(EntryParsing.InvalidEntryException::class.java) { + EntryParsing.headerPatch(JavaOnlyMap.of("Authorization", "Bearer\nnew")) + } + assertFalse(e.message!!.contains("Bearer")) + } + + @Test + fun `file scheme stripping`() { + assertEquals("/a/b c.jpg", EntryParsing.stripFileScheme("file:///a/b%20c.jpg")) + assertEquals("/a/b.jpg", EntryParsing.stripFileScheme("/a/b.jpg")) + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/EntryRunTest.kt b/android/src/test/java/ai/openspace/backgroundupload/EntryRunTest.kt new file mode 100644 index 00000000..1e7e0511 --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/EntryRunTest.kt @@ -0,0 +1,464 @@ +package ai.openspace.backgroundupload + +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.runBlocking +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertThrows +import org.junit.Assert.assertTrue +import org.junit.Before +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import java.io.File +import java.io.IOException + +/** A scripted [TransferHost]: a manual clock, a send handler, and a log of what the run did. */ +internal class FakeHost(var clock: Long = 10_000L) : TransferHost { + val requests = mutableListOf() + val sleeps = mutableListOf() + val attempts = mutableListOf() + val progress = mutableListOf() + var handler: suspend (TransferRequest, (Long) -> Unit) -> UploadResponse = { _, _ -> UploadResponse(200, "ok", mapOf()) } + var onSleep: () -> Unit = {} + var network = ArrayDeque() + var timeout = false + var foregroundError: Throwable? = null + + override fun now() = clock + + override suspend fun sleep(ms: Long) { + sleeps += ms + clock += ms + onSleep() + } + + override suspend fun send(request: TransferRequest, onProgress: (Long) -> Unit): UploadResponse { + requests += request + return handler(request, onProgress) + } + + override fun connectivity(wifiOnly: Boolean) = network.removeFirstOrNull() ?: Connectivity.Ok + + override suspend fun foreground(entry: QueueEntry) { + foregroundError?.let { throw it } + } + + override fun progressStarted(id: String, total: Long, sent: Long) { + progress += "start:$total:$sent" + } + + override fun progress(id: String, sent: Long, total: Long) { + progress += "$sent" + } + + override fun progressEnded(id: String, completed: Boolean) { + progress += "end:$completed" + } + + override fun attempt(event: AttemptEvent) { + attempts += event + } + + override fun stoppedByTimeout() = timeout +} + +class EntryRunTest { + @get:Rule + val tmp = TemporaryFolder() + + private lateinit var store: QueueStore + private lateinit var journal: EventJournal + private lateinit var settings: QueueSettingsStore + private val events = RecordingEvents() + private val scheduler = FakeScheduler() + private val host = FakeHost() + private lateinit var ops: WorkerOps + private lateinit var controller: QueueController + + @Before + fun setUp() { + val root = tmp.newFolder("queue") + store = QueueStore(root, RequestIndex()) + journal = EventJournal(tmp.newFolder("journal"), retryLater = { _, _ -> }) + settings = QueueSettingsStore(File(root, "settings.json")) + ops = WorkerOps(store, journal, settings, events, scheduler) { host.clock } + controller = QueueController(store, journal, settings, events, scheduler, { false }, { host.clock }) + controller.configureRetry(RetryDefaults(baseMs = 1_000, jitter = 0.0)) + journal.drain(Any()) + } + + private fun run(id: String = "e1") = runBlocking { EntryRun(id, store, ops, host).run() } + + private fun respond(vararg codes: Int) { + val queue = ArrayDeque(codes.toList()) + host.handler = { _, onProgress -> + onProgress(7) + UploadResponse(queue.removeFirst(), "body", mapOf()) + } + } + + private fun outcome() = journal.unacknowledged().single() + + // MARK: - simple + + @Test + fun `an accepted response settles completed after one write-ahead attempt`() { + controller.enqueue(parsed()) + respond(200) + run() + val e = store.load("e1")!! + assertEquals(EntryState.COMPLETED, e.state) + assertEquals(1, e.attempts) + val request = host.requests.single() + assertEquals(e.lastRequestId, request.headers["X-Request-Id"]) + assertEquals("Bearer old", request.headers["Authorization"]) + assertEquals("application/json", request.headers["Content-Type"]) + assertEquals(7L, request.body!!.contentLength()) + assertEquals(listOf("completed"), host.attempts.map { it.outcome }) + assertEquals(EventJournal.KIND_COMPLETED, outcome().kind) + assertEquals(200, outcome().response!!.status) + assertEquals(listOf("start:7:0", "7", "end:true"), host.progress) + } + + @Test + fun `a transient response waits a short backoff in place, with nextAttemptAt on the running row`() { + controller.enqueue(parsed()) + respond(503, 200) + run() + assertEquals(listOf(1_000L), host.sleeps) + assertEquals(listOf("error", "completed"), host.attempts.map { it.outcome }) + assertEquals("http", host.attempts[0].errorKind) + assertEquals(503, host.attempts[0].httpCode) + val waiting = events.rows.first { it.toMap().containsKey("nextAttemptAt") } + assertEquals("running", waiting.state) + assertEquals(11_000.0, waiting.toMap()["nextAttemptAt"]) + assertEquals(2, store.load("e1")!!.attempts) + assertEquals(listOf("start:7:0", "7", "0", "7", "end:true"), host.progress) // bytes reset between attempts + } + + @Test + fun `a backoff longer than 30 s releases the run back to queued with a wake`() { + controller.configureRetry(RetryDefaults(baseMs = 60_000, jitter = 0.0)) + controller.enqueue(parsed()) + respond(503) + run() + val e = store.load("e1")!! + assertEquals(EntryState.QUEUED, e.state) + assertEquals(host.clock + 60_000, e.nextAttemptAt) + assertEquals(1, e.backoffStreak) + assertEquals(listOf("e1" to host.clock + 60_000), scheduler.wakes) + assertEquals(emptyList(), host.sleeps) + assertEquals("end:false", host.progress.last()) + assertEquals(emptyList(), journal.unacknowledged()) + } + + @Test + fun `a transport failure is a network attempt and retries`() { + controller.enqueue(parsed()) + var calls = 0 + host.handler = { _, _ -> if (calls++ == 0) throw IOException("reset") else UploadResponse(200, "", mapOf()) } + run() + assertEquals(listOf("network", null), host.attempts.map { it.errorKind }) + assertEquals("reset", host.attempts[0].errorMessage) + assertEquals(EntryState.COMPLETED, store.load("e1")!!.state) + } + + @Test + fun `a terminal response settles error http with the response and the live bytes`() { + controller.enqueue(parsed()) + respond(400) + run() + val r = outcome() + assertEquals(EventJournal.KIND_ERROR, r.kind) + assertEquals("http", r.errorKind) + assertEquals(400, r.response!!.status) + assertEquals(7, r.bytesSent) + assertEquals(7, store.load("e1")!!.bytesSent) + assertEquals("end:false", host.progress.last()) + } + + @Test + fun `a staged body gone before the run settles error file with no request`() { + controller.enqueue(parsed()) + store.bodyFile(store.load("e1")!!)!!.delete() + run() + assertEquals("file", outcome().errorKind) + assertEquals(emptyList(), host.requests) + } + + @Test + fun `a body that vanishes mid-send settles error file, and the attempt says file`() { + controller.enqueue(parsed()) + host.handler = { _, _ -> + store.bodyFile(store.load("e1")!!)!!.delete() + throw IOException("ENOENT") + } + run() + assertEquals("file", outcome().errorKind) + assertEquals(listOf("file"), host.attempts.map { it.errorKind }) + } + + @Test + fun `a 401 parks the entry and wakes it at expiry`() { + controller.enqueue(parsed(expiresAt = 90_000)) + respond(401) + run() + assertEquals(EntryState.AWAITING_AUTH, store.load("e1")!!.state) + assertEquals(listOf("e1" to 90_000L), scheduler.wakes) + assertEquals(listOf("error"), host.attempts.map { it.outcome }) + assertEquals(emptyList(), journal.unacknowledged()) + } + + @Test + fun `a 401 with newer headers from updateHeaders mid-flight re-issues at once with them`() { + controller.enqueue(parsed()) + var calls = 0 + host.handler = { _, _ -> + if (calls++ == 0) { + controller.updateHeaders(mapOf("Authorization" to "Bearer new")) + UploadResponse(401, "", mapOf()) + } else UploadResponse(200, "", mapOf()) + } + run() + assertEquals(listOf("Bearer old", "Bearer new"), host.requests.map { it.headers["Authorization"] }) + assertEquals(emptyList(), host.sleeps) + assertEquals(EntryState.COMPLETED, store.load("e1")!!.state) + } + + @Test + fun `the expiry wake of a parked entry settles expired and keeps the stored bytes`() { + controller.enqueue(parsed(expiresAt = 20_000)) + store.save(store.load("e1")!!.copy(state = EntryState.AWAITING_AUTH, bytesSent = 3)) + run() + assertEquals(EntryState.AWAITING_AUTH, store.load("e1")!!.state) // not yet expired + host.clock = 20_000 + run() + assertEquals("expired", outcome().errorKind) + assertEquals(3, outcome().bytesSent) + assertEquals(emptyList(), host.requests) + } + + @Test + fun `an entry past expiresAt settles error expired`() { + controller.enqueue(parsed(expiresAt = 5_000)) + run() + assertEquals("expired", outcome().errorKind) + assertEquals(emptyList(), host.requests) + } + + @Test + fun `a paused entry past expiresAt stays paused, and settles expired at resume`() { + controller.enqueue(parsed(expiresAt = 20_000)) + controller.pause() + host.clock = 30_000 + run() // a wake that fires during the pause + assertEquals(EntryState.PAUSED, store.load("e1")!!.state) + assertEquals(emptyList(), journal.unacknowledged()) + controller.resume() + assertEquals(EntryState.QUEUED, store.load("e1")!!.state) + run() + assertEquals("expired", outcome().errorKind) + assertEquals(EntryState.ERROR, store.load("e1")!!.state) + } + + @Test + fun `a failure that lands after pause is not an outcome`() { + controller.enqueue(parsed()) + host.handler = { _, _ -> + controller.pause() + UploadResponse(400, "", mapOf()) + } + run() + assertEquals(EntryState.PAUSED, store.load("e1")!!.state) + assertEquals(emptyList(), journal.unacknowledged()) + } + + @Test + fun `an accepted response that lands after pause settles completed`() { + controller.enqueue(parsed()) + host.handler = { _, _ -> + controller.pause() + UploadResponse(200, "", mapOf()) + } + run() + assertEquals(EntryState.COMPLETED, store.load("e1")!!.state) + } + + @Test + fun `a transient response after cancel stops the run with only the cancel outcome`() { + controller.enqueue(parsed()) + host.handler = { _, _ -> + controller.cancel("e1") + UploadResponse(503, "", mapOf()) + } + run() + assertEquals(EventJournal.KIND_CANCELLED, outcome().kind) + assertEquals("end:false", host.progress.last()) + } + + @Test + fun `a system stop moves the entry back to queued with no outcome and no attempt event`() { + controller.enqueue(parsed()) + host.handler = { _, _ -> throw CancellationException("stopped") } + assertThrows(CancellationException::class.java) { run() } + val e = store.load("e1")!! + assertEquals(EntryState.QUEUED, e.state) + assertNull(e.nextAttemptAt) + assertEquals(emptyList(), host.attempts) + assertEquals(emptyList(), journal.unacknowledged()) + assertEquals("end:false", host.progress.last()) + } + + @Test + fun `a timeout stop takes one more backoff step instead of restarting at once`() { + controller.enqueue(parsed()) + store.save(store.load("e1")!!.copy(backoffStreak = 2)) + host.timeout = true + host.handler = { _, _ -> throw CancellationException("timeout") } + assertThrows(CancellationException::class.java) { run() } + val e = store.load("e1")!! + assertEquals(EntryState.QUEUED, e.state) + assertEquals(3, e.backoffStreak) + assertEquals(host.clock + 4_000, e.nextAttemptAt) // base 1 s * 2^(3-1) + assertEquals(listOf("e1" to host.clock + 4_000), scheduler.wakes) + } + + @Test + fun `a store write that fails mid-run throws with no outcome`() { + controller.enqueue(parsed()) + respond(503, 200) + val dir = store.entryDir("e1") + host.onSleep = { dir.setWritable(false) } // the next attempt's write-ahead fails + try { + assertThrows(IOException::class.java) { run() } + } finally { + dir.setWritable(true) + } + assertEquals(emptyList(), journal.unacknowledged()) + assertEquals(1, host.requests.size) + } + + @Test + fun `an unexpected error settles error unknown`() { + controller.enqueue(parsed()) + host.foregroundError = IllegalStateException("boom") + run() + assertEquals("unknown", outcome().errorKind) + assertEquals("boom", outcome().message) + assertEquals(listOf("end:false"), host.progress) + } + + @Test + fun `a re-run after a lost settle write applies the record and sends nothing`() { + controller.enqueue(parsed()) + store.save(store.load("e1")!!.copy(state = EntryState.RUNNING)) + journal.append(record("00000000-0000-0000-0000-000000000041")) + run() + assertEquals(EntryState.COMPLETED, store.load("e1")!!.state) + assertEquals(emptyList(), host.requests) + } + + @Test + fun `a short remaining backoff is slept out before the run, a long one is left to the wake`() { + controller.enqueue(parsed()) + store.save(store.load("e1")!!.copy(nextAttemptAt = host.clock + 5_000)) + run() + assertEquals(listOf(5_000L), host.sleeps) + assertEquals(EntryState.COMPLETED, store.load("e1")!!.state) + + controller.enqueue(parsed(id = "later")) + store.save(store.load("later")!!.copy(nextAttemptAt = host.clock + 60_000)) + run("later") + assertEquals(EntryState.QUEUED, store.load("later")!!.state) + assertEquals(1, host.requests.size) + } + + @Test + fun `no usable network polls until it returns`() { + controller.enqueue(parsed()) + host.network = ArrayDeque(listOf(Connectivity.NoWifi, Connectivity.NoInternet)) + run() + assertEquals(listOf(EntryRun.CONNECTIVITY_POLL_MS, EntryRun.CONNECTIVITY_POLL_MS), host.sleeps) + assertEquals(EntryState.COMPLETED, store.load("e1")!!.state) + } + + // MARK: - chunked + + private fun chunked(size: Int = 30) { + val src = File(tmp.newFolder(), "video.bin").apply { writeBytes(ByteArray(size) { it.toByte() }) } + controller.enqueue(parsed(descriptor = desc(url = null, method = "PUT", file = src.path, + parts = listOf(part(0, 10), part(10, 20), part(20, 30))))) + } + + @Test + fun `every part accepted settles completed with no status, one attempt per part`() { + chunked() + run() + val e = store.load("e1")!! + assertEquals(EntryState.COMPLETED, e.state) + assertEquals(3, e.attempts) + assertTrue(e.descriptor!!.parts!!.all { it.accepted }) + assertNull(outcome().response!!.status) + val byUrl = host.requests.associateBy { it.url } + assertEquals("20-29", byUrl["https://example.com/part?start=20"]!!.headers["Content-Range"]) + assertEquals(10L, byUrl["https://example.com/part?start=20"]!!.body!!.contentLength()) + assertEquals(listOf(0, 1, 2), host.attempts.map { it.partIndex }.sortedBy { it }) + } + + @Test + fun `a part that fails terminally settles error with its index, and the other parts stop`() { + chunked() + host.handler = { r, _ -> + UploadResponse(if (r.url.endsWith("start=10")) 400 else 200, "", mapOf()) + } + run() + val r = outcome() + assertEquals("http", r.errorKind) + assertEquals(1, r.partIndex) + assertEquals("HTTP 400 on part 1", r.message) + assertEquals("https://example.com/part?start=10", r.url) + val accepted = store.load("e1")!!.descriptor!!.parts!!.map { it.accepted } + assertFalse(accepted[1]) + } + + @Test + fun `a part 503 backs off in that part while the others go on`() { + chunked() + var failed = false + host.handler = { r, _ -> + if (r.url.endsWith("start=0") && !failed) { + failed = true + UploadResponse(503, "", mapOf()) + } else UploadResponse(200, "", mapOf()) + } + run() + assertEquals(EntryState.COMPLETED, store.load("e1")!!.state) + assertEquals(4, host.requests.size) + assertEquals(listOf(1_000L), host.sleeps) + } + + @Test + fun `a part 401 parks the whole entry and keeps the accepted parts`() { + chunked() + host.handler = { r, _ -> + UploadResponse(if (r.url.endsWith("start=20")) 401 else 200, "", mapOf()) + } + run() + val e = store.load("e1")!! + assertEquals(EntryState.AWAITING_AUTH, e.state) + assertEquals(listOf(true, true, false), e.descriptor!!.parts!!.map { it.accepted }) + } + + @Test + fun `a failed chunked settle keeps the accepted bytes, not the in-flight ones`() { + chunked() + host.handler = { r, onProgress -> + onProgress(5) + UploadResponse(if (r.url.endsWith("start=20")) 400 else 200, "", mapOf()) + } + run() + assertEquals(20, outcome().bytesSent) + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/EntryTransitionsTest.kt b/android/src/test/java/ai/openspace/backgroundupload/EntryTransitionsTest.kt new file mode 100644 index 00000000..a40a156f --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/EntryTransitionsTest.kt @@ -0,0 +1,212 @@ +package ai.openspace.backgroundupload + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test + +class EntryTransitionsTest { + + @Test + fun `a settle on a cancelled entry or an older generation is not allowed`() { + for (accepted in listOf(true, false)) { + assertTrue(EntryTransitions.canSettle(entry(state = EntryState.RUNNING), 1, accepted)) + assertTrue(EntryTransitions.canSettle(entry(state = EntryState.AWAITING_AUTH), 1, accepted)) // the expiry wake + assertFalse(EntryTransitions.canSettle(entry(state = EntryState.CANCELLED), 1, accepted)) + assertFalse(EntryTransitions.canSettle(entry(state = EntryState.COMPLETED), 1, accepted)) + assertFalse(EntryTransitions.canSettle(entry(state = EntryState.RUNNING, generation = 2), 1, accepted)) + assertFalse(EntryTransitions.canSettle(null, 1, accepted)) + } + } + + @Test + fun `under pause only an accepted response settles`() { + assertTrue(EntryTransitions.canSettle(entry(state = EntryState.PAUSED), 1, accepted = true)) + assertFalse(EntryTransitions.canSettle(entry(state = EntryState.PAUSED), 1, accepted = false)) + } + + @Test + fun `a live entry with a record of its own generation settles from the newest one`() { + val live = entry(state = EntryState.RUNNING, generation = 2) + val older = record("00000000-0000-0000-0000-0000000000a1", generation = 2, at = 1) + val newest = record("00000000-0000-0000-0000-0000000000a2", kind = EventJournal.KIND_ERROR, generation = 2, at = 2) + val otherLife = record("00000000-0000-0000-0000-0000000000a3", generation = 1, at = 3) + val otherId = record("00000000-0000-0000-0000-0000000000a4", id = "x", generation = 2, at = 4) + val j = EntryTransitions.journaledSettle(live, listOf(older, newest, otherLife, otherId), now = 9)!! + assertEquals(EntryState.ERROR, j.entry.state) + assertEquals(newest.eventId, j.entry.settledEventId) + assertEquals(listOf(older.eventId), j.extraEventIds) + assertNull(EntryTransitions.journaledSettle(live, listOf(otherLife, otherId), 9)) + assertNull(EntryTransitions.journaledSettle(entry(state = EntryState.ERROR, generation = 2), listOf(newest), 9)) + } + + @Test + fun `resume returns to awaiting-auth only when the parked generation is current`() { + val paused = entry(state = EntryState.PAUSED, parkedGeneration = 3) + assertEquals(EntryState.AWAITING_AUTH, EntryTransitions.toResumed(paused, headerGeneration = 3, now = 9).state) + val stale = EntryTransitions.toResumed(paused, headerGeneration = 4, now = 9) + assertEquals(EntryState.QUEUED, stale.state) + assertNull(stale.parkedGeneration) + assertEquals(EntryState.QUEUED, EntryTransitions.toResumed(entry(state = EntryState.PAUSED), 0, 9).state) + } + + @Test + fun `pause keeps the parked generation and clears the wake time`() { + val p = EntryTransitions.toPaused(entry(state = EntryState.AWAITING_AUTH, parkedGeneration = 2, nextAttemptAt = 50), 9) + assertEquals(EntryState.PAUSED, p.state) + assertEquals(2, p.parkedGeneration) + assertNull(p.nextAttemptAt) + } + + @Test + fun `park, release, run, stop, settle`() { + val running = EntryTransitions.toRunning(entry(nextAttemptAt = 5), 9) + assertEquals(EntryState.RUNNING, running.state) + assertNull(running.nextAttemptAt) + + val parked = EntryTransitions.toParked(running.copy(backoffStreak = 4), 7, 10) + assertEquals(EntryState.AWAITING_AUTH, parked.state) + assertEquals(7, parked.parkedGeneration) + assertEquals(0, parked.backoffStreak) + + val released = EntryTransitions.toReleased(running, 99, 6, 11) + assertEquals(EntryState.QUEUED, released.state) + assertEquals(99L, released.nextAttemptAt) + assertEquals(6, released.backoffStreak) + + assertEquals(EntryState.QUEUED, EntryTransitions.toStopped(running, 12).state) + + val settled = EntryTransitions.toSettled(parked, EntryState.ERROR, "ev", 5, 13) + assertEquals("ev", settled.settledEventId) + assertNull(settled.parkedGeneration) + assertEquals(13, settled.updatedAt) + } + + @Test + fun `a short backoff keeps the row running and shows the time, and the next attempt clears it`() { + val waiting = EntryTransitions.toBackingOff(entry(state = EntryState.RUNNING), 9_000, 5) + assertEquals(EntryState.RUNNING, waiting.state) + assertEquals(9_000L, waiting.nextAttemptAt) + assertEquals(9_000.0, waiting.toRow().toMap()["nextAttemptAt"]) + val attempt = EntryTransitions.toAttempt(waiting, "req-2", 6) + assertNull(attempt.nextAttemptAt) + assertEquals(1, attempt.attempts) + assertEquals("req-2", attempt.lastRequestId) + assertFalse(attempt.toRow().toMap().containsKey("nextAttemptAt")) + } +} + +class EnqueueRulesTest { + private val body = desc(dataJson = """{"a":1}""") + private val other = desc(dataJson = """{"a":2}""") + + private fun decide(existing: QueueEntry?, incoming: Descriptor = body, hasRecord: Boolean = true, v9: LegacyManifest? = null) = + EnqueueRules.decide(existing, v9, incoming) { hasRecord } + + @Test + fun `the same-id table`() { + assertEquals(EnqueueRules.Action.Create, decide(null)) + val v9 = LegacyManifest("e1", listOf(part(0, 10)), emptyList()) + assertEquals(EnqueueRules.Action.AdoptV9(v9, 1), decide(null, v9 = v9)) + val legacy = entry(legacy = true, descriptor = null, body = null) + assertEquals(EnqueueRules.Action.Replace, decide(legacy)) + // A legacy row over a v9 manifest: adopt it, one generation up. + assertEquals(EnqueueRules.Action.AdoptV9(v9, 2), decide(legacy, v9 = v9)) + assertEquals(EnqueueRules.Action.ReEmit("ev"), decide(entry(state = EntryState.COMPLETED, settledEventId = "ev"))) + assertEquals(EnqueueRules.Action.Replace, decide(entry(state = EntryState.COMPLETED, settledEventId = "ev"), hasRecord = false)) + EntryState.values().filter { it != EntryState.COMPLETED }.forEach { state -> + assertEquals("$state", EnqueueRules.Action.Resume, decide(entry(state = state))) + } + assertEquals(EnqueueRules.Action.RejectRunning, decide(entry(state = EntryState.RUNNING), other)) + EntryState.values().filter { it != EntryState.RUNNING }.forEach { state -> + assertEquals("$state", EnqueueRules.Action.Replace, decide(entry(state = state), other)) + } + } + + @Test + fun `resume replaces the metadata, keeps the body and accepted parts`() { + val parts = listOf(part(0, 10, accepted = true), part(10, 20)) + val stored = entry( + state = EntryState.AWAITING_AUTH, + descriptor = desc(url = null, method = "PUT", file = "/f", parts = parts), + body = StagedBody(StagedBody.CHUNKED, "blob", null, 20), + attempts = 4, + parkedGeneration = 1, + ) + val incoming = parsed( + descriptor = desc(url = null, method = "PUT", file = "/f", headers = mapOf("Authorization" to "Bearer new"), + parts = parts.map { it.copy(accepted = false) }), + varsJson = """{"n":2}""", + expiresAt = 77, + ) + val next = EnqueueRules.resumed(stored, incoming, paused = false, headerGeneration = 5, now = 9) + assertEquals(EntryState.QUEUED, next.state) + assertEquals(mapOf("Authorization" to "Bearer new"), next.descriptor!!.headers) + assertTrue(next.descriptor!!.parts!![0].accepted) + assertEquals(10, next.bytesSent) + assertEquals(4, next.attempts) + assertEquals(77, next.expiresAt) + assertEquals("""{"n":2}""", next.varsJson) + assertEquals(5, next.headerGeneration) + assertNull(next.parkedGeneration) + assertEquals(1, next.generation) // a live entry keeps its life + } + + @Test + fun `resume of a settled entry reopens it with a fresh generation and attempts 0`() { + val settled = entry(state = EntryState.ERROR, settledEventId = "ev", generation = 2, attempts = 3) + val next = EnqueueRules.resumed(settled, parsed(), false, 0, 9) + assertEquals(EntryState.QUEUED, next.state) + assertEquals(3, next.generation) + assertEquals(0, next.attempts) + assertNull(next.settledEventId) + } + + @Test + fun `resume clears a pending backoff so the entry runs now`() { + val waiting = entry(state = EntryState.QUEUED, nextAttemptAt = 99_000).copy(backoffStreak = 6) + val next = EnqueueRules.resumed(waiting, parsed(), false, 0, 9) + assertNull(next.nextAttemptAt) + assertEquals(0, next.backoffStreak) + } + + @Test + fun `resume of a running entry stays running, and under pause becomes paused`() { + assertEquals(EntryState.RUNNING, EnqueueRules.resumed(entry(state = EntryState.RUNNING), parsed(), true, 0, 9).state) + assertEquals(EntryState.PAUSED, EnqueueRules.resumed(entry(state = EntryState.QUEUED), parsed(), true, 0, 9).state) + } + + @Test + fun `resume re-applies the json content type`() { + val next = EnqueueRules.resumed(entry(), parsed(descriptor = desc(dataJson = """{"a":1}""", headers = mapOf())), false, 0, 9) + assertEquals(mapOf("Content-Type" to "application/json"), next.descriptor!!.headers) + } + + @Test + fun `adopting v9 parts carries the flags only for the same parts`() { + val v9 = LegacyManifest("e1", listOf(part(0, 10, accepted = true), part(10, 20)), emptyList()) + assertTrue(EnqueueRules.adoptedParts(v9, listOf(part(0, 10), part(10, 20)))[0].accepted) + assertFalse(EnqueueRules.adoptedParts(v9, listOf(part(0, 20)))[0].accepted) + } + + @Test + fun `a copied body is staged outside the lock only when no worker can change the decision`() { + val json = desc(dataJson = """{"a":2}""") + val create = EnqueueRules.Action.Create + val replace = EnqueueRules.Action.Replace + assertEquals(1, EnqueueRules.preStageGeneration(null, create, json)) + assertEquals(1, EnqueueRules.preStageGeneration(null, create, desc(file = "/f"))) + assertEquals(3, EnqueueRules.preStageGeneration(entry(state = EntryState.ERROR, generation = 2), replace, json)) + assertEquals(2, EnqueueRules.preStageGeneration(entry(state = EntryState.PAUSED), replace, json)) + // A worker can take a queued entry meanwhile, and a running one is the worker's. + assertNull(EnqueueRules.preStageGeneration(entry(state = EntryState.QUEUED), replace, json)) + assertNull(EnqueueRules.preStageGeneration(entry(state = EntryState.RUNNING), replace, json)) + // A chunked move and a bodiless request stage under the lock. + assertNull(EnqueueRules.preStageGeneration(null, create, desc(url = null, file = "/f", parts = listOf(part(0, 10))))) + assertNull(EnqueueRules.preStageGeneration(null, create, desc())) + // Nothing to stage. + assertNull(EnqueueRules.preStageGeneration(entry(), EnqueueRules.Action.Resume, json)) + assertNull(EnqueueRules.preStageGeneration(entry(), EnqueueRules.Action.RejectRunning, json)) + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/EventJournalTest.kt b/android/src/test/java/ai/openspace/backgroundupload/EventJournalTest.kt index 811f1bd2..474c570c 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/EventJournalTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/EventJournalTest.kt @@ -2,103 +2,296 @@ package ai.openspace.backgroundupload import org.junit.Assert.assertEquals import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertThrows import org.junit.Assert.assertTrue import org.junit.Rule import org.junit.Test import org.junit.rules.TemporaryFolder import java.io.File +import java.io.IOException class EventJournalTest { @get:Rule val tmp = TemporaryFolder() - private fun entry(id: String, uploadId: String = "u1") = EventJournal.Entry( - eventId = id, - uploadId = uploadId, - type = "completed", - timestamp = System.currentTimeMillis(), - responseCode = 200, - responseBody = "ok", - responseHeaders = mapOf("x-a" to "b"), - ) + private val id1 = "00000000-0000-0000-0000-000000000001" + private val id2 = "00000000-0000-0000-0000-000000000002" + + private val listener = Any() + + @Test + fun `append then read returns the record`() { + val journal = EventJournal(tmp.newFolder()) + journal.drain(listener) + assertEquals(record(id1), journal.append(record(id1))) + assertEquals(listOf(record(id1)), journal.unacknowledged()) + } + + @Test + fun `append starts at 1 delivery with a listener and at 0 without one`() { + val journal = EventJournal(tmp.newFolder()) + assertFalse(journal.isListening()) + assertEquals(0, journal.append(record(id1)).deliveries) + assertEquals(listOf(1), journal.drain(listener).map { it.deliveries }) // the drain delivers it + assertTrue(journal.isListening()) + assertEquals(1, journal.append(record(id2)).deliveries) + } + + @Test + fun `a drain for a torn-down module sets no listener and counts nothing`() { + val journal = EventJournal(tmp.newFolder()) + journal.append(record(id1)) + assertEquals(emptyList(), journal.drain(listener) { false }) + assertFalse(journal.isListening()) + assertEquals(0, journal.find(id1)!!.deliveries) + } + + @Test + fun `listener is the owner of the last drain`() { + val journal = EventJournal(tmp.newFolder()) + assertNull(journal.listener()) + journal.drain(listener) + assertTrue(journal.listener() === listener) + val next = Any() + journal.drain(next) + assertTrue(journal.listener() === next) + } @Test - fun `append then read returns the entry`() { + fun `stopListening clears only its own listener`() { val journal = EventJournal(tmp.newFolder()) - journal.append(entry("e1")) - val events = journal.unacknowledged() - assertEquals(1, events.size) - assertEquals("e1", events[0].eventId) - assertEquals(200, events[0].responseCode) - assertEquals("ok", events[0].responseBody) + val next = Any() + journal.drain(listener) + journal.drain(next) // a reload: the next module drains before the old one is torn down + journal.stopListening(listener) + assertTrue(journal.isListening()) + journal.stopListening(next) + assertFalse(journal.isListening()) } @Test - fun `ack removes only the acked entry`() { + fun `redeliver counts a delivery only with a listener`() { val journal = EventJournal(tmp.newFolder()) - journal.append(entry("e1")) - journal.append(entry("e2")) - journal.ack(listOf("e1")) - assertEquals(listOf("e2"), journal.unacknowledged().map { it.eventId }) + journal.append(record(id1)) + assertNull(journal.redeliver(id1)) + assertEquals(0, journal.find(id1)!!.deliveries) + journal.drain(listener) // 1 + assertEquals(2, journal.redeliver(id1)!!.deliveries) } @Test - fun `entries survive a new journal instance over the same dir`() { + fun `ack removes only the acked record and is idempotent`() { + val journal = EventJournal(tmp.newFolder()) + journal.append(record(id1)) + journal.append(record(id2)) + journal.ack(listOf(id1, id1, "unknown")) + assertEquals(listOf(id2), journal.unacknowledged().map { it.eventId }) + } + + @Test + fun `ack ignores ids that are not event ids`() { + val dir = tmp.newFolder() + val outside = File(dir.parentFile, "precious.json").apply { writeText("x") } + EventJournal(dir).ack(listOf("../precious")) + assertTrue(outside.exists()) + } + + @Test + fun `records survive a new journal instance`() { val dir = tmp.newFolder() - EventJournal(dir).append(entry("e1")) + EventJournal(dir).append(record(id1)) assertEquals(1, EventJournal(dir).unacknowledged().size) } @Test - fun `oversized body is truncated and flagged`() { + fun `a body over 1 MB of UTF-8 is cut on a character and flagged`() { val journal = EventJournal(tmp.newFolder()) - val big = "x".repeat(EventJournal.MAX_BODY_CHARS + 100) - journal.append(entry("e1").copy(responseBody = big)) - val read = journal.unacknowledged()[0] - assertTrue(read.responseBodyTruncated) - assertTrue(read.responseBody!!.length <= EventJournal.MAX_BODY_CHARS) + // 2-byte characters, so a char cap would keep 2 MB. + val big = "\u00e9".repeat(BodyCap.SETTLED_MAX_BYTES) + journal.append(record(id1).copy(response = EventJournal.Response(200, null, big, false))) + val read = journal.unacknowledged()[0].response!! + assertTrue(read.bodyTruncated) + assertEquals(BodyCap.SETTLED_MAX_BYTES, read.body!!.toByteArray(Charsets.UTF_8).size) + assertEquals(BodyCap.SETTLED_MAX_BYTES / 2, read.body!!.length) } @Test - fun `corrupt file is skipped, not fatal`() { + fun `a response the stream cap cut stays flagged`() { + val r = EventJournal.Response.of(UploadResponse(500, "partial", mapOf(), truncated = true)) + assertEquals("partial", r.body) + assertTrue(r.bodyTruncated) + assertFalse(EventJournal.Response.of(UploadResponse(500, "whole", mapOf())).bodyTruncated) + } + + @Test + fun `a corrupt file is skipped`() { val dir = tmp.newFolder() val journal = EventJournal(dir) - journal.append(entry("e1")) - java.io.File(dir, "garbage.json").writeText("{not json") - assertEquals(1, journal.unacknowledged().size) + journal.append(record(id1)) + File(dir, "garbage.json").writeText("{not json") + File(dir, "partial.json").writeText("""{"eventId":"x"}""") + assertEquals(listOf(id1), journal.unacknowledged().map { it.eventId }) + } + + @Test + fun `records are ordered by time`() { + val journal = EventJournal(tmp.newFolder()) + journal.append(record(id1, at = 2_000)) + journal.append(record(id2, at = 1_000)) + assertEquals(listOf(id2, id1), journal.unacknowledged().map { it.eventId }) + } + + @Test + fun `append throws when it could not write and keeps nothing`() { + val journal = EventJournal(tmp.newFile()) // a file where the directory should be + assertThrows(IOException::class.java) { journal.append(record(id1)) } + assertEquals(emptyList(), journal.unacknowledged()) } @Test - fun `entries are ordered by timestamp`() { + fun `appendOrHold holds a record it could not write, and every read and ack sees it`() { + val tasks = mutableListOf Unit>>() + val dir = tmp.newFolder() + val journal = EventJournal(dir, retryLater = { delay, task -> tasks += delay to task }) + dir.setWritable(false) + try { + val held = journal.appendOrHold(record(id1)) + assertTrue(journal.isHeld(id1)) + assertEquals(listOf(held), journal.unacknowledged()) + assertEquals(held, journal.find(id1)) + assertEquals(listOf(id1), journal.forEntry("e1").map { it.eventId }) + assertEquals(1, journal.drain(listener).single().deliveries) + // The retry fails while the disk is full, and waits twice as long. + tasks.removeAt(0).also { (delay, task) -> assertEquals(EventJournal.RETRY_MS, delay); task() } + assertEquals(EventJournal.RETRY_MS * 2, tasks.single().first) + } finally { + dir.setWritable(true) + } + tasks.removeAt(0).second() + assertFalse(journal.isHeld(id1)) + assertTrue(File(dir, "$id1.json").exists()) + assertEquals(1, EventJournal(dir).find(id1)!!.deliveries) + assertEquals(emptyList Unit>>(), tasks) + } + + @Test + fun `an ack removes a held record`() { + val dir = tmp.newFolder() + val journal = EventJournal(dir, retryLater = { _, _ -> }) + dir.setWritable(false) + try { + journal.appendOrHold(record(id1)) + } finally { + dir.setWritable(true) + } + journal.ack(listOf(id1)) + assertFalse(journal.isHeld(id1)) + assertEquals(emptyList(), journal.unacknowledged()) + } + + @Test + fun `ackEntry removes every record of one entry`() { val journal = EventJournal(tmp.newFolder()) - journal.append(entry("late").copy(timestamp = 2000)) - journal.append(entry("early").copy(timestamp = 1000)) - assertEquals(listOf("early", "late"), journal.unacknowledged().map { it.eventId }) + journal.append(record(id1, id = "a")) + journal.append(record(id2, id = "a", generation = 2)) + journal.append(record("00000000-0000-0000-0000-000000000003", id = "b")) + journal.ackEntry("a") + assertEquals(listOf("b"), journal.unacknowledged().map { it.id }) } @Test - fun `append does not throw when the directory is unwritable`() { - // A regular file where a directory is expected: mkdirs() and every write fail. - val notADir = tmp.newFile() - val journal = EventJournal(notADir) - journal.append(entry("e1")) // must not throw - assertEquals(emptyList(), journal.unacknowledged().map { it.eventId }) + fun `prunes the oldest records beyond the cap`() { + val dir = tmp.newFolder() + val journal = EventJournal(dir, maxEntries = 3) + val ids = (1..4).map { "00000000-0000-0000-0000-00000000000$it" } + ids.take(3).forEachIndexed { i, id -> + journal.append(record(id)) + File(dir, "$id.json").setLastModified(1_000L * (i + 1)) + } + journal.append(record(ids[3])) + val left = journal.unacknowledged().map { it.eventId } + assertEquals(3, left.size) + assertFalse(left.contains(ids[0])) } @Test - fun `prunes the oldest entries beyond the cap`() { + fun `the prune never deletes a record a row names`() { val dir = tmp.newFolder() val journal = EventJournal(dir, maxEntries = 3) - // Stamp increasing mtimes so pruning order is deterministic. Each mtime is - // set before the next append, which is when pruning reads it. - journal.append(entry("e1")); File(dir, "e1.json").setLastModified(1000) - journal.append(entry("e2")); File(dir, "e2.json").setLastModified(2000) - journal.append(entry("e3")); File(dir, "e3.json").setLastModified(3000) - journal.append(entry("e4")) // 4th write trips the cap; oldest (e1) is dropped - - val ids = journal.unacknowledged().map { it.eventId } - assertEquals(3, ids.size) - assertFalse(ids.contains("e1")) - assertTrue(ids.contains("e4")) + val ids = (1..4).map { "00000000-0000-0000-0000-00000000000$it" } + ids.take(3).forEachIndexed { i, id -> + journal.append(record(id)) + File(dir, "$id.json").setLastModified(1_000L * (i + 1)) + } + // The oldest is named by a row; the next oldest goes instead. + journal.append(record(ids[3])) { setOf(ids[0]) } + assertEquals(setOf(ids[0], ids[2], ids[3]), journal.unacknowledged().map { it.eventId }.toSet()) + } + + @Test + fun `incrementDeliveries persists`() { + val dir = tmp.newFolder() + val journal = EventJournal(dir) + journal.drain(listener) + journal.append(record(id1)) + assertEquals(2, journal.incrementDeliveries(id1)!!.deliveries) + assertEquals(3, journal.incrementDeliveries(id1)!!.deliveries) + assertEquals(3, EventJournal(dir).find(id1)!!.deliveries) + assertNull(journal.incrementDeliveries(id2)) + } + + @Test + fun `forEntry filters by entry id`() { + val journal = EventJournal(tmp.newFolder()) + journal.append(record(id1, id = "a")) + journal.append(record(id2, id = "b")) + assertEquals(listOf(id2), journal.forEntry("b").map { it.eventId }) + } + + @Test + fun `the completed shape is SettledEvent`() { + val map = record(id1).toMap() + assertEquals( + listOf( + "eventId", "id", "key", "vars", "at", "attempts", "requestId", "deliveries", "state", + "bytesSent", "totalBytes", "url", "method", "kind", "response", + ), + map.keys.toList(), + ) + assertEquals(mapOf("n" to 1.0), map["vars"]) + assertEquals(mapOf("status" to 200.0, "headers" to mapOf(), "body" to "ok", "bodyTruncated" to false), map["response"]) + } + + @Test + fun `a chunked completion has a response with no status`() { + val map = record(id1).copy(response = null).toMap() + assertEquals(mapOf("bodyTruncated" to false), map["response"]) + } + + @Test + fun `the error shape nests errorKind, message, response, and partIndex`() { + val map = record(id1, kind = EventJournal.KIND_ERROR).copy( + partIndex = 2, + response = EventJournal.Response(404, null, "gone", false), + ).toMap() + assertEquals(2.0, map["partIndex"]) + assertEquals( + mapOf( + "errorKind" to "http", "message" to "HTTP 400", + "response" to mapOf("status" to 404.0, "body" to "gone", "bodyTruncated" to false), + "partIndex" to 2.0, + ), + map["error"], + ) + assertFalse(map.containsKey("response")) + } + + @Test + fun `the cancelled shape carries the reason`() { + val map = record(id1, kind = EventJournal.KIND_CANCELLED).toMap() + assertEquals("user", map["cancelReason"]) + assertFalse(map.containsKey("error")) + assertFalse(map.containsKey("response")) } } diff --git a/android/src/test/java/ai/openspace/backgroundupload/JsonBridgeTest.kt b/android/src/test/java/ai/openspace/backgroundupload/JsonBridgeTest.kt new file mode 100644 index 00000000..928540e7 --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/JsonBridgeTest.kt @@ -0,0 +1,84 @@ +package ai.openspace.backgroundupload + +import com.facebook.react.bridge.JavaOnlyArray +import com.facebook.react.bridge.JavaOnlyMap +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Assert.assertThrows +import org.junit.Test + +class JsonBridgeTest { + + @Test + fun `integral doubles print as integers, as JSON stringify does`() { + assertEquals("1", JsonBridge.numberText(1.0)) + assertEquals("-3", JsonBridge.numberText(-3.0)) + assertEquals("0", JsonBridge.numberText(-0.0)) + assertEquals("12345678901", JsonBridge.numberText(12_345_678_901.0)) + } + + @Test + fun `fractions and very large magnitudes keep a decimal form`() { + assertEquals("1.5", JsonBridge.numberText(1.5)) + assertEquals("0.1", JsonBridge.numberText(0.1)) + // Above 2^53 a double can not hold every integer, so it stays a double. + assertEquals(1e20, (JsonBridge.parse(JsonBridge.numberText(1e20)) as Double), 0.0) + } + + @Test + fun `JSON text parses to plain values`() { + val value = mapOf("a" to listOf(1.0, "x", true, null, mapOf("b" to 2.5)), "c" to mapOf()) + assertEquals(value, JsonBridge.parse("""{"a":[1,"x",true,null,{"b":2.5}],"c":{}}""")) + assertNull(JsonBridge.parse("null")) + } + + @Test + fun `malformed text throws`() { + assertThrows(Exception::class.java) { JsonBridge.parse("{\"a\":") } + assertThrows(Exception::class.java) { JsonBridge.parse("[1,") } + } + + @Test + fun `bridge maps read into plain values`() { + val map = JavaOnlyMap.of( + "n", 2.0, + "s", "x", + "b", false, + "z", null, + "m", JavaOnlyMap.of("k", 1.0), + "a", JavaOnlyArray.of(1.0, "y"), + ) + assertEquals( + mapOf("n" to 2.0, "s" to "x", "b" to false, "z" to null, "m" to mapOf("k" to 1.0), "a" to listOf(1.0, "y")), + JsonBridge.fromReadable(map), + ) + assertNull(JsonBridge.valueOf(map, "absent")) + } + + @Test + fun `plain values write to the bridge`() { + val out = JsonBridge.toWritableMap( + mapOf("n" to 1.0, "list" to listOf("a", 2.0), "nested" to mapOf("k" to true), "none" to null), + ::JavaOnlyMap, + ::JavaOnlyArray, + ) as JavaOnlyMap + assertEquals(1.0, out.getDouble("n"), 0.0) + assertEquals("a", out.getArray("list")!!.getString(0)) + assertEquals(true, out.getMap("nested")!!.getBoolean("k")) + assertEquals(true, out.isNull("none")) + } + + @Test + fun `isJson accepts one strict JSON value of any kind`() { + listOf("null", "1", "-0.5e3", "\"s\"", "true", "[]", "{}", """{"a":[1,null,{"b":"c"}]}""", " {\"a\":1} ").forEach { + assertEquals(it, true, JsonBridge.isJson(it)) + } + } + + @Test + fun `isJson rejects lenient and malformed text`() { + listOf("", " ", "{a:1}", "{'a':1}", "[1,]", "{\"a\":1} x", "undefined", "NaN", "{\"a\":1}{}").forEach { + assertEquals(it, false, JsonBridge.isJson(it)) + } + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/LegacyImportTest.kt b/android/src/test/java/ai/openspace/backgroundupload/LegacyImportTest.kt new file mode 100644 index 00000000..1386c2cb --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/LegacyImportTest.kt @@ -0,0 +1,114 @@ +package ai.openspace.backgroundupload + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import java.io.File + +class LegacyImportTest { + @get:Rule + val tmp = TemporaryFolder() + + private fun v9(dir: File, eventId: String, uploadId: String, type: String, timestamp: Long) = + File(dir, "$eventId.json").writeText( + """{"eventId":"$eventId","uploadId":"$uploadId","type":"$type","timestamp":$timestamp,"responseCode":200}""", + ) + + @Test + fun `each v9 id becomes one legacy row with its newest outcome`() { + val v9Dir = tmp.newFolder() + v9(v9Dir, "a", "up-1", "completed", 100) + v9(v9Dir, "b", "up-2", "error", 200) + v9(v9Dir, "c", "up-3", "cancelled", 300) + v9(v9Dir, "d", "up-2", "completed", 250) // newer for up-2 + File(v9Dir, "bad.json").writeText("{not json") + val store = QueueStore(tmp.newFolder(), RequestIndex()) + + assertTrue(LegacyImport.import(v9Dir, store)) + + val rows = store.all().associateBy { it.id } + assertEquals(setOf("up-1", "up-2", "up-3"), rows.keys) + assertEquals(EntryState.COMPLETED, rows["up-1"]!!.state) + assertEquals(EntryState.COMPLETED, rows["up-2"]!!.state) + assertEquals(EntryState.CANCELLED, rows["up-3"]!!.state) + val row = rows["up-2"]!! + assertEquals("legacy", row.key) + assertEquals("null", row.varsJson) + assertTrue(row.legacy) + assertEquals(0, row.attempts) + assertEquals(250, row.updatedAt) + assertNull(row.descriptor) + assertEquals(0, v9Dir.list()!!.size) // every v9 file is gone + } + + @Test + fun `a second run imports nothing`() { + val v9Dir = tmp.newFolder() + val store = QueueStore(tmp.newFolder(), RequestIndex()) + v9(v9Dir, "a", "up-1", "completed", 100) + LegacyImport.import(v9Dir, store) + store.remove("up-1") + assertTrue(LegacyImport.import(v9Dir, store)) + assertEquals(emptyList(), store.all()) + } + + @Test + fun `an id that a v10 entry owns is left alone`() { + val v9Dir = tmp.newFolder() + val store = QueueStore(tmp.newFolder(), RequestIndex()) + store.save(entry(id = "up-1")) + v9(v9Dir, "a", "up-1", "error", 100) + LegacyImport.import(v9Dir, store) + assertEquals("note", store.load("up-1")!!.key) + } + + @Test + fun `a failed save keeps the v9 file and reports incomplete`() { + val v9Dir = tmp.newFolder() + v9(v9Dir, "a", "up-1", "error", 100) + val broken = QueueStore(tmp.newFile(), RequestIndex()) // a file where the directory should be + assertEquals(false, LegacyImport.import(v9Dir, broken)) + assertTrue(File(v9Dir, "a.json").exists()) + } + + @Test + fun `runOnce imports, then writes the marker, and a second launch does nothing`() { + val v9Dir = tmp.newFolder() + val marker = File(tmp.newFolder(), LegacyImport.MARKER) + val store = QueueStore(tmp.newFolder(), RequestIndex()) + v9(v9Dir, "a", "up-1", "completed", 100) + assertTrue(LegacyImport.runOnce(marker, v9Dir, store)) + assertTrue(marker.exists()) + assertEquals(listOf("up-1"), store.all().map { it.id }) + // A v9 file that shows up later is not imported: the marker says done. + v9(v9Dir, "b", "up-2", "error", 200) + store.remove("up-1") + assertFalse(LegacyImport.runOnce(marker, v9Dir, store)) + assertEquals(emptyList(), store.all()) + assertTrue(File(v9Dir, "b.json").exists()) + } + + @Test + fun `runOnce with a failed row save writes no marker, so the next launch tries again`() { + val v9Dir = tmp.newFolder() + val marker = File(tmp.newFolder(), LegacyImport.MARKER) + v9(v9Dir, "a", "up-1", "error", 100) + val broken = QueueStore(tmp.newFile(), RequestIndex()) // a file where the directory should be + assertTrue(LegacyImport.runOnce(marker, v9Dir, broken)) // it ran, so v9 work is cancelled + assertFalse(marker.exists()) + val store = QueueStore(tmp.newFolder(), RequestIndex()) + assertTrue(LegacyImport.runOnce(marker, v9Dir, store)) + assertTrue(marker.exists()) + assertEquals(listOf("up-1"), store.all().map { it.id }) + } + + @Test + fun `an unknown type makes no row`() { + assertNull(LegacyImport.legacyRow(LegacyImport.V9Entry("a", "up", "progress", 1))) + assertNull(LegacyImport.legacyRow(LegacyImport.V9Entry("a", null, "completed", 1))) + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/QueueControllerTest.kt b/android/src/test/java/ai/openspace/backgroundupload/QueueControllerTest.kt new file mode 100644 index 00000000..6cfc408f --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/QueueControllerTest.kt @@ -0,0 +1,777 @@ +package ai.openspace.backgroundupload + +import org.junit.Assert.assertArrayEquals +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNotNull +import org.junit.Assert.assertNull +import org.junit.Assert.assertThrows +import org.junit.Assert.assertTrue +import org.junit.Before +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import java.io.File + +class QueueControllerTest { + @get:Rule + val tmp = TemporaryFolder() + + private lateinit var root: File + private lateinit var store: QueueStore + private lateinit var journal: EventJournal + private lateinit var settings: QueueSettingsStore + private val events = RecordingEvents() + private val scheduler = FakeScheduler() + private val running = mutableSetOf() + private var now = 10_000L + private lateinit var controller: QueueController + private val listener = Any() + + @Before + fun setUp() { + root = tmp.newFolder("queue") + store = QueueStore(root, RequestIndex()) + journal = EventJournal(tmp.newFolder("journal"), retryLater = { _, _ -> }) + settings = QueueSettingsStore(File(root, "settings.json")) + controller = QueueController(store, journal, settings, events, scheduler, { it in running }, { now }) + journal.drain(listener) // JS is subscribed + } + + private fun source(name: String, size: Int) = File(tmp.newFolder(), name).apply { writeBytes(ByteArray(size) { it.toByte() }) } + + private fun dirFiles(id: String = "e1") = store.entryDir(id).list()!!.toSet() + + // MARK: - enqueue + + @Test + fun `create stages the json body, persists, schedules, then emits`() { + assertEquals("e1", controller.enqueue(parsed())) + val e = store.load("e1")!! + assertEquals(EntryState.QUEUED, e.state) + assertEquals(1, e.generation) + assertEquals("""{"a":1}""", File(store.entryDir("e1"), e.body!!.fileName!!).readText()) + assertEquals("application/json", e.descriptor!!.headers["Content-Type"]) + assertEquals(setOf("entry.json", "body-1.json"), dirFiles()) + assertEquals(listOf("e1"), scheduler.scheduled) + assertEquals(listOf("state:e1:queued"), events.log) + } + + @Test + fun `create while paused is paused and not scheduled`() { + controller.pause() + controller.enqueue(parsed()) + assertEquals(EntryState.PAUSED, store.load("e1")!!.state) + assertEquals(emptyList(), scheduler.scheduled) + } + + @Test + fun `a file body is copied, so the caller may delete its source`() { + val src = source("photo.jpg", 100) + controller.enqueue(parsed(descriptor = desc(file = src.path))) + src.delete() + val e = store.load("e1")!! + assertEquals(100, File(store.entryDir("e1"), e.body!!.fileName!!).length()) + assertEquals(100, e.totalBytes) + } + + @Test + fun `a missing file rejects E_FILE_MISSING and persists nothing`() { + val e = assertThrows(QueueException::class.java) { controller.enqueue(parsed(descriptor = desc(file = "/nope.bin"))) } + assertEquals(QueueException.E_FILE_MISSING, e.code) + assertNull(store.load("e1")) + assertEquals(emptyList(), events.log) + } + + @Test + fun `same body on a queued entry resumes with the new headers, vars, and expiry`() { + controller.enqueue(parsed()) + val bodyFile = store.load("e1")!!.body!!.fileName + controller.enqueue(parsed(descriptor = desc(dataJson = """{"a":1}""", headers = mapOf("Authorization" to "Bearer new")), varsJson = """{"n":2}""", expiresAt = 5)) + val e = store.load("e1")!! + assertEquals("Bearer new", e.descriptor!!.headers["Authorization"]) + assertEquals("""{"n":2}""", e.varsJson) + assertEquals(5, e.expiresAt) + assertEquals(1, e.generation) + assertEquals(bodyFile, e.body!!.fileName) + } + + @Test + fun `same body on a running entry stays running and is not scheduled again`() { + store.save(entry(state = EntryState.RUNNING)) + controller.enqueue(parsed(descriptor = desc(dataJson = """{"a":1}""", headers = mapOf("Authorization" to "Bearer new")))) + val e = store.load("e1")!! + assertEquals(EntryState.RUNNING, e.state) + assertEquals("Bearer new", e.descriptor!!.headers["Authorization"]) + assertEquals(emptyList(), scheduler.scheduled) + } + + @Test + fun `a different body on a running entry rejects E_RUNNING and changes nothing`() { + store.save(entry(state = EntryState.RUNNING)) + val e = assertThrows(QueueException::class.java) { controller.enqueue(parsed(descriptor = desc(dataJson = """{"a":2}"""))) } + assertEquals(QueueException.E_RUNNING, e.code) + assertEquals(entry(state = EntryState.RUNNING), store.load("e1")) + } + + @Test + fun `a different body on a queued entry with a worker sleeping out a short backoff is accepted`() { + // The gate is held, but the entry is queued. The contract: queued is not running. + controller.enqueue(parsed()) + running += "e1" + controller.enqueue(parsed(descriptor = desc(dataJson = """{"a":2}"""))) + assertEquals("""{"a":2}""", store.load("e1")!!.descriptor!!.dataJson) + } + + @Test + fun `a different body on an error entry replaces it and reopens it`() { + controller.enqueue(parsed()) + store.save(store.load("e1")!!.copy(state = EntryState.ERROR, attempts = 3, settledEventId = "ev")) + controller.enqueue(parsed(descriptor = desc(dataJson = """{"a":2}"""))) + val e = store.load("e1")!! + assertEquals(EntryState.QUEUED, e.state) + assertEquals(2, e.generation) + assertEquals(0, e.attempts) + assertNull(e.settledEventId) + assertEquals("""{"a":2}""", File(store.entryDir("e1"), "body-2.json").readText()) + assertEquals(setOf("entry.json", "body-2.json"), dirFiles()) // the old body is pruned + } + + @Test + fun `same body on a completed unacked entry re-emits with one more delivery and does not re-run`() { + controller.enqueue(parsed()) + val eventId = "00000000-0000-0000-0000-00000000000a" + journal.append(record(eventId)) + store.save(store.load("e1")!!.copy(state = EntryState.COMPLETED, settledEventId = eventId)) + scheduler.scheduled.clear() + events.log.clear() + controller.enqueue(parsed()) + assertEquals(listOf("settled:e1:completed"), events.log) + assertEquals(2, events.records.single().deliveries) + assertEquals(2, journal.find(eventId)!!.deliveries) + assertEquals(EntryState.COMPLETED, store.load("e1")!!.state) + assertEquals(emptyList(), scheduler.scheduled) + } + + @Test + fun `same body on a cancelled unacked entry gets a fresh generation, and the old ack forgets nothing`() { + controller.enqueue(parsed()) + controller.cancel("e1") + val cancelled = journal.unacknowledged().single() + controller.enqueue(parsed()) + val e = store.load("e1")!! + assertEquals(EntryState.QUEUED, e.state) + assertEquals(2, e.generation) + controller.ack(listOf(cancelled.eventId)) + assertNotNull(store.load("e1")) + } + + @Test + fun `same body on an error entry reopens it with attempts 0`() { + controller.enqueue(parsed()) + store.save(store.load("e1")!!.copy(state = EntryState.ERROR, attempts = 3, settledEventId = "ev")) + controller.enqueue(parsed(expiresAt = FAR_FUTURE + 1)) + val e = store.load("e1")!! + assertEquals(EntryState.QUEUED, e.state) + assertEquals(2, e.generation) + assertEquals(0, e.attempts) // attempts count the current generation + } + + @Test + fun `same body on a live entry keeps its attempts`() { + controller.enqueue(parsed()) + store.save(store.load("e1")!!.copy(attempts = 3)) + controller.enqueue(parsed(expiresAt = FAR_FUTURE + 1)) + assertEquals(3, store.load("e1")!!.attempts) + } + + @Test + fun `a same-id resume of a queued entry waiting out a backoff clears it and runs now`() { + controller.enqueue(parsed()) + store.save(store.load("e1")!!.copy(nextAttemptAt = now + 3_600_000, backoffStreak = 9)) + scheduler.scheduled.clear() + controller.enqueue(parsed()) + val e = store.load("e1")!! + assertNull(e.nextAttemptAt) + assertEquals(0, e.backoffStreak) + assertEquals(listOf("e1"), scheduler.scheduled) + assertEquals(emptyList>(), scheduler.wakes) + assertFalse(events.rows.last().toMap().containsKey("nextAttemptAt")) + } + + @Test + fun `a completed unacked re-emit with no listener yet waits for the drain`() { + controller.enqueue(parsed()) + val eventId = "00000000-0000-0000-0000-00000000001a" + journal.append(record(eventId)) + store.save(store.load("e1")!!.copy(state = EntryState.COMPLETED, settledEventId = eventId)) + journal.stopListening(listener) + events.log.clear() + controller.enqueue(parsed()) + assertEquals(emptyList(), events.log) + assertEquals(1, journal.find(eventId)!!.deliveries) + assertEquals(2, controller.unacknowledged(listener).single().deliveries) + } + + @Test + fun `enqueue over a legacy row replaces it`() { + store.save(LegacyImport.legacyRow(LegacyImport.V9Entry("x", "e1", "completed", 5))!!) + controller.enqueue(parsed()) + val e = store.load("e1")!! + assertFalse(e.legacy) + assertEquals(2, e.generation) + assertEquals(EntryState.QUEUED, e.state) + } + + // MARK: - legacy row over a v9 manifest + + @Test + fun `a same-id enqueue over a legacy row with a v9 manifest adopts it one generation up`() { + v9Dir("e1", v9Parts, 20) + store.save(LegacyImport.legacyRow(LegacyImport.V9Entry("x", "e1", "error", 5))!!) + val legacy = store.load("e1")!! + assertEquals(0L to 0L, legacy.bytesSent to legacy.totalBytes) // legacy rows report 0/0 bytes + controller.enqueue(parsed(descriptor = desc(url = null, method = "PUT", file = "/gone.bin", parts = listOf(part(0, 10), part(10, 20))))) + val e = store.load("e1")!! + assertFalse(e.legacy) + assertEquals(2, e.generation) + assertEquals(5, e.createdAt) + assertTrue(e.descriptor!!.parts!![0].accepted) // the v9 progress is kept + assertFalse(e.descriptor!!.parts!![1].accepted) + assertEquals(10, e.bytesSent) + assertEquals(setOf("entry.json", "blob"), dirFiles()) // the manifest is pruned + } + + @Test + fun `a same-id enqueue over a legacy row with no manifest replaces it`() { + store.save(LegacyImport.legacyRow(LegacyImport.V9Entry("x", "e1", "error", 5))!!) + controller.enqueue(parsed()) + val e = store.load("e1")!! + assertEquals(2, e.generation) + assertEquals(0, e.bytesSent) + } + + // MARK: - chunked replace (a present file wins over the old blob) + + private fun chunkedErrorEntry(): File { + val first = source("first.bin", 20) + controller.enqueue(parsed(descriptor = desc(url = null, method = "PUT", file = first.path, parts = listOf(part(0, 10), part(10, 20))))) + store.save(store.load("e1")!!.copy(state = EntryState.ERROR, settledEventId = "ev")) + return File(store.entryDir("e1"), "blob") + } + + @Test + fun `a different-parts replace with a present file uploads that file, not the old blob`() { + chunkedErrorEntry() + val bytes = ByteArray(20) { (it * 3).toByte() } + val second = File(tmp.newFolder(), "second.bin").apply { writeBytes(bytes) } + controller.enqueue(parsed(descriptor = desc(url = null, method = "PUT", file = second.path, parts = listOf(part(0, 20))))) + val e = store.load("e1")!! + assertEquals(2, e.generation) + assertEquals("blob-2", e.body!!.fileName) + assertArrayEquals(bytes, store.bodyFile(e)!!.readBytes()) + assertEquals(setOf("entry.json", "blob-2"), dirFiles()) // the old blob is pruned + } + + @Test + fun `a different-parts replace with a present file of another size is accepted`() { + chunkedErrorEntry() + val second = source("second.bin", 30) + controller.enqueue(parsed(descriptor = desc(url = null, method = "PUT", file = second.path, parts = listOf(part(0, 30))))) + assertEquals(30, store.load("e1")!!.totalBytes) + } + + @Test + fun `a different-parts replace whose file was moved away runs over the old blob`() { + chunkedErrorEntry() + controller.enqueue(parsed(descriptor = desc(url = null, method = "PUT", file = "/moved.bin", parts = listOf(part(0, 20))))) + val e = store.load("e1")!! + assertEquals("blob", e.body!!.fileName) + assertEquals(setOf("entry.json", "blob"), dirFiles()) + } + + @Test + fun `a replace whose plan does not tile the present file changes nothing`() { + val oldBlob = chunkedErrorEntry() + val second = source("second.bin", 30) + val e = assertThrows(QueueException::class.java) { + controller.enqueue(parsed(descriptor = desc(url = null, method = "PUT", file = second.path, parts = listOf(part(0, 20))))) + } + assertEquals(QueueException.E_INVALID, e.code) + assertTrue(second.exists()) + assertEquals(20, oldBlob.length()) + assertEquals(1, store.load("e1")!!.generation) + } + + // MARK: - staging outside the lock + + @Test + fun `a new file body is copied outside the store lock`() { + val src = source("big.bin", 1000) + var hookRan = false + controller.afterPreStage = { + hookRan = true + assertFalse(Thread.holdsLock(store)) + assertTrue(File(store.entryDir("e1"), "file-1").exists()) + } + controller.enqueue(parsed(descriptor = desc(file = src.path))) + assertTrue(hookRan) + assertEquals("file-1", store.load("e1")!!.body!!.fileName) + assertEquals(setOf("entry.json", "file-1"), dirFiles()) + } + + @Test + fun `a replace over a queued entry stages under the lock`() { + controller.enqueue(parsed()) + var hookRan = false + controller.afterPreStage = { hookRan = true } + controller.enqueue(parsed(descriptor = desc(dataJson = """{"a":2}"""))) + assertFalse(hookRan) + assertEquals(setOf("entry.json", "body-2.json"), dirFiles()) + } + + @Test + fun `a pre-staged body that no longer fits is deleted and staged again under the lock`() { + controller.enqueue(parsed()) + store.save(store.load("e1")!!.copy(state = EntryState.ERROR)) + // Between the staging and the lock, the entry moves on to another generation. + controller.afterPreStage = { store.save(store.load("e1")!!.copy(generation = 5)) } + controller.enqueue(parsed(descriptor = desc(dataJson = """{"a":2}"""))) + val e = store.load("e1")!! + assertEquals(6, e.generation) + assertEquals("body-6.json", e.body!!.fileName) + assertEquals(setOf("entry.json", "body-6.json"), dirFiles()) // body-2.json is gone + } + + @Test + fun `a pre-staged body is deleted when the enqueue rejects`() { + controller.enqueue(parsed()) + store.save(store.load("e1")!!.copy(state = EntryState.ERROR)) + controller.afterPreStage = { store.save(store.load("e1")!!.copy(state = EntryState.RUNNING)) } + val e = assertThrows(QueueException::class.java) { controller.enqueue(parsed(descriptor = desc(dataJson = """{"a":2}"""))) } + assertEquals(QueueException.E_RUNNING, e.code) + assertEquals(setOf("entry.json", "body-1.json"), dirFiles()) + } + + // MARK: - v9 adoption + + private fun v9Dir(id: String, parts: String, blobSize: Int): File { + val dir = store.entryDir(id).apply { mkdirs() } + File(dir, "blob").writeBytes(ByteArray(blobSize)) + File(dir, "manifest.json").writeText( + """{"id":"$id","sourcePath":"${File(dir, "blob").path}","parts":$parts,"accept":[],"expiresAt":1,"wifiOnly":false,"noNotification":false,"createdAt":1}""", + ) + return dir + } + + private val v9Parts = """[{"url":"https://example.com/part?start=0","headers":{},"start":0,"end":10,"accepted":true},""" + + """{"url":"https://example.com/part?start=10","headers":{},"start":10,"end":20,"accepted":false}]""" + + @Test + fun `a same-id enqueue with the same parts adopts the v9 blob and accepted parts`() { + v9Dir("e1", v9Parts, 20) + controller.enqueue(parsed(descriptor = desc(url = null, method = "PUT", file = "/gone.bin", parts = listOf(part(0, 10), part(10, 20))))) + val e = store.load("e1")!! + assertTrue(e.descriptor!!.parts!![0].accepted) + assertFalse(e.descriptor!!.parts!![1].accepted) + assertEquals(10, e.bytesSent) + assertEquals(setOf("entry.json", "blob"), dirFiles()) + } + + @Test + fun `a v9 adoption with the same parts keeps the v9 blob even when the caller's file is present`() { + v9Dir("e1", v9Parts, 20) + val present = source("again.bin", 20) + controller.enqueue(parsed(descriptor = desc(url = null, method = "PUT", file = present.path, parts = listOf(part(0, 10), part(10, 20))))) + assertTrue(present.exists()) + assertTrue(store.load("e1")!!.descriptor!!.parts!![0].accepted) + assertArrayEquals(ByteArray(20), File(store.entryDir("e1"), "blob").readBytes()) + } + + @Test + fun `a v9 adoption with different parts and a present file uploads that file`() { + v9Dir("e1", v9Parts, 20) + val present = source("new.bin", 30) + controller.enqueue(parsed(descriptor = desc(url = null, method = "PUT", file = present.path, parts = listOf(part(0, 30))))) + assertFalse(present.exists()) + val e = store.load("e1")!! + assertFalse(e.descriptor!!.parts!![0].accepted) + assertEquals(30, store.bodyFile(e)!!.length()) + assertEquals(setOf("entry.json", "blob"), dirFiles()) + } + + @Test + fun `a v9 adoption with parts that do not tile rejects E_INVALID and keeps the v9 files`() { + v9Dir("e1", v9Parts, 20) + val e = assertThrows(QueueException::class.java) { + controller.enqueue(parsed(descriptor = desc(url = null, method = "PUT", file = "/gone.bin", parts = listOf(part(0, 30))))) + } + assertEquals(QueueException.E_INVALID, e.code) + assertEquals(setOf("manifest.json", "blob"), dirFiles()) + } + + // MARK: - cancel + + @Test + fun `cancel of a live entry journals, settles cancelled, stops work, emits, and forgets after ack`() { + controller.enqueue(parsed()) + events.log.clear() + controller.cancel("e1") + val record = journal.unacknowledged().single() + assertEquals(EventJournal.KIND_CANCELLED, record.kind) + assertEquals("user", record.cancelReason) + assertEquals("https://example.com/items", record.url) + val e = store.load("e1")!! + assertEquals(EntryState.CANCELLED, e.state) + assertEquals(record.eventId, e.settledEventId) + assertEquals(listOf("e1"), scheduler.cancelled) + assertEquals(listOf("settled:e1:cancelled", "state:e1:cancelled"), events.log) + assertTrue(events.listeners.single() === listener) + controller.ack(listOf(record.eventId)) + assertNull(store.load("e1")) + assertFalse(store.entryDir("e1").exists()) + } + + @Test + fun `cancel of a settled entry forgets it now with its unacked outcomes`() { + controller.enqueue(parsed()) + val eventId = "00000000-0000-0000-0000-00000000000b" + val older = "00000000-0000-0000-0000-00000000001b" + journal.append(record(older, generation = 0)) + journal.append(record(eventId, kind = EventJournal.KIND_ERROR)) + journal.append(record("00000000-0000-0000-0000-00000000002b", id = "other")) + store.save(store.load("e1")!!.copy(state = EntryState.ERROR, settledEventId = eventId)) + events.log.clear() + controller.cancel("e1") + assertNull(store.load("e1")) + assertFalse(store.entryDir("e1").exists()) + assertEquals(emptyList(), events.log) + assertEquals(listOf("other"), journal.unacknowledged().map { it.id }) + } + + @Test + fun `cancel of a live entry whose journal can not write rejects E_STORAGE and changes nothing`() { + controller.enqueue(parsed()) + val before = store.load("e1")!! + events.log.clear() + scheduler.cancelled.clear() + val dir = tmp.root.resolve("journal") + dir.setWritable(false) + val e = try { + assertThrows(QueueException::class.java) { controller.cancel("e1") } + } finally { + dir.setWritable(true) + } + assertEquals(QueueException.E_STORAGE, e.code) + assertEquals(before, store.load("e1")) + assertEquals(emptyList(), journal.unacknowledged()) + assertEquals(emptyList(), events.log) + assertEquals(emptyList(), scheduler.cancelled) + } + + @Test + fun `cancel whose entry save fails stops the work, emits, rejects, and a retry adds no second outcome`() { + controller.enqueue(parsed()) + events.log.clear() + scheduler.cancelled.clear() + val dir = store.entryDir("e1") + dir.setWritable(false) + val error = try { + assertThrows(QueueException::class.java) { controller.cancel("e1") } + } finally { + dir.setWritable(true) + } + assertEquals(QueueException.E_STORAGE, error.code) + val record = journal.unacknowledged().single() + assertEquals(EventJournal.KIND_CANCELLED, record.kind) + assertEquals(EntryState.QUEUED, store.load("e1")!!.state) // the save was lost + assertEquals(listOf("e1"), scheduler.cancelled) // a running request can not settle again + assertEquals(listOf("settled:e1:cancelled"), events.log) + // JS calls cancel() again: the journaled cancel is applied, not a second one. + events.log.clear() + controller.cancel("e1") + val e = store.load("e1")!! + assertEquals(EntryState.CANCELLED, e.state) + assertEquals(record.eventId, e.settledEventId) + assertEquals(listOf(record.eventId), journal.unacknowledged().map { it.eventId }) + assertEquals(listOf("state:e1:cancelled"), events.log) + } + + @Test + fun `cancel over the journal cap keeps its own record`() { + val small = EventJournal(tmp.newFolder("small"), maxEntries = 1) + val smallController = QueueController(store, small, settings, events, scheduler, { false }, { now }) + val named = "00000000-0000-0000-0000-000000000042" + small.append(record(named, id = "done")) + store.save(entry(id = "done", state = EntryState.ERROR, settledEventId = named)) + smallController.enqueue(parsed()) + smallController.cancel("e1") + val own = store.load("e1")!!.settledEventId!! + assertNotNull(small.find(own)) // without it, the sweep forgets the row with no outcome + assertNotNull(small.find(named)) + } + + @Test + fun `cancel with no listener journals at 0 and does not emit the outcome`() { + controller.enqueue(parsed()) + journal.stopListening(listener) + events.log.clear() + controller.cancel("e1") + assertEquals(listOf("state:e1:cancelled"), events.log) + assertEquals(0, journal.unacknowledged().single().deliveries) + } + + @Test + fun `cancel of an unknown id is a no-op`() { + controller.cancel("nope") + assertEquals(emptyList(), events.log) + } + + // MARK: - pause, resume, wifi, headers + + @Test + fun `pause moves live rows to paused with no outcome, and resume brings them back`() { + store.save(entry(id = "q", state = EntryState.QUEUED)) + store.save(entry(id = "r", state = EntryState.RUNNING)) + store.save(entry(id = "a", state = EntryState.AWAITING_AUTH, parkedGeneration = 0)) + store.save(entry(id = "s", state = EntryState.AWAITING_AUTH, parkedGeneration = 0)) + store.save(entry(id = "x", state = EntryState.ERROR)) + controller.pause() + assertTrue(settings.load().paused) + assertEquals(listOf("paused", "paused", "paused", "paused", "error"), listOf("q", "r", "a", "s", "x").map { store.load(it)!!.state.wire }) + assertEquals(setOf("q", "r", "a", "s"), scheduler.cancelled.toSet()) + assertEquals(emptyList(), journal.unacknowledged()) + + // A header change while paused makes one parked entry's generation stale. + store.save(store.load("s")!!.copy(parkedGeneration = -1)) + controller.resume() + assertFalse(settings.load().paused) + assertEquals(EntryState.QUEUED, store.load("q")!!.state) + assertEquals(EntryState.QUEUED, store.load("r")!!.state) + assertEquals(EntryState.AWAITING_AUTH, store.load("a")!!.state) + assertEquals(EntryState.QUEUED, store.load("s")!!.state) + assertTrue(scheduler.scheduled.containsAll(listOf("q", "r", "s"))) + assertEquals(listOf("a" to FAR_FUTURE), scheduler.wakes) // the parked one waits for its expiry + } + + @Test + fun `setWifiOnly persists`() { + controller.setWifiOnly(true) + assertTrue(QueueSettingsStore(File(root, "settings.json")).load().wifiOnly) + } + + @Test + fun `updateHeaders patches every entry, bumps the generation, and requeues the parked`() { + store.save(entry(id = "p", state = EntryState.AWAITING_AUTH, parkedGeneration = 0)) + store.save(entry(id = "x", state = EntryState.ERROR)) + store.save(entry(id = "c", state = EntryState.QUEUED, descriptor = desc(url = null, file = "/f", + parts = listOf(Part("https://p/1", mapOf("authorization" to "stale"), 0, 10))))) + controller.updateHeaders(mapOf("Authorization" to "Bearer new")) + assertEquals(1, settings.load().headerGeneration) + val p = store.load("p")!! + assertEquals(EntryState.QUEUED, p.state) + assertNull(p.parkedGeneration) + assertEquals("Bearer new", p.descriptor!!.headers["Authorization"]) + assertEquals(1, p.headerGeneration) + assertEquals("Bearer new", store.load("x")!!.descriptor!!.headers["Authorization"]) + assertEquals(mapOf("Authorization" to "Bearer new"), store.load("c")!!.descriptor!!.parts!![0].headers) + assertEquals(listOf("p"), scheduler.scheduled) + assertEquals(listOf("state:p:queued"), events.log) + } + + // MARK: - ack and replay + + @Test + fun `ack forgets a completed entry of the current generation only`() { + val current = "00000000-0000-0000-0000-00000000000c" + val old = "00000000-0000-0000-0000-00000000000d" + store.save(entry(state = EntryState.COMPLETED, settledEventId = current, generation = 2)) + journal.append(record(old, generation = 1)) + journal.append(record(current, generation = 2)) + controller.ack(listOf(old, "unknown", "../../x")) + assertNotNull(store.load("e1")) + controller.ack(listOf(current)) + assertNull(store.load("e1")) + assertEquals(listOf("e1"), scheduler.cancelled) + controller.ack(listOf(current)) // idempotent + } + + @Test + fun `ack of an error removes the record and keeps the row`() { + val id = "00000000-0000-0000-0000-00000000000e" + store.save(entry(state = EntryState.ERROR, settledEventId = id)) + journal.append(record(id, kind = EventJournal.KIND_ERROR)) + controller.ack(listOf(id)) + assertNull(journal.find(id)) + assertNotNull(store.load("e1")) + } + + // A settle journaled and emitted its record, but the store write failed: + // the entry is still live at the record's generation. + + @Test + fun `ack of a completed record on a still-running entry settles and forgets it, so the sweep does not re-run it`() { + val id = "00000000-0000-0000-0000-000000000011" + store.save(entry(state = EntryState.RUNNING)) + journal.append(record(id)) + controller.ack(listOf(id)) + assertNull(store.load("e1")) + assertNull(journal.find(id)) + controller.sweep() + assertEquals(emptyList(), scheduler.scheduled) + } + + @Test + fun `ack of an error record on a still-running entry settles it as error and keeps the row`() { + val id = "00000000-0000-0000-0000-000000000012" + store.save(entry(state = EntryState.RUNNING)) + journal.append(record(id, kind = EventJournal.KIND_ERROR)) + controller.ack(listOf(id)) + val e = store.load("e1")!! + assertEquals(EntryState.ERROR, e.state) + assertEquals(id, e.settledEventId) + assertNull(journal.find(id)) + assertEquals(listOf("state:e1:error"), events.log) + controller.sweep() + assertEquals(emptyList(), scheduler.scheduled) + } + + @Test + fun `ack of a record of an older generation does not settle the live entry`() { + val id = "00000000-0000-0000-0000-000000000013" + store.save(entry(state = EntryState.QUEUED, generation = 2)) + journal.append(record(id, generation = 1)) + controller.ack(listOf(id)) + assertEquals(EntryState.QUEUED, store.load("e1")!!.state) + assertNull(journal.find(id)) + } + + @Test + fun `when the ack repair can not save, the record stays for the sweep`() { + val id = "00000000-0000-0000-0000-000000000014" + store.save(entry(state = EntryState.RUNNING)) + journal.append(record(id)) + val dir = store.entryDir("e1") + dir.setWritable(false) + try { + controller.ack(listOf(id)) + } finally { + dir.setWritable(true) + } + assertNotNull(journal.find(id)) + assertEquals(EntryState.RUNNING, store.load("e1")!!.state) + controller.sweep() + assertEquals(EntryState.COMPLETED, store.load("e1")!!.state) + } + + @Test + fun `each replay counts one more delivery`() { + journal.append(record("00000000-0000-0000-0000-00000000000f")) + assertEquals(2, controller.unacknowledged(listener).single().deliveries) + assertEquals(3, controller.unacknowledged(listener).single().deliveries) + } + + // MARK: - boot sweep + + @Test + fun `sweep applies a record that the store transition missed`() { + val id = "00000000-0000-0000-0000-000000000010" + store.save(entry(state = EntryState.RUNNING)) + journal.append(record(id)) + controller.sweep() + val e = store.load("e1")!! + assertEquals(EntryState.COMPLETED, e.state) + assertEquals(id, e.settledEventId) + assertEquals(listOf("state:e1:completed"), events.log) + } + + @Test + fun `sweep applies a cancel whose store save was lost`() { + val id = "00000000-0000-0000-0000-000000000011" + store.save(entry(state = EntryState.QUEUED)) + journal.append(record(id, kind = EventJournal.KIND_CANCELLED)) + controller.sweep() + assertEquals(EntryState.CANCELLED, store.load("e1")!!.state) + assertEquals(emptyList(), scheduler.scheduled) + } + + @Test + fun `sweep queues a running entry with no worker and schedules queued work`() { + store.save(entry(id = "r", state = EntryState.RUNNING)) + store.save(entry(id = "q", state = EntryState.QUEUED)) + store.save(entry(id = "w", state = EntryState.QUEUED, nextAttemptAt = now + 3_600_000)) + store.save(entry(id = "p", state = EntryState.AWAITING_AUTH, expiresAt = now + 50)) + controller.sweep() + assertEquals(EntryState.QUEUED, store.load("r")!!.state) + assertEquals(setOf("r", "q"), scheduler.scheduled.toSet()) + assertEquals(setOf("w" to now + 3_600_000, "p" to now + 50), scheduler.wakes.toSet()) + } + + @Test + fun `sweep skips an entry whose worker runs in this process`() { + store.save(entry(state = EntryState.RUNNING)) + journal.append(record("00000000-0000-0000-0000-000000000012")) + running += "e1" + controller.sweep() + assertEquals(EntryState.RUNNING, store.load("e1")!!.state) + } + + @Test + fun `sweep forgets a completed entry whose record was acked, and acks orphans`() { + store.save(entry(id = "done", state = EntryState.COMPLETED, settledEventId = "gone")) + val own = "00000000-0000-0000-0000-000000000013" + val orphan = "00000000-0000-0000-0000-000000000014" + store.save(entry(id = "c", state = EntryState.CANCELLED, settledEventId = own)) + journal.append(record(own, id = "c", kind = EventJournal.KIND_CANCELLED)) + journal.append(record(orphan, id = "c")) + controller.sweep() + assertNull(store.load("done")) + assertEquals(listOf(own), journal.unacknowledged().map { it.eventId }) + assertNotNull(store.load("c")) + } + + @Test + fun `sweep keeps a completed entry whose record is held in memory`() { + val eventId = "00000000-0000-0000-0000-000000000015" + val dir = tmp.root.resolve("journal") + dir.setWritable(false) + try { + journal.appendOrHold(record(eventId)) + } finally { + dir.setWritable(true) + } + store.save(entry(state = EntryState.COMPLETED, settledEventId = eventId)) + controller.sweep() + assertNotNull(store.load("e1")) + // Its ack still forgets it. + controller.ack(listOf(eventId)) + assertNull(store.load("e1")) + } + + @Test + fun `sweep leaves paused entries alone`() { + controller.pause() + store.save(entry(state = EntryState.PAUSED)) + controller.sweep() + assertEquals(EntryState.PAUSED, store.load("e1")!!.state) + assertEquals(emptyList(), scheduler.scheduled) + } + + @Test + fun `sweep finishes a pause or resume that a process death cut short`() { + // resume() saved the setting, then died before the rows. + store.save(entry(id = "p", state = EntryState.PAUSED)) + controller.sweep() + assertEquals(EntryState.QUEUED, store.load("p")!!.state) + assertEquals(listOf("p"), scheduler.scheduled) + + // pause() saved the setting, then died before the rows. + settings.update { it.copy(paused = true) } + store.save(entry(id = "q", state = EntryState.QUEUED)) + store.save(entry(id = "r", state = EntryState.RUNNING)) + controller.sweep() + assertEquals(EntryState.PAUSED, store.load("q")!!.state) + assertEquals(EntryState.PAUSED, store.load("r")!!.state) + assertEquals(listOf("p"), scheduler.scheduled) + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/QueueEntryTest.kt b/android/src/test/java/ai/openspace/backgroundupload/QueueEntryTest.kt new file mode 100644 index 00000000..c0b286b3 --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/QueueEntryTest.kt @@ -0,0 +1,103 @@ +package ai.openspace.backgroundupload + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test + +class QueueEntryTest { + + private val form = listOf(FormPart("photo", "image/jpeg", null, "/p.jpg", null)) + + @Test + fun `sameBodyAs compares each body kind by content`() { + val json = entry(descriptor = desc(dataJson = "{\"a\":1}")) + assertTrue(json.sameBodyAs(desc(dataJson = "{\"a\":1}", headers = mapOf("New" to "h")))) + assertFalse(json.sameBodyAs(desc(dataJson = "{\"a\":2}"))) + + val multipart = entry(descriptor = desc(form = form)) + assertTrue(multipart.sameBodyAs(desc(form = form))) + assertFalse(multipart.sameBodyAs(desc(form = form.map { it.copy(name = "other") }))) + + val file = entry(descriptor = desc(file = "/a.bin")) + assertTrue(file.sameBodyAs(desc(file = "/a.bin"))) + assertFalse(file.sameBodyAs(desc(file = "/b.bin"))) + + val none = entry(descriptor = desc(method = "DELETE")) + assertTrue(none.sameBodyAs(desc(method = "DELETE"))) + } + + @Test + fun `chunked compares parts and ignores the path`() { + val parts = listOf(part(0, 100), part(100, 250)) + val chunked = entry(descriptor = desc(url = null, method = "PUT", file = "/moved.bin", parts = parts)) + assertTrue(chunked.sameBodyAs(desc(url = null, method = "PUT", file = "/elsewhere.bin", parts = parts.reversed()))) + assertFalse(chunked.sameBodyAs(desc(url = null, method = "PUT", file = "/moved.bin", parts = listOf(part(0, 250))))) + } + + @Test + fun `a kind change, url change, or method change is a different body`() { + val json = entry(descriptor = desc(dataJson = "{}")) + assertFalse(json.sameBodyAs(desc(form = form))) + assertFalse(json.sameBodyAs(desc(url = "https://example.com/other", dataJson = "{}"))) + assertFalse(json.sameBodyAs(desc(method = "PUT", dataJson = "{}"))) + assertFalse(entry(descriptor = null, legacy = true).sameBodyAs(desc(dataJson = "{}"))) + } + + @Test + fun `withHeadersPatched matches names in any case and keeps the patch spelling`() { + val e = entry(descriptor = desc(headers = mapOf("authorization" to "Bearer old", "X-Keep" to "1"))) + val patched = e.withHeadersPatched(mapOf("Authorization" to "Bearer new", "X-Add" to "2"), generation = 3) + assertEquals( + mapOf("X-Keep" to "1", "Authorization" to "Bearer new", "X-Add" to "2"), + patched.descriptor!!.headers, + ) + assertEquals(3, patched.headerGeneration) + } + + @Test + fun `withHeadersPatched replaces a part's own copy of a patched header only`() { + val parts = listOf( + Part("https://p/1", mapOf("AUTHORIZATION" to "Bearer stale", "Content-Range" to "0-99"), 0, 100), + Part("https://p/2", mapOf("Content-Range" to "100-249"), 100, 250), + ) + val e = entry(descriptor = desc(url = null, file = "/f", parts = parts)) + val patched = e.withHeadersPatched(mapOf("Authorization" to "Bearer new"), 1).descriptor!!.parts!! + assertEquals(mapOf("Content-Range" to "0-99", "Authorization" to "Bearer new"), patched[0].headers) + assertEquals(mapOf("Content-Range" to "100-249"), patched[1].headers) + } + + @Test + fun `toRow carries vars as an object and nextAttemptAt only when set`() { + val row = entry().toRow().toMap() + assertEquals(mapOf("n" to 1.0), row["vars"]) + assertEquals("queued", row["state"]) + assertFalse(row.containsKey("nextAttemptAt")) + assertEquals( + setOf("id", "key", "vars", "state", "bytesSent", "totalBytes", "attempts", "updatedAt"), + row.keys, + ) + val waiting = entry(nextAttemptAt = 9_000).toRow().toMap() + assertEquals(9_000.0, waiting["nextAttemptAt"]) + } + + @Test + fun `a malformed vars text reads as null in the row`() { + assertNull(entry().copy(varsJson = "{bad").toRow().vars) + } + + @Test + fun `isLive covers the four live states`() { + val live = EntryState.values().filter { it.isLive }.toSet() + assertEquals(setOf(EntryState.QUEUED, EntryState.RUNNING, EntryState.AWAITING_AUTH, EntryState.PAUSED), live) + } + + @Test + fun `header maps match names without regard to case`() { + assertTrue(HeaderMap.contains(mapOf("Content-Type" to "x"), "content-type")) + assertEquals("x", HeaderMap.get(mapOf("content-type" to "x"), "Content-Type")) + assertEquals(mapOf("B" to "2", "a" to "3"), HeaderMap.merge(mapOf("A" to "1", "B" to "2"), mapOf("a" to "3"))) + assertEquals(mapOf("B" to "2"), HeaderMap.without(mapOf("a" to "1", "B" to "2"), "A")) + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/QueueSettingsTest.kt b/android/src/test/java/ai/openspace/backgroundupload/QueueSettingsTest.kt new file mode 100644 index 00000000..6dbcfff7 --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/QueueSettingsTest.kt @@ -0,0 +1,68 @@ +package ai.openspace.backgroundupload + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import java.io.File + +class QueueSettingsTest { + @get:Rule + val tmp = TemporaryFolder() + + @Test + fun `defaults when there is no file`() { + val s = QueueSettingsStore(File(tmp.newFolder(), "settings.json")).load() + assertEquals(QueueSettings(), s) + assertEquals(RetryDefaults(1_000, 7_200_000, 0.2, listOf(404)), s.retry) + } + + @Test + fun `update persists and reloads in a new instance`() { + val file = File(tmp.newFolder(), "settings.json") + QueueSettingsStore(file).update { it.copy(wifiOnly = true, paused = true) } + val reloaded = QueueSettingsStore(file).load() + assertTrue(reloaded.wifiOnly) + assertTrue(reloaded.paused) + } + + @Test + fun `a corrupt file reads as the defaults`() { + val file = File(tmp.newFolder(), "settings.json").apply { writeText("{not json") } + assertEquals(QueueSettings(), QueueSettingsStore(file).load()) + } + + @Test + fun `a file missing fields keeps the defaults for them`() { + val file = File(tmp.newFolder(), "settings.json").apply { writeText("""{"wifiOnly":true}""") } + val s = QueueSettingsStore(file).load() + assertTrue(s.wifiOnly) + assertFalse(s.paused) + assertEquals(RetryDefaults(), s.retry) + } + + @Test + fun `header generation increments`() { + val store = QueueSettingsStore(File(tmp.newFolder(), "settings.json")) + assertEquals(1, store.update { it.copy(headerGeneration = it.headerGeneration + 1) }.headerGeneration) + assertEquals(2, store.update { it.copy(headerGeneration = it.headerGeneration + 1) }.headerGeneration) + } + + @Test + fun `a failed write keeps the old value`() { + // A regular file where the directory should be: the write fails. + val notADir = tmp.newFile() + val store = QueueSettingsStore(File(notADir, "settings.json")) + runCatching { store.update { it.copy(paused = true) } } + assertFalse(store.load().paused) + } + + @Test + fun `configure retry fills absent fields with the library defaults`() { + val d = QueueSettingsStore.retryDefaults(mapOf("backoff" to mapOf("baseMs" to 500.0), "terminalHttp" to mapOf("exempt" to listOf()))) + assertEquals(RetryDefaults(baseMs = 500, exempt = emptyList()), d) + assertEquals(RetryDefaults(), QueueSettingsStore.retryDefaults(null)) + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/QueueStoreTest.kt b/android/src/test/java/ai/openspace/backgroundupload/QueueStoreTest.kt new file mode 100644 index 00000000..98d107ba --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/QueueStoreTest.kt @@ -0,0 +1,253 @@ +package ai.openspace.backgroundupload + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNotNull +import org.junit.Assert.assertNull +import org.junit.Assert.assertThrows +import org.junit.Assert.assertTrue +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import java.io.File +import java.io.IOException +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit + +class QueueStoreTest { + @get:Rule + val tmp = TemporaryFolder() + + private fun store(dir: File = tmp.newFolder(), index: RequestIndex = RequestIndex()) = QueueStore(dir, index) + + @Test + fun `save then load round-trips across store instances`() { + val dir = tmp.newFolder() + val e = entry( + descriptor = desc(url = null, method = "PUT", file = "/f", parts = listOf(part(0, 10, accepted = true), part(10, 20))), + body = StagedBody(StagedBody.CHUNKED, "blob", null, 20), + nextAttemptAt = 5, + parkedGeneration = 2, + ) + store(dir).save(e) + // A new instance over the same dir is what a process relaunch looks like. + assertEquals(e, store(dir).load("e1")) + } + + @Test + fun `the state enum is stored by its wire string`() { + val s = store() + s.save(entry(state = EntryState.AWAITING_AUTH)) + assertTrue(File(s.entryDir("e1"), QueueStore.ENTRY_FILE).readText().contains("\"awaiting-auth\"")) + } + + @Test + fun `ids with filesystem-hostile characters round-trip`() { + val s = store() + val id = "a/b:c dü..\\e" + s.save(entry(id = id)) + assertEquals(id, s.load(id)!!.id) + s.remove(id) + assertNull(s.load(id)) + } + + @Test + fun `a corrupt entry reads as absent`() { + val s = store() + s.save(entry()) + File(s.entryDir("e1"), QueueStore.ENTRY_FILE).writeText("{not json") + assertNull(s.load("e1")) + assertEquals(emptyList(), s.all()) + } + + @Test + fun `an entry missing a required field reads as absent`() { + val s = store() + s.entryDir("e1").mkdirs() + File(s.entryDir("e1"), QueueStore.ENTRY_FILE).writeText("""{"id":"e1","key":"k","state":"queued"}""") + assertNull(s.load("e1")) // not legacy, and no descriptor or body + } + + @Test + fun `an older file gets safe defaults`() { + val s = store() + s.entryDir("e1").mkdirs() + File(s.entryDir("e1"), QueueStore.ENTRY_FILE).writeText( + """{"id":"e1","key":"k","state":"queued","descriptor":{"url":"https://x"},"body":{"kind":"none","totalBytes":0}}""", + ) + val e = s.load("e1")!! + assertEquals("null", e.varsJson) + assertEquals("POST", e.descriptor!!.method) + assertEquals(emptyMap(), e.descriptor!!.headers) + assertEquals(1, e.generation) + } + + @Test + fun `all skips v9-only directories and legacyManifest reads them`() { + val s = store() + val dir = s.entryDir("v9").apply { mkdirs() } + File(dir, QueueStore.V9_MANIFEST_FILE).writeText( + """{"id":"v9","sourcePath":"/x/blob","parts":[{"url":"https://p/1","headers":{},"start":0,"end":10,"accepted":true}],""" + + """"accept":[{"status":409}],"expiresAt":99,"wifiOnly":false,"noNotification":true,"createdAt":1}""", + ) + s.save(entry(id = "v10")) + assertEquals(listOf("v10"), s.all().map { it.id }) + val m = s.legacyManifest("v9")!! + assertEquals(listOf(Part("https://p/1", mapOf(), 0, 10, accepted = true)), m.parts) + assertEquals(listOf(UploadOutcome.AcceptRule(409)), m.accept) + assertNull(s.legacyManifest("v10")) + assertNull(s.legacyManifest("nope")) + } + + @Test + fun `compute holds the lock across load, transform, and save`() { + // A module transition and a worker's update race. If the lock did not + // span all three steps, the update could land between load and save and + // be erased. The update must block while the transform runs. + val s = store() + s.save(entry(descriptor = desc(url = null, file = "/f", parts = listOf(part(0, 10), part(10, 20))))) + val inTransform = CountDownLatch(1) + val finish = CountDownLatch(1) + val computing = Thread { + s.compute("e1") { e -> + inTransform.countDown() + finish.await(5, TimeUnit.SECONDS) + e!!.copy(expiresAt = 99_000) + } + }.apply { start() } + assertTrue(inTransform.await(5, TimeUnit.SECONDS)) + val updating = Thread { + s.update("e1") { e -> e.copy(descriptor = e.descriptor!!.copy(parts = ChunkedParts.withAccepted(e.descriptor.parts!!, 0))) } + }.apply { start() } + // Without the lock the update finishes (TERMINATED) while the transform waits. + val deadline = System.currentTimeMillis() + 5_000 + while (updating.state != Thread.State.BLOCKED && updating.state != Thread.State.TERMINATED && + System.currentTimeMillis() < deadline + ) Thread.sleep(5) + assertEquals(Thread.State.BLOCKED, updating.state) + finish.countDown() + computing.join() + updating.join() + val final = s.load("e1")!! + assertEquals(99_000, final.expiresAt) + assertTrue(final.descriptor!!.parts!![0].accepted) + } + + @Test + fun `a throwing transform writes nothing`() { + val s = store() + s.save(entry()) + assertThrows(IllegalStateException::class.java) { s.compute("e1") { throw IllegalStateException("no") } } + assertEquals(entry(), s.load("e1")) + } + + @Test + fun `compute returning the same object or null writes nothing`() { + val s = store() + assertNull(s.compute("e1") { null }) + assertFalse(s.entryDir("e1").exists() && File(s.entryDir("e1"), QueueStore.ENTRY_FILE).exists()) + s.save(entry()) + val file = File(s.entryDir("e1"), QueueStore.ENTRY_FILE) + file.setLastModified(1_000) + s.compute("e1") { it } + assertEquals(1_000, file.lastModified()) + } + + @Test + fun `remove deletes the row and every staged byte`() { + val index = RequestIndex() + val s = store(index = index) + s.save(entry()) + File(s.entryDir("e1"), "body-1.json").writeText("{}") + File(s.entryDir("e1"), "blob").writeText("bytes") + s.remove("e1") + assertNull(s.load("e1")) + assertFalse(s.entryDir("e1").exists()) + assertEquals(emptyList(), index.snapshot()) + } + + @Test + fun `the index follows every save and remove, and loads on start`() { + val dir = tmp.newFolder() + val index = RequestIndex() + val s = store(dir, index) + s.save(entry(id = "a")) + s.save(entry(id = "b", state = EntryState.ERROR)) + assertEquals(listOf("a", "b"), index.snapshot().map { it.id }) + s.remove("a") + assertEquals(listOf("b"), index.snapshot().map { it.id }) + // A process relaunch: a fresh index loaded from disk. + val fresh = RequestIndex() + QueueStore(dir, fresh).loadIndex() + assertEquals(listOf("error"), fresh.snapshot().map { it.state }) + } + + @Test + fun `pruneUnreferenced keeps only the entry file and its body`() { + val s = store() + val e = entry(body = StagedBody(StagedBody.JSON, "body-2.json", null, 2)) + s.save(e) + val dir = s.entryDir("e1") + listOf("body-1.json", "body-2.json", "blob", "manifest.json", "entry.json.tmp").forEach { File(dir, it).writeText("x") } + s.pruneUnreferenced(e) + assertEquals(setOf("entry.json", "body-2.json"), dir.list()!!.toSet()) + } + + // MARK: - crash mid-write + + @Test + fun `crash mid-write (a) a partial entry tmp next to a valid entry`() { + val s = store() + s.save(entry(attempts = 1)) + // The process died while writing the next version: only the tmp is partial. + File(s.entryDir("e1"), "entry.json.tmp").writeText("""{"id":"e1","key":"no""") + assertEquals(1, s.load("e1")!!.attempts) + s.save(entry(attempts = 2)) + assertEquals(2, s.load("e1")!!.attempts) + assertFalse(File(s.entryDir("e1"), "entry.json.tmp").exists()) + } + + @Test + fun `crash mid-write (b) a staged body with no entry is not a row`() { + val s = store() + val dir = s.entryDir("e1").apply { mkdirs() } + File(dir, "body-1.json.tmp").writeText("{\"a\":") + File(dir, "body-1.json").writeText("{\"a\":1}") + assertEquals(emptyList(), s.all()) + assertNull(s.load("e1")) + // The next create saves an entry and prunes what it does not use. + val e = entry(body = StagedBody(StagedBody.JSON, "body-1.json", null, 7)) + s.save(e) + s.pruneUnreferenced(e) + assertEquals(setOf("entry.json", "body-1.json"), dir.list()!!.toSet()) + } + + @Test + fun `crash mid-write (c) a write that throws leaves the old target intact`() { + val target = File(tmp.newFolder(), "entry.json").apply { writeText("old") } + assertThrows(IOException::class.java) { + AtomicFiles.writeAtomically(target) { out -> + out.write("new, half".toByteArray()) + throw IOException("disk full") + } + } + assertEquals("old", target.readText()) + assertFalse(AtomicFiles.tmpFor(target).exists()) + } + + @Test + fun `a save into an unwritable directory throws`() { + val notADir = tmp.newFile() + val s = QueueStore(notADir, RequestIndex()) + assertThrows(IOException::class.java) { s.save(entry()) } + } + + @Test + fun `update is best effort`() { + val s = store() + assertNull(s.update("nope") { it }) + s.save(entry()) + assertNotNull(s.update("e1") { it.copy(attempts = 3) }) + assertEquals(3, s.load("e1")!!.attempts) + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/RetryClassifierTest.kt b/android/src/test/java/ai/openspace/backgroundupload/RetryClassifierTest.kt new file mode 100644 index 00000000..189bf8be --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/RetryClassifierTest.kt @@ -0,0 +1,108 @@ +package ai.openspace.backgroundupload + +import ai.openspace.backgroundupload.RetryClassifier.Verdict +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test +import java.io.IOException +import kotlin.random.Random + +class RetryClassifierTest { + private val defaultExempt = listOf(404) + + private fun classify(code: Int, body: String = "", accept: List = emptyList(), exempt: List = defaultExempt) = + RetryClassifier.classifyResponse(code, body, accept, exempt) + + @Test + fun `the retry table`() { + assertEquals(Verdict.Accepted, classify(200)) + assertEquals(Verdict.Accepted, classify(204)) + assertEquals(Verdict.Accepted, classify(409, "upload already completed", listOf(UploadOutcome.AcceptRule(409, "already completed")))) + assertEquals(Verdict.Terminal("http", "HTTP 409"), classify(409, "conflict", listOf(UploadOutcome.AcceptRule(409, "already completed")))) + assertEquals(Verdict.Auth, classify(401)) + assertEquals(Verdict.Auth, classify(403)) + assertEquals(Verdict.Transient, classify(408)) + assertEquals(Verdict.Transient, classify(429)) + assertEquals(Verdict.Transient, classify(500)) + assertEquals(Verdict.Transient, classify(599)) + assertEquals(Verdict.Transient, classify(404)) // default exempt + assertEquals(Verdict.Terminal("http", "HTTP 400"), classify(400)) + assertEquals(Verdict.Terminal("http", "HTTP 409"), classify(409)) + assertEquals(Verdict.Terminal("http", "HTTP 304"), classify(304)) + assertEquals(Verdict.Terminal("http", "HTTP 101"), classify(101)) + } + + @Test + fun `a chunked part 404 with exempt empty is terminal`() { + assertEquals(Verdict.Terminal("http", "HTTP 404"), classify(404, exempt = emptyList())) + } + + @Test + fun `an auth status stays auth even when exempt lists it`() { + assertEquals(Verdict.Auth, classify(401, exempt = listOf(401))) + } + + @Test + fun `transport failures`() { + assertEquals(Verdict.Transient, RetryClassifier.classifyFailure(IOException("reset"), fileExists = true)) + val file = RetryClassifier.classifyFailure(IOException("ENOENT"), fileExists = false) + assertTrue(file is Verdict.Terminal && file.errorKind == "file") + val other = RetryClassifier.classifyFailure(IllegalArgumentException("bad url"), fileExists = true) + assertEquals(Verdict.Terminal("unknown", "bad url"), other) + } + + @Test + fun `the attempt errorKind of a failure comes from its verdict`() { + assertEquals("network", RetryClassifier.failureKind(RetryClassifier.classifyFailure(IOException(), true))) + assertEquals("file", RetryClassifier.failureKind(RetryClassifier.classifyFailure(IOException(), false))) + assertEquals("unknown", RetryClassifier.failureKind(RetryClassifier.classifyFailure(RuntimeException("boom"), true))) + } + + private val policy = RetryClassifier.Policy(baseMs = 1_000, maxMs = 7_200_000, jitter = 0.0, exempt = defaultExempt) + + @Test + fun `backoff doubles from base and caps at max`() { + assertEquals(1_000, RetryClassifier.backoffMs(policy, 1)) + assertEquals(2_000, RetryClassifier.backoffMs(policy, 2)) + assertEquals(4_000, RetryClassifier.backoffMs(policy, 3)) + assertEquals(7_200_000, RetryClassifier.backoffMs(policy, 14)) + assertEquals(7_200_000, RetryClassifier.backoffMs(policy, 10_000)) + assertEquals(1_000, RetryClassifier.backoffMs(policy, 0)) // defensive + } + + @Test + fun `jitter stays within bounds and under max`() { + val jittered = policy.copy(jitter = 0.2) + val random = Random(42) + repeat(1_000) { + val ms = RetryClassifier.backoffMs(jittered, 3, random) + assertTrue("$ms", ms in 3_200..4_800) + } + repeat(1_000) { + assertTrue(RetryClassifier.backoffMs(jittered, 30, random) <= 7_200_000) + } + } + + @Test + fun `nextAttemptAt clamps to expiresAt`() { + assertEquals(3_000, RetryClassifier.nextAttemptAt(now = 1_000, backoffMs = 2_000, expiresAt = 10_000)) + assertEquals(10_000, RetryClassifier.nextAttemptAt(now = 1_000, backoffMs = 7_200_000, expiresAt = 10_000)) + } + + @Test + fun `expiry is inclusive of the deadline`() { + assertFalse(RetryClassifier.isExpired(4_999, 5_000)) + assertTrue(RetryClassifier.isExpired(5_000, 5_000)) + } + + @Test + fun `policy overrides field by field`() { + val defaults = RetryDefaults() + assertEquals(RetryClassifier.Policy(1_000, 7_200_000, 0.2, listOf(404)), RetryClassifier.policy(defaults, null)) + assertEquals( + RetryClassifier.Policy(50, 7_200_000, 0.2, emptyList()), + RetryClassifier.policy(defaults, RetryOverride(baseMs = 50, maxMs = null, jitter = null, exempt = emptyList())), + ) + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/SmallPartsTest.kt b/android/src/test/java/ai/openspace/backgroundupload/SmallPartsTest.kt new file mode 100644 index 00000000..a5bfda74 --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/SmallPartsTest.kt @@ -0,0 +1,176 @@ +package ai.openspace.backgroundupload + +import androidx.work.WorkInfo.State.BLOCKED +import androidx.work.WorkInfo.State.CANCELLED +import androidx.work.WorkInfo.State.ENQUEUED +import androidx.work.WorkInfo.State.FAILED +import androidx.work.WorkInfo.State.RUNNING +import androidx.work.WorkInfo.State.SUCCEEDED +import kotlinx.coroutines.runBlocking +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test + +class AttemptEventTest { + private val response = UploadResponse(401, "x".repeat(5_000), mapOf("a" to "b")) + + @Test + fun `the body is cut at 4 KB of UTF-8 and flagged`() { + val e = AttemptEvent.ofResponse(entry(attempts = 2), "r1", "https://x", null, response, accepted = false, at = 7) + assertEquals(BodyCap.ATTEMPT_MAX_BYTES, e.responseBody!!.length) + assertEquals(true, e.responseBodyTruncated) + assertEquals(2, e.attempt) + // 3-byte characters: the cut backs off to a whole character. + val wide = AttemptEvent.ofResponse(entry(), "r1", "https://x", null, response.copy(body = "\u20ac".repeat(2_000)), false, 7) + assertEquals(BodyCap.ATTEMPT_MAX_BYTES / 3, wide.responseBody!!.length) + } + + @Test + fun `a response the stream cap cut is flagged even under 4 KB`() { + val e = AttemptEvent.ofResponse(entry(), "r1", "https://x", null, response.copy(body = "short", truncated = true), false, 7) + assertEquals("short", e.responseBody) + assertEquals(true, e.responseBodyTruncated) + } + + @Test + fun `outcome is completed only when accepted`() { + val rejected = AttemptEvent.ofResponse(entry(), "r1", "https://x", null, response, accepted = false, at = 7) + assertEquals("error", rejected.outcome) + assertEquals(401, rejected.httpCode) + assertEquals("http", rejected.errorKind) + val ok = AttemptEvent.ofResponse(entry(), "r1", "https://x", 3, response.copy(code = 200, body = "ok"), accepted = true, at = 7) + assertEquals("completed", ok.outcome) + assertNull(ok.errorKind) + assertEquals(3.0, ok.toMap()["partIndex"]) + } + + @Test + fun `partIndex and response fields are optional in the map`() { + val failure = AttemptEvent.ofFailure(entry(), "r1", "https://x", null, "network", "reset", 7).toMap() + assertEquals( + setOf("id", "key", "requestId", "attempt", "url", "method", "outcome", "errorKind", "errorMessage", "at"), + failure.keys, + ) + } +} + +class ProgressThrottleTest { + private var now = 0L + private val emitted = mutableListOf() + private val throttle = ProgressThrottle({ now }) { _, sent, _ -> emitted += sent } + + @Test + fun `the first offer emits, then at most one per second in the foreground`() { + throttle.offer("a", 1, 10, foreground = true) + now = 500 + throttle.offer("a", 2, 10, foreground = true) + assertEquals(listOf(1L), emitted) + now = 1_000 + throttle.offer("a", 3, 10, foreground = true) + assertEquals(listOf(1L, 3L), emitted) + } + + @Test + fun `the background interval is 10 minutes`() { + throttle.offer("a", 1, 10, foreground = false) + now = 599_999 + throttle.offer("a", 2, 10, foreground = false) + assertEquals(listOf(1L), emitted) + now = 600_000 + throttle.offer("a", 3, 10, foreground = false) + assertEquals(listOf(1L, 3L), emitted) + } + + @Test + fun `flush sends the held value once`() { + throttle.offer("a", 1, 10, foreground = true) + throttle.offer("a", 2, 10, foreground = true) + throttle.flush("a") + throttle.flush("a") + assertEquals(listOf(1L, 2L), emitted) + } + + @Test + fun `ids are independent, and drop forgets an id`() { + throttle.offer("a", 1, 10, foreground = true) + throttle.offer("b", 5, 10, foreground = true) + assertEquals(listOf(1L, 5L), emitted) + throttle.offer("a", 2, 10, foreground = true) + throttle.drop("a") + throttle.flush("a") + assertEquals(listOf(1L, 5L), emitted) + } +} + +class RequestIndexTest { + @Test + fun `put, remove, and snapshot order`() { + val index = RequestIndex() + index.put(entry(id = "b", createdAt = 2).toRow()) + index.put(entry(id = "a", createdAt = 2).toRow()) + index.put(entry(id = "c", createdAt = 1).toRow()) + assertEquals(listOf("c", "a", "b"), index.snapshot().map { it.id }) + index.remove("a") + assertEquals(listOf("c", "b"), index.snapshot().map { it.id }) + } + + @Test + fun `setBytes on a missing id is a no-op`() { + val index = RequestIndex() + index.setBytes("nope", 5) + assertEquals(emptyList(), index.snapshot()) + } + + @Test + fun `a save of a running entry does not move bytes backwards`() { + val index = RequestIndex() + val running = entry(state = EntryState.RUNNING, body = StagedBody(StagedBody.FILE, "f", null, 100)) + index.put(running.toRow()) + index.setBytes("e1", 60) + index.put(running.copy(attempts = 2).toRow()) + assertEquals(60, index.snapshot().single().bytesSent) + index.put(running.copy(state = EntryState.QUEUED).toRow()) + assertEquals(0, index.snapshot().single().bytesSent) + } +} + +class SchedulerTest { + @Test + fun `initialDelayMs is the time left, never negative`() { + assertEquals(0, WorkManagerScheduler.initialDelayMs(null, 1_000)) + assertEquals(0, WorkManagerScheduler.initialDelayMs(500, 1_000)) + assertEquals(4_000, WorkManagerScheduler.initialDelayMs(5_000, 1_000)) + } + + @Test + fun `a queued successor suppresses another append`() { + assertTrue(WorkManagerScheduler.hasQueuedSuccessor(listOf(RUNNING, BLOCKED))) + assertTrue(WorkManagerScheduler.hasQueuedSuccessor(listOf(ENQUEUED))) + assertFalse(WorkManagerScheduler.hasQueuedSuccessor(listOf(RUNNING))) + assertFalse(WorkManagerScheduler.hasQueuedSuccessor(listOf(SUCCEEDED, FAILED, CANCELLED))) + assertFalse(WorkManagerScheduler.hasQueuedSuccessor(emptyList())) + } + + @Test + fun `the wake name differs from the main chain`() { + assertEquals("e1#wake", WorkManagerScheduler.wakeName("e1")) + } +} + +class TransferSemaphoreTest { + @Test + fun `the global cap is 4 and a fifth request waits`() = runBlocking { + assertEquals(4, MAX_TRANSFER_CONCURRENCY) + repeat(4) { transferSemaphore.acquire() } + try { + assertFalse(transferSemaphore.tryAcquire()) // no fifth permit + transferSemaphore.release() + assertTrue(transferSemaphore.tryAcquire()) // one freed, one taken + } finally { + repeat(4) { transferSemaphore.release() } + } + assertEquals(4, transferSemaphore.availablePermits) + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/TestSupport.kt b/android/src/test/java/ai/openspace/backgroundupload/TestSupport.kt new file mode 100644 index 00000000..1effda61 --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/TestSupport.kt @@ -0,0 +1,125 @@ +package ai.openspace.backgroundupload + +// Builders and fakes shared by the JVM tests. No android.* here. + +internal const val FAR_FUTURE = 4_000_000_000_000L + +internal fun desc( + url: String? = "https://example.com/items", + method: String = "POST", + headers: Map = mapOf("Authorization" to "Bearer old"), + dataJson: String? = null, + form: List? = null, + file: String? = null, + parts: List? = null, + accept: List = emptyList(), + retry: RetryOverride? = null, + noNotification: Boolean = false, +) = Descriptor(url, method, headers, dataJson, form, file, parts, accept, retry, noNotification) + +internal fun part(start: Long, end: Long, accepted: Boolean = false, url: String? = null) = Part( + url = url ?: "https://example.com/part?start=$start", + headers = mapOf("Content-Range" to "$start-${end - 1}"), + start = start, + end = end, + accepted = accepted, +) + +internal fun entry( + id: String = "e1", + state: EntryState = EntryState.QUEUED, + descriptor: Descriptor? = desc(dataJson = """{"a":1}"""), + body: StagedBody? = StagedBody(StagedBody.JSON, "body-1.json", null, 7), + generation: Int = 1, + attempts: Int = 0, + settledEventId: String? = null, + parkedGeneration: Int? = null, + nextAttemptAt: Long? = null, + expiresAt: Long = FAR_FUTURE, + legacy: Boolean = false, + key: String = "note", + createdAt: Long = 1_000, +) = QueueEntry( + id = id, + key = key, + varsJson = """{"n":1}""", + descriptor = descriptor, + body = body, + state = state, + attempts = attempts, + bytesSent = 0, + totalBytes = body?.totalBytes ?: 0, + expiresAt = expiresAt, + createdAt = createdAt, + updatedAt = createdAt, + nextAttemptAt = nextAttemptAt, + parkedGeneration = parkedGeneration, + generation = generation, + settledEventId = settledEventId, + legacy = legacy, +) + +internal fun parsed( + id: String = "e1", + descriptor: Descriptor = desc(dataJson = """{"a":1}"""), + varsJson: String = """{"n":1}""", + expiresAt: Long = FAR_FUTURE, + key: String = "note", +) = EntryParsing.Parsed(id, key, varsJson, descriptor, expiresAt) + +internal fun record( + eventId: String, + id: String = "e1", + kind: String = EventJournal.KIND_COMPLETED, + generation: Int = 1, + at: Long = 5_000, + state: String = kind, +) = EventJournal.SettledRecord( + eventId = eventId, id = id, key = "note", varsJson = """{"n":1}""", at = at, attempts = 1, + requestId = "r1", deliveries = 1, state = state, bytesSent = 0, totalBytes = 0, + url = "https://example.com/items", method = "POST", partIndex = null, kind = kind, + response = if (kind == EventJournal.KIND_COMPLETED) EventJournal.Response(200, mapOf(), "ok", false) else null, + errorKind = if (kind == EventJournal.KIND_ERROR) "http" else null, + message = if (kind == EventJournal.KIND_ERROR) "HTTP 400" else null, + cancelReason = if (kind == EventJournal.KIND_CANCELLED) "user" else null, + generation = generation, +) + +/** Records every event in order, as "state::" and "settled::". */ +internal class RecordingEvents : QueueEvents { + val log = mutableListOf() + val rows = mutableListOf() + val records = mutableListOf() + + /** The listener each settled event went to, in order. */ + val listeners = mutableListOf() + + override fun state(row: RequestRow) { + rows += row + log += "state:${row.id}:${row.state}" + } + + override fun settled(record: EventJournal.SettledRecord, listener: Any) { + records += record + listeners += listener + log += "settled:${record.id}:${record.kind}" + } +} + +internal class FakeScheduler : WorkScheduler { + val scheduled = mutableListOf() + val wakes = mutableListOf>() + val cancelled = mutableListOf() + + override fun schedule(entry: QueueEntry) { + scheduled += entry.id + } + + override fun scheduleWake(entry: QueueEntry, at: Long, replace: Boolean) { + wakes += entry.id to at + } + + override fun cancel(id: String) { + cancelled += id + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/UploadOutcomeTest.kt b/android/src/test/java/ai/openspace/backgroundupload/UploadOutcomeTest.kt index 187349ed..9976f240 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/UploadOutcomeTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/UploadOutcomeTest.kt @@ -5,7 +5,6 @@ import org.junit.Assert.assertEquals import org.junit.Assert.assertFalse import org.junit.Assert.assertTrue import org.junit.Test -import java.io.IOException class UploadOutcomeTest { @@ -57,19 +56,4 @@ class UploadOutcomeTest { assertTrue(UploadOutcome.isAccepted(409, "already completed", rules)) assertFalse(UploadOutcome.isAccepted(410, "already completed", rules)) } - - @Test - fun `IOException with a missing file is a file error`() { - assertEquals("file", UploadOutcome.errorKind(IOException("gone"), fileExists = false)) - } - - @Test - fun `IOException with the file present is a network error`() { - assertEquals("network", UploadOutcome.errorKind(IOException("reset"), fileExists = true)) - } - - @Test - fun `a non-IO error is unknown`() { - assertEquals("unknown", UploadOutcome.errorKind(RuntimeException("boom"), fileExists = true)) - } } diff --git a/android/src/test/java/ai/openspace/backgroundupload/UploadStatesTest.kt b/android/src/test/java/ai/openspace/backgroundupload/UploadStatesTest.kt deleted file mode 100644 index bef1858c..00000000 --- a/android/src/test/java/ai/openspace/backgroundupload/UploadStatesTest.kt +++ /dev/null @@ -1,84 +0,0 @@ -package ai.openspace.backgroundupload - -import androidx.work.WorkInfo.State.BLOCKED -import androidx.work.WorkInfo.State.CANCELLED -import androidx.work.WorkInfo.State.ENQUEUED -import androidx.work.WorkInfo.State.FAILED -import androidx.work.WorkInfo.State.RUNNING -import androidx.work.WorkInfo.State.SUCCEEDED -import androidx.work.ListenableWorker -import org.junit.Assert.assertEquals -import org.junit.Assert.assertFalse -import org.junit.Assert.assertTrue -import org.junit.Test - -// getAllUploads must return ONE row per upload id, although WorkManager can -// hold several rows for it (finished chains linger for roughly a day, and -// APPEND_OR_REPLACE resumes add rows). The state vocabulary is the same as -// iOS's. -class UploadStatesTest { - - @Test - fun `a live row wins for a chunked upload`() { - assertEquals("running", chunkedUploadState(listOf(CANCELLED, RUNNING), allAccepted = false)) - assertEquals("running", chunkedUploadState(listOf(RUNNING, BLOCKED), allAccepted = false)) - assertEquals("pending", chunkedUploadState(listOf(FAILED, ENQUEUED), allAccepted = false)) - assertEquals("pending", chunkedUploadState(listOf(BLOCKED), allAccepted = false)) - } - - @Test - fun `with no live row the manifest speaks, never a lingering finished row`() { - // A cancelled chunked upload keeps its manifest. Its truthful state is - // stalled-awaiting-resume ("error"), not "cancelled". iOS's getAllUploads - // never reports "cancelled" for a lingering upload. - assertEquals("error", chunkedUploadState(listOf(CANCELLED), allAccepted = false)) - assertEquals("error", chunkedUploadState(listOf(FAILED), allAccepted = false)) - assertEquals("error", chunkedUploadState(emptyList(), allAccepted = false)) - assertEquals("completed", chunkedUploadState(listOf(SUCCEEDED), allAccepted = true)) - assertEquals("completed", chunkedUploadState(emptyList(), allAccepted = true)) - // The row that a run leaves after it journals a terminal error is - // SUCCEEDED (see terminalErrorResult). The manifest, not the row, carries - // the outcome. - assertEquals("error", chunkedUploadState(listOf(SUCCEEDED), allAccepted = false)) - } - - @Test - fun `a journaled terminal error still succeeds the row`() { - // WorkManager marks the dependents of a FAILED prerequisite FAILED without - // a run. Thus a resume appended during a failing run's teardown would - // silently never run. The journal and the manifest are the outcome record, - // never the row state. - assertTrue(terminalErrorResult() is ListenableWorker.Result.Success) - } - - @Test - fun `cancel reports from the module only when no worker is running`() { - assertTrue(cancelReportsFromModule(listOf(ENQUEUED))) - // An appended chain's dependent is BLOCKED, not ENQUEUED. It is still - // never-started, and it is still owed a module-side 'cancelled'. - assertTrue(cancelReportsFromModule(listOf(BLOCKED))) - assertTrue(cancelReportsFromModule(listOf(ENQUEUED, BLOCKED))) - // A RUNNING worker's stop handler owns the report. - assertFalse(cancelReportsFromModule(listOf(RUNNING))) - assertFalse(cancelReportsFromModule(listOf(RUNNING, BLOCKED))) - assertFalse(cancelReportsFromModule(emptyList())) - } - - @Test - fun `a queued successor suppresses another append`() { - assertTrue(hasQueuedSuccessor(listOf(RUNNING, BLOCKED))) - assertTrue(hasQueuedSuccessor(listOf(ENQUEUED))) - assertFalse(hasQueuedSuccessor(listOf(RUNNING))) - assertFalse(hasQueuedSuccessor(listOf(SUCCEEDED, FAILED, CANCELLED))) - assertFalse(hasQueuedSuccessor(emptyList())) - } - - @Test - fun `a simple upload reports its live row first, then the most conclusive finished one`() { - assertEquals("running", simpleUploadState(listOf(CANCELLED, RUNNING))) - assertEquals("pending", simpleUploadState(listOf(SUCCEEDED, ENQUEUED))) - assertEquals("completed", simpleUploadState(listOf(CANCELLED, SUCCEEDED))) - assertEquals("error", simpleUploadState(listOf(CANCELLED, FAILED))) - assertEquals("cancelled", simpleUploadState(listOf(CANCELLED))) - } -} diff --git a/android/src/test/java/ai/openspace/backgroundupload/UploadTest.kt b/android/src/test/java/ai/openspace/backgroundupload/UploadTest.kt deleted file mode 100644 index a99178d1..00000000 --- a/android/src/test/java/ai/openspace/backgroundupload/UploadTest.kt +++ /dev/null @@ -1,112 +0,0 @@ -package ai.openspace.backgroundupload - -import com.google.gson.Gson -import org.junit.Assert.assertEquals -import org.junit.Assert.assertFalse -import org.junit.Assert.assertTrue -import org.junit.Test - -class UploadTest { - private val gson = Gson() - - private fun upload(noNotification: Boolean) = Upload( - id = "u1", - url = "https://example.com/upload", - path = "/tmp/file", - method = "POST", - wifiOnly = false, - accept = listOf(), - headers = mapOf(), - noNotification = noNotification, - ) - - @Test - fun `an upload notifies unless it opts out`() { - assertTrue(upload(noNotification = false).showsNotification) - assertFalse(upload(noNotification = true).showsNotification) - } - - @Test - fun `the opt-out survives a serialization round trip`() { - val json = gson.toJson(upload(noNotification = true)) - assertFalse(gson.fromJson(json, Upload::class.java).showsNotification) - } - - // WorkManager stores this model as JSON, so an upload can be enqueued by one - // build and run by the next. A job from a build without the option must keep - // its notification rather than silently losing foreground mode. - @Test - fun `a job enqueued without the option still notifies`() { - val json = gson.toJsonTree(upload(noNotification = true)).asJsonObject - json.remove(Upload::noNotification.name) - assertTrue(gson.fromJson(json, Upload::class.java).showsNotification) - } - - // One build can enqueue a WorkManager job, and the next build can replay it. - // This is the exact JSON shape that a v8 build serialized into input data - // (Gson.toJson of the v8 Upload model): `acceptStatus: List`, and no - // `accept`. Gson does not use the constructor. Thus, without normalized(), - // the replayed object's `accept` is NULL, and the worker NPEs after the file - // has fully transmitted. WorkManager then re-runs it and re-sends the whole - // file. - private val v8JobJson = """ - { - "id": "u1", - "url": "https://example.com/upload", - "path": "/tmp/file", - "method": "PUT", - "maxRetries": 5, - "wifiOnly": false, - "acceptStatus": [409, 208], - "headers": {"Authorization": "Bearer t"}, - "notificationId": 123456, - "notificationTitle": "Uploading…", - "notificationTitleNoInternet": "Waiting for connection…", - "notificationTitleNoWifi": "Waiting for Wi-Fi…", - "notificationChannel": "background-upload", - "noNotification": false - } - """ - - @Test - fun `a replayed v8 job maps acceptStatus to accept rules and is safe to run`() { - val replayed = gson.fromJson(v8JobJson, Upload::class.java).normalized() - assertEquals( - listOf(UploadOutcome.AcceptRule(409), UploadOutcome.AcceptRule(208)), - replayed.accept, - ) - // The worker-facing calls that NPE'd on the un-normalized object. - assertTrue(UploadOutcome.isAccepted(409, "duplicate", replayed.accept)) - assertFalse(UploadOutcome.isAccepted(400, "", replayed.accept)) - assertEquals("u1", replayed.id) - assertEquals(mapOf("Authorization" to "Bearer t"), replayed.headers) - assertTrue(replayed.showsNotification) - } - - @Test - fun `a replayed v8 job with an empty acceptStatus gets no rules`() { - val json = gson.fromJson(v8JobJson, com.google.gson.JsonObject::class.java) - json.add("acceptStatus", com.google.gson.JsonArray()) - val replayed = gson.fromJson(json, Upload::class.java).normalized() - assertEquals(emptyList(), replayed.accept) - assertTrue(UploadOutcome.isAccepted(200, "", replayed.accept)) - } - - @Test - fun `a job with neither accept nor acceptStatus normalizes to no rules`() { - val json = gson.fromJson(v8JobJson, com.google.gson.JsonObject::class.java) - json.remove("acceptStatus") - val replayed = gson.fromJson(json, Upload::class.java).normalized() - assertEquals(emptyList(), replayed.accept) - assertFalse(UploadOutcome.isAccepted(409, "duplicate", replayed.accept)) - } - - @Test - fun `normalized passes a current-shape job through unchanged`() { - val current = upload(noNotification = true).copy( - accept = listOf(UploadOutcome.AcceptRule(409, "already completed")), - ) - val replayed = gson.fromJson(gson.toJson(current), Upload::class.java).normalized() - assertEquals(current, replayed) - } -} diff --git a/android/src/test/java/ai/openspace/backgroundupload/WorkerGateTest.kt b/android/src/test/java/ai/openspace/backgroundupload/WorkerGateTest.kt new file mode 100644 index 00000000..dde4a6b0 --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/WorkerGateTest.kt @@ -0,0 +1,51 @@ +package ai.openspace.backgroundupload + +import org.junit.After +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test + +class WorkerGateTest { + private val a = Any() + private val b = Any() + + @After + fun tearDown() { + // A process-wide singleton. Leave nothing for other tests. + listOf("g1", "g2").forEach { id -> WorkerGate.release(id, a); WorkerGate.release(id, b) } + } + + @Test + fun `a second worker for the same id must wait`() { + assertTrue(WorkerGate.tryAcquire("g1", a)) + assertFalse(WorkerGate.tryAcquire("g1", b)) + WorkerGate.release("g1", a) + assertTrue(WorkerGate.tryAcquire("g1", b)) + } + + @Test + fun `reacquiring with the same token is idempotent`() { + assertTrue(WorkerGate.tryAcquire("g1", a)) + assertTrue(WorkerGate.tryAcquire("g1", a)) + } + + @Test + fun `a stale release can not evict a successor`() { + assertTrue(WorkerGate.tryAcquire("g1", a)) + WorkerGate.release("g1", a) + assertTrue(WorkerGate.tryAcquire("g1", b)) + WorkerGate.release("g1", a) + assertTrue(WorkerGate.isRunning("g1")) + assertFalse(WorkerGate.tryAcquire("g1", a)) + } + + @Test + fun `ids are independent`() { + assertTrue(WorkerGate.tryAcquire("g1", a)) + assertFalse(WorkerGate.isRunning("g2")) + assertTrue(WorkerGate.tryAcquire("g2", a)) + WorkerGate.release("g1", a) + assertFalse(WorkerGate.isRunning("g1")) + assertTrue(WorkerGate.isRunning("g2")) + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/WorkerOpsTest.kt b/android/src/test/java/ai/openspace/backgroundupload/WorkerOpsTest.kt new file mode 100644 index 00000000..4ffb7fac --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/WorkerOpsTest.kt @@ -0,0 +1,436 @@ +package ai.openspace.backgroundupload + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNotNull +import org.junit.Assert.assertNull +import org.junit.Assert.assertThrows +import org.junit.Assert.assertTrue +import org.junit.Before +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import java.io.File +import java.util.concurrent.CyclicBarrier + +class WorkerOpsTest { + @get:Rule + val tmp = TemporaryFolder() + + private lateinit var store: QueueStore + private lateinit var journal: EventJournal + private lateinit var settings: QueueSettingsStore + private val events = RecordingEvents() + private val scheduler = FakeScheduler() + private var now = 20_000L + private lateinit var ops: WorkerOps + private lateinit var controller: QueueController + private val listener = Any() + private lateinit var journalDir: File + + @Before + fun setUp() { + val root = tmp.newFolder("queue") + store = QueueStore(root, RequestIndex()) + journalDir = tmp.newFolder("journal") + journal = EventJournal(journalDir, retryLater = { _, _ -> }) + settings = QueueSettingsStore(File(root, "settings.json")) + ops = WorkerOps(store, journal, settings, events, scheduler) { now } + controller = QueueController(store, journal, settings, events, scheduler, { false }, { now }) + journal.drain(listener) // JS is subscribed + } + + private fun withJournalReadOnly(block: () -> T): T { + journalDir.setWritable(false) + try { + return block() + } finally { + journalDir.setWritable(true) + } + } + + private val ok = UploadResponse(200, """{"id":7}""", mapOf("x" to "y")) + + @Test + fun `begin takes a queued entry and emits running`() { + store.save(entry(nextAttemptAt = 5)) + val e = ops.begin("e1")!! + assertEquals(EntryState.RUNNING, e.state) + assertNull(e.nextAttemptAt) + assertEquals(listOf("state:e1:running"), events.log) + } + + @Test + fun `begin applies a record of the entry's own generation and does not run it again`() { + // The process died between the journal write and the store transition, + // and WorkManager runs the entry again before any boot sweep. + val own = "00000000-0000-0000-0000-000000000021" + val older = "00000000-0000-0000-0000-000000000022" + store.save(entry(state = EntryState.RUNNING, generation = 2)) + journal.append(record(older, generation = 2, at = 1)) + journal.append(record(own, generation = 2, at = 2)) + assertNull(ops.begin("e1")) + val e = store.load("e1")!! + assertEquals(EntryState.COMPLETED, e.state) + assertEquals(own, e.settledEventId) + assertEquals(listOf(own), journal.unacknowledged().map { it.eventId }) // the extra one is acked + assertEquals(listOf("state:e1:completed"), events.log) + } + + @Test + fun `begin runs an entry whose only record is of an older generation`() { + store.save(entry(state = EntryState.QUEUED, generation = 2)) + journal.append(record("00000000-0000-0000-0000-000000000023", generation = 1)) + assertEquals(EntryState.RUNNING, ops.begin("e1")!!.state) + } + + @Test + fun `begin does nothing for a paused, settled, or legacy entry`() { + store.save(entry(id = "p", state = EntryState.PAUSED)) + store.save(entry(id = "x", state = EntryState.ERROR)) + store.save(entry(id = "l", state = EntryState.QUEUED, legacy = true, descriptor = null, body = null)) + assertNull(ops.begin("p")) + assertNull(ops.begin("x")) + assertNull(ops.begin("l")) + assertNull(ops.begin("nope")) + } + + @Test + fun `recordAttempt persists attempts and the request id before the send`() { + store.save(entry(state = EntryState.RUNNING, attempts = 2)) + val e = ops.recordAttempt("e1", 1, "req-3") + assertEquals(3, e.attempts) + assertEquals("req-3", store.load("e1")!!.lastRequestId) + assertEquals(3, store.load("e1")!!.attempts) + } + + @Test + fun `recordAttempt refuses once the module took the entry`() { + store.save(entry(state = EntryState.PAUSED)) + assertThrows(NotOwnedException::class.java) { ops.recordAttempt("e1", 1, "r") } + store.save(entry(state = EntryState.RUNNING, generation = 2)) + assertThrows(NotOwnedException::class.java) { ops.recordAttempt("e1", 1, "r") } + } + + @Test + fun `settle journals, transitions, then emits settled before state`() { + store.save(entry(state = EntryState.RUNNING, attempts = 1).copy(lastRequestId = "req-1")) + assertTrue(ops.settle("e1", 1, Settlement.Completed(ok, "https://example.com/items", "POST"))) + val record = journal.unacknowledged().single() + assertEquals(1, record.deliveries) + assertEquals("req-1", record.requestId) + assertEquals(200, record.response!!.status) + assertEquals(1, record.generation) + val e = store.load("e1")!! + assertEquals(EntryState.COMPLETED, e.state) + assertEquals(record.eventId, e.settledEventId) + assertEquals(e.totalBytes, e.bytesSent) + assertEquals(listOf("settled:e1:completed", "state:e1:completed"), events.log) + } + + @Test + fun `a settle with no listener starts at 0 deliveries, so the first replay is 1`() { + journal.stopListening(listener) + store.save(entry(state = EntryState.RUNNING)) + assertTrue(ops.settle("e1", 1, Settlement.Completed(ok, "u", "POST"))) + assertEquals(0, journal.unacknowledged().single().deliveries) + assertEquals(listOf("state:e1:completed"), events.log) // nothing to emit to + assertEquals(1, controller.unacknowledged(listener).single().deliveries) + } + + @Test + fun `a settle whose journal can not write holds the record, settles, and emits`() { + store.save(entry(state = EntryState.RUNNING)) + assertTrue(withJournalReadOnly { ops.settle("e1", 1, Settlement.Completed(ok, "u", "POST")) }) + val e = store.load("e1")!! + assertEquals(EntryState.COMPLETED, e.state) + assertTrue(journal.isHeld(e.settledEventId!!)) + assertEquals(listOf("settled:e1:completed", "state:e1:completed"), events.log) + // The sweep does not forget a row whose record is held, and the ack does. + controller.sweep() + assertNotNull(store.load("e1")) + controller.ack(listOf(e.settledEventId!!)) + assertNull(store.load("e1")) + } + + @Test + fun `a settle whose store write fails leaves the record for the next run to apply`() { + store.save(entry(state = EntryState.RUNNING)) + val dir = store.entryDir("e1") + dir.setWritable(false) + val stood = try { + ops.settle("e1", 1, Settlement.Completed(ok, "u", "POST")) + } finally { + dir.setWritable(true) + } + assertTrue(stood) + assertEquals(EntryState.RUNNING, store.load("e1")!!.state) + val record = journal.unacknowledged().single() + assertEquals(listOf("settled:e1:completed"), events.log) // no state event: the row did not change + // WorkManager runs it again: begin applies the record, no second send. + assertNull(ops.begin("e1")) + assertEquals(record.eventId, store.load("e1")!!.settledEventId) + } + + @Test + fun `a settle after a cancel whose save failed applies the cancel, not a second outcome`() { + // cancel() journaled its record, then its entry save failed: the entry + // is still running, and its request comes back. + val cancelId = "00000000-0000-0000-0000-000000000041" + store.save(entry(state = EntryState.RUNNING)) + journal.append(record(cancelId, kind = EventJournal.KIND_CANCELLED)) + assertFalse(ops.settle("e1", 1, Settlement.Completed(ok, "u", "POST"))) + val e = store.load("e1")!! + assertEquals(EntryState.CANCELLED, e.state) + assertEquals(cancelId, e.settledEventId) + assertEquals(listOf(cancelId), journal.unacknowledged().map { it.eventId }) + assertEquals(listOf("state:e1:cancelled"), events.log) // no second outcome + } + + @Test + fun `a live settle goes to the listener that drained, not the newest module`() { + store.save(entry(id = "a", state = EntryState.RUNNING)) + ops.settle("a", 1, Settlement.Completed(ok, "u", "POST")) + // A reload: the next module's JS drains and takes over. + val next = Any() + controller.unacknowledged(next) + store.save(entry(id = "b", state = EntryState.RUNNING)) + ops.settle("b", 1, Settlement.Completed(ok, "u", "POST")) + assertEquals(2, events.listeners.size) + assertTrue(events.listeners[0] === listener) + assertTrue(events.listeners[1] === next) + } + + @Test + fun `a settle over the journal cap spares every record a row names`() { + val small = EventJournal(tmp.newFolder("small"), maxEntries = 2) + val smallOps = WorkerOps(store, small, settings, events, scheduler) { now } + val named = "00000000-0000-0000-0000-000000000031" + small.append(record(named, id = "done")) + store.save(entry(id = "done", state = EntryState.ERROR, settledEventId = named)) + File(tmp.root, "small/$named.json").setLastModified(1_000) + small.append(record("00000000-0000-0000-0000-000000000032", id = "orphan")) + File(tmp.root, "small/00000000-0000-0000-0000-000000000032.json").setLastModified(2_000) + store.save(entry(state = EntryState.RUNNING)) + smallOps.settle("e1", 1, Settlement.Completed(ok, "u", "POST")) + val left = small.unacknowledged().map { it.eventId } + assertTrue(left.contains(named)) + assertTrue(left.contains(store.load("e1")!!.settledEventId)) + assertEquals(2, left.size) + } + + @Test + fun `a response that lands after cancel drops its record`() { + controller.enqueue(parsed()) + ops.begin("e1") + controller.cancel("e1") + events.log.clear() + assertFalse(ops.settle("e1", 1, Settlement.Completed(ok, "u", "POST"))) + val left = journal.unacknowledged().single() + assertEquals(EventJournal.KIND_CANCELLED, left.kind) + assertEquals(EntryState.CANCELLED, store.load("e1")!!.state) + assertEquals(emptyList(), events.log) + } + + @Test + fun `a settle from an older generation is dropped`() { + store.save(entry(state = EntryState.QUEUED, generation = 2)) + assertFalse(ops.settle("e1", 1, Settlement.Completed(ok, "u", "POST"))) + assertEquals(emptyList(), journal.unacknowledged()) + } + + @Test + fun `an accepted response that lands during pause still settles`() { + store.save(entry(state = EntryState.PAUSED)) + assertTrue(ops.settle("e1", 1, Settlement.Completed(ok, "u", "POST"))) + assertEquals(EntryState.COMPLETED, store.load("e1")!!.state) + } + + @Test + fun `a failure that lands during pause does not settle`() { + store.save(entry(state = EntryState.PAUSED)) + assertFalse(ops.settle("e1", 1, Settlement.Failed("http", "HTTP 400", ok.copy(code = 400), null, "u", "POST"))) + assertEquals(EntryState.PAUSED, store.load("e1")!!.state) + assertEquals(emptyList(), journal.unacknowledged()) + assertEquals(emptyList(), events.log) + } + + @Test + fun `a failed simple settle keeps the live bytes, and a chunked one keeps its accepted bytes`() { + store.save(entry(state = EntryState.RUNNING)) + ops.settle("e1", 1, Settlement.Failed("http", "HTTP 400", ok.copy(code = 400), null, "u", "POST", bytesSent = 5)) + assertEquals(5, store.load("e1")!!.bytesSent) + assertEquals(5, journal.unacknowledged().single().bytesSent) + store.save(entry(id = "c", state = EntryState.RUNNING).copy(bytesSent = 10)) + ops.settle("c", 1, Settlement.Failed("file", "gone", null, 1, "u", "PUT")) + assertEquals(10, store.load("c")!!.bytesSent) + } + + @Test + fun `park sets awaiting-auth once and wakes at expiry`() { + store.save(entry(state = EntryState.RUNNING, expiresAt = 90_000)) + assertEquals(WorkerOps.ParkResult.PARKED, ops.park("e1", 1, headerGeneration = 0)) + val e = store.load("e1")!! + assertEquals(EntryState.AWAITING_AUTH, e.state) + assertEquals(0, e.parkedGeneration) + assertEquals(listOf("state:e1:awaiting-auth"), events.log) + assertEquals(listOf("e1" to 90_000L), scheduler.wakes) + assertEquals(emptyList(), journal.unacknowledged()) + } + + @Test + fun `a 401 sent under older headers re-issues instead of parking`() { + store.save(entry(state = EntryState.RUNNING)) + controller.updateHeaders(mapOf("Authorization" to "Bearer new")) + assertEquals(WorkerOps.ParkResult.REISSUE, ops.park("e1", 1, headerGeneration = 0)) + assertEquals(EntryState.RUNNING, store.load("e1")!!.state) + } + + @Test + fun `updateHeaders after a park requeues it`() { + store.save(entry(state = EntryState.RUNNING)) + ops.park("e1", 1, 0) + controller.updateHeaders(mapOf("Authorization" to "Bearer new")) + assertEquals(EntryState.QUEUED, store.load("e1")!!.state) + assertEquals(listOf("e1"), scheduler.scheduled) + } + + @Test + fun `release queues the entry with its wake time and streak`() { + store.save(entry(state = EntryState.RUNNING)) + assertTrue(ops.release("e1", 1, nextAttemptAt = 80_000, streak = 7)) + val e = store.load("e1")!! + assertEquals(EntryState.QUEUED, e.state) + assertEquals(80_000L, e.nextAttemptAt) + assertEquals(7, e.backoffStreak) + assertEquals(listOf("e1" to 80_000L), scheduler.wakes) + assertEquals(80_000.0, events.rows.single().toMap()["nextAttemptAt"]) + } + + @Test + fun `a system stop queues a running entry and leaves a paused one`() { + store.save(entry(state = EntryState.RUNNING)) + ops.stopped("e1", 1) + assertEquals(EntryState.QUEUED, store.load("e1")!!.state) + store.save(entry(state = EntryState.PAUSED)) + ops.stopped("e1", 1) + assertEquals(EntryState.PAUSED, store.load("e1")!!.state) + assertEquals(emptyList(), journal.unacknowledged()) + } + + @Test + fun `markAccepted persists the flag and the accepted bytes`() { + store.save(entry( + state = EntryState.RUNNING, + descriptor = desc(url = null, file = "/f", parts = listOf(part(0, 10), part(10, 25))), + body = StagedBody(StagedBody.CHUNKED, "blob", null, 25), + ).copy(backoffStreak = 3)) + ops.markAccepted("e1", 1, 1) + val e = store.load("e1")!! + assertTrue(e.descriptor!!.parts!![1].accepted) + assertEquals(15, e.bytesSent) + assertEquals(0, e.backoffStreak) + } + + // MARK: - header generation of an attempt + + @Test + fun `an attempt carries the header generation of the headers it sends`() { + store.save(entry(state = EntryState.RUNNING)) + controller.updateHeaders(mapOf("Authorization" to "Bearer new")) + val e = ops.recordAttempt("e1", 1, "r1") + assertEquals(1, e.headerGeneration) + assertEquals("Bearer new", e.descriptor!!.headers["Authorization"]) + } + + @Test + fun `a 401 between the settings bump and the entry patch parks, and the patch requeues it`() { + store.save(entry(state = EntryState.RUNNING)) + // updateHeaders() has bumped the settings but not yet patched the entry. + settings.update { it.copy(headerGeneration = it.headerGeneration + 1) } + val sent = ops.recordAttempt("e1", 1, "r1") + assertEquals(0, sent.headerGeneration) + assertFalse(ops.hasNewerHeaders("e1", 1, sent.headerGeneration)) // no re-issue with the same old headers + assertEquals(WorkerOps.ParkResult.PARKED, ops.park("e1", 1, sent.headerGeneration)) + // The rest of updateHeaders() finds it parked. + controller.updateHeaders(mapOf("Authorization" to "Bearer new")) + val e = store.load("e1")!! + assertEquals(EntryState.QUEUED, e.state) + assertEquals("Bearer new", e.descriptor!!.headers["Authorization"]) + } + + @Test + fun `a re-issue sends the patched headers, and a 401 on them parks with them`() { + store.save(entry(state = EntryState.RUNNING)) + val first = ops.recordAttempt("e1", 1, "r1") + controller.updateHeaders(mapOf("Authorization" to "Bearer new")) + assertTrue(ops.hasNewerHeaders("e1", 1, first.headerGeneration)) + val second = ops.recordAttempt("e1", 1, "r2") + assertEquals("Bearer new", second.descriptor!!.headers["Authorization"]) + assertFalse(ops.hasNewerHeaders("e1", 1, second.headerGeneration)) + assertEquals(WorkerOps.ParkResult.PARKED, ops.park("e1", 1, second.headerGeneration)) + assertEquals(1, store.load("e1")!!.parkedGeneration) + } + + // MARK: - short backoff + + @Test + fun `a short backoff shows nextAttemptAt on the running row until the next attempt`() { + store.save(entry(state = EntryState.RUNNING)) + ops.backingOff("e1", 1, nextAttemptAt = 24_000) + val waiting = store.load("e1")!! + assertEquals(EntryState.RUNNING, waiting.state) + assertEquals(24_000L, waiting.nextAttemptAt) + assertEquals(24_000.0, events.rows.last().toMap()["nextAttemptAt"]) + ops.recordAttempt("e1", 1, "r2") + assertNull(store.load("e1")!!.nextAttemptAt) + assertEquals(listOf("state:e1:running", "state:e1:running"), events.log) + assertFalse(events.rows.last().toMap().containsKey("nextAttemptAt")) + } + + @Test + fun `an attempt with no pending backoff emits no state event`() { + store.save(entry(state = EntryState.RUNNING)) + ops.recordAttempt("e1", 1, "r1") + assertEquals(emptyList(), events.log) + } + + @Test + fun `a short backoff on an entry the run no longer owns changes nothing`() { + store.save(entry(state = EntryState.PAUSED)) + ops.backingOff("e1", 1, nextAttemptAt = 24_000) + assertNull(store.load("e1")!!.nextAttemptAt) + assertEquals(emptyList(), events.log) + } + + // MARK: - deliveries + + @Test + fun `a settle racing the first drain reaches JS exactly once with deliveries 1`() { + // The drain sets the listener and scans under one journal lock, and the + // append decides 0 or 1 under it. Either the drain returns the record, + // or the settle emits it live; never both, never neither. + repeat(200) { i -> + val id = "race-$i" + val owner = Any() + // The previous iteration's drain left its owner as the listener. Clear + // it, so the race starts with no listener every time. + journal.listener()?.let { journal.stopListening(it) } + assertFalse(journal.isListening()) + store.save(entry(id = id, state = EntryState.RUNNING)) + events.records.clear() + val start = CyclicBarrier(2) + var drained: List = emptyList() + val drain = Thread { start.await(); drained = controller.unacknowledged(owner).filter { it.id == id } } + drain.start() + start.await() + ops.settle(id, 1, Settlement.Completed(ok, "u", "POST")) + drain.join() + val live = events.records.filter { it.id == id } + val seen = drained + live + assertEquals("iteration $i", 1, seen.size) + assertEquals("iteration $i", 1, seen.single().deliveries) + journal.ack(listOf(seen.single().eventId)) + } + } +}