diff --git a/ios/.gitignore b/ios/.gitignore new file mode 100644 index 00000000..2d9f16e2 --- /dev/null +++ b/ios/.gitignore @@ -0,0 +1,2 @@ +.build/ +.swiftpm/ diff --git a/ios/BodyStaging.swift b/ios/BodyStaging.swift new file mode 100644 index 00000000..73656426 --- /dev/null +++ b/ios/BodyStaging.swift @@ -0,0 +1,209 @@ +import Foundation + +/// The staged body of one entry: a file inside the entry directory. A +/// background URLSession uploads from a file only, so every kind gets one, +/// a bodiless request included (0 bytes). +struct StagedBody: Equatable { + let kind: QueueEntry.BodyKind + let relativePath: String + let contentType: String? + let forceContentType: Bool + let totalBytes: Int64 + /// true when the file already existed (an adopted blob). A failed enqueue + /// must not delete it. + let adopted: Bool +} + +enum StagingError: Error, Equatable { + /// A source file is gone. Rejects E_FILE_MISSING. + case fileMissing(String) + /// The parts do not tile the file. Rejects E_INVALID; JS validates the + /// plan, so this is a size mismatch between the plan and the real file. + case invalid(String) + case io(String) +} + +enum BodyStaging { + static let bodyPrefix = "body-" + static let blobPrefix = "blob-" + + /// Stages `body` into `dir` under a fresh name, tmp + fsync + rename. Every + /// check that can reject runs before the caller's file is touched. + /// + /// `fallbackBlob` is an existing blob in `dir` that a chunked body may keep + /// when its source file is gone: the bytes a crash left between the move + /// and the entry save, a v9 blob, or the current entry's blob on a replace. + static func stage(_ body: ParsedEnqueue.Body, parts: [QueueEntry.Part], into dir: URL, + fallbackBlob: String?, fm: FileManager = .default) throws -> StagedBody { + do { + try fm.createDirectory(at: dir, withIntermediateDirectories: true) + } catch { + throw StagingError.io("cannot create the entry directory: \(error.localizedDescription)") + } + switch body { + case .none: + let name = uniqueName(bodyPrefix) + try write(Data(), dir.appendingPathComponent(name)) + return StagedBody(kind: .none, relativePath: name, contentType: nil, + forceContentType: false, totalBytes: 0, adopted: false) + + case .data(let json): + let name = uniqueName(bodyPrefix) + let data = Data(json.utf8) + try write(data, dir.appendingPathComponent(name)) + return StagedBody(kind: .data, relativePath: name, contentType: "application/json", + forceContentType: false, totalBytes: Int64(data.count), adopted: false) + + case .form(let fields): + for f in fields { + if let path = f.path, !fm.fileExists(atPath: fileURL(path).path) { + throw StagingError.fileMissing(path) + } + } + let name = uniqueName(bodyPrefix) + let boundary = "rnbgu-" + UUID().uuidString + let dest = dir.appendingPathComponent(name) + let tmp = FileIO.tmpURL(for: dest) + do { + let size = try writeMultipart(fields, boundary: boundary, to: tmp, fm: fm) + try FileIO.rename(tmp, onto: dest) + return StagedBody(kind: .form, relativePath: name, + contentType: "multipart/form-data; boundary=\(boundary)", + forceContentType: true, totalBytes: size, adopted: false) + } catch let e as StagingError { + try? fm.removeItem(at: tmp) + throw e + } catch { + try? fm.removeItem(at: tmp) + throw StagingError.io("cannot write the form body: \(error.localizedDescription)") + } + + case .file(let path): + let src = fileURL(path) + guard fm.fileExists(atPath: src.path) else { throw StagingError.fileMissing(path) } + let name = uniqueName(bodyPrefix) + let dest = dir.appendingPathComponent(name) + let tmp = FileIO.tmpURL(for: dest) + do { + try? fm.removeItem(at: tmp) + try fm.copyItem(at: src, to: tmp) + try FileHandle(forUpdating: tmp).synchronizeAndClose() + try FileIO.rename(tmp, onto: dest) + } catch { + try? fm.removeItem(at: tmp) + // The source vanished between the check and the copy. + if !fm.fileExists(atPath: src.path) { throw StagingError.fileMissing(path) } + throw StagingError.io("cannot copy the file body: \(error.localizedDescription)") + } + return StagedBody(kind: .file, relativePath: name, contentType: nil, + forceContentType: false, totalBytes: FileIO.size(dest) ?? 0, adopted: false) + + case .parts(let path): + let src = fileURL(path) + if fm.fileExists(atPath: src.path) { + let size = FileIO.size(src) ?? 0 + try requireTiling(parts, size: size) + let name = uniqueName(blobPrefix) + do { + // An O(1) rename on the same volume. Across volumes FileManager copies. + try fm.moveItem(at: src, to: dir.appendingPathComponent(name)) + } catch { + throw StagingError.io("cannot move the chunked file: \(error.localizedDescription)") + } + return StagedBody(kind: .parts, relativePath: name, contentType: nil, + forceContentType: false, totalBytes: size, adopted: false) + } + if let fallbackBlob, let size = FileIO.size(dir.appendingPathComponent(fallbackBlob)) { + try requireTiling(parts, size: size) + return StagedBody(kind: .parts, relativePath: fallbackBlob, contentType: nil, + forceContentType: false, totalBytes: size, adopted: true) + } + throw StagingError.fileMissing(path) + } + } + + /// Writes a multipart/form-data body. String fields are written as they + /// are; file fields are streamed in 1 MB reads. Returns the byte count. + static func writeMultipart(_ fields: [ParsedEnqueue.FormField], boundary: String, to url: URL, + fm: FileManager = .default) throws -> Int64 { + try? fm.removeItem(at: url) + guard fm.createFile(atPath: url.path, contents: nil) else { + throw StagingError.io("cannot create the form body") + } + let out = try FileHandle(forWritingTo: url) + defer { try? out.close() } + var total: Int64 = 0 + func put(_ data: Data) throws { + try out.write(contentsOf: data) + total += Int64(data.count) + } + for f in fields { + var head = "--\(boundary)\r\nContent-Disposition: form-data; name=\"\(quote(f.name))\"" + if let path = f.path { + let fileName = f.fileName ?? fileURL(path).lastPathComponent + head += "; filename=\"\(quote(fileName))\"" + } + head += "\r\nContent-Type: \(f.contentType)\r\n\r\n" + try put(Data(head.utf8)) + if let s = f.string { + try put(Data(s.utf8)) + } else if let path = f.path { + let src = fileURL(path) + guard let reader = try? FileHandle(forReadingFrom: src) else { + throw StagingError.fileMissing(path) + } + defer { try? reader.close() } + while let chunk = try reader.read(upToCount: 1 << 20), !chunk.isEmpty { + try put(chunk) + } + } + try put(Data("\r\n".utf8)) + } + try put(Data("--\(boundary)--\r\n".utf8)) + try out.synchronize() + return total + } + + /// Accepts a `file://` URL and a plain path. The JS layer forwards the + /// path as the caller wrote it. + static func fileURL(_ pathOrURL: String) -> URL { + if pathOrURL.hasPrefix("file://") { + if let u = URL(string: pathOrURL), u.isFileURL { return u } + return URL(fileURLWithPath: String(pathOrURL.dropFirst("file://".count))) + } + return URL(fileURLWithPath: pathOrURL) + } + + static func uniqueName(_ prefix: String) -> String { + prefix + UUID().uuidString.lowercased() + } + + static func requireTiling(_ parts: [QueueEntry.Part], size: Int64) throws { + guard QueueEntry.tilesExactly(parts, size: size) else { + throw StagingError.invalid("parts must tile exactly [0, \(size)), the size of the file") + } + } + + // Quotes and line breaks in a field name or file name would end the header. + // Percent-encode them, as browsers do. + private static func quote(_ s: String) -> String { + s.replacingOccurrences(of: "\"", with: "%22") + .replacingOccurrences(of: "\r", with: "%0D") + .replacingOccurrences(of: "\n", with: "%0A") + } + + private static func write(_ data: Data, _ url: URL) throws { + do { + try FileIO.writeAtomically(data, to: url) + } catch { + throw StagingError.io("cannot write the body: \(error.localizedDescription)") + } + } +} + +private extension FileHandle { + func synchronizeAndClose() throws { + defer { try? close() } + try synchronize() + } +} diff --git a/ios/ChunkedCoordinator.swift b/ios/ChunkedCoordinator.swift index 08173fa4..af126bfb 100644 --- a/ios/ChunkedCoordinator.swift +++ b/ios/ChunkedCoordinator.swift @@ -1,729 +1,397 @@ import Foundation -/// Runs chunked uploads against the background sessions. It keeps the sliding -/// window of part tasks enqueued with the daemon. It evaluates the outcome of -/// each part. After a relaunch, it reconciles the durable [ChunkedManifest] -/// with the tasks that the daemon still holds. +/// Runs chunked entries against the background sessions: keeps the sliding +/// window of part tasks enqueued with the daemon, evaluates each part's +/// outcome, and after a relaunch rebuilds the window from the tasks the +/// daemon still holds. /// -/// Every state transition occurs on one serial queue. The queue enforces the -/// invariants that the design marks binding: at most [ChunkedEngine.window] -/// part tasks are enqueued per upload, and never two for the same part index. -/// `inFlight` maps each enqueued part to the task key that owns it. Only -/// refill, on this queue, creates tasks. +/// Every call runs on the QueueCoordinator's serial queue, which enforces the +/// binding invariants: at most ChunkedEngine.window part tasks per entry, and +/// never two for one part index. `inFlight` maps each enqueued part to the +/// key of the one task that owns it. Only enqueuePart creates part tasks. +/// +/// The entry (QueueEntry) is the durable truth: parts, accepted flags, +/// incarnation. Terminals go through QueueCoordinator.settle. final class ChunkedCoordinator { + struct LiveTask { + let task: UploadTask + let part: Int + let incarnation: String? + } - // The singleton that owns the background sessions. It outlives this object. - // Both live for the whole process. - private unowned let uploader: RNBackgroundUpload - - private let queue = DispatchQueue(label: "ai.openspace.rnbgupload.chunked") - - // partIndex -> the TaskMap key of the one task that may be in flight for it. - // The key lets us tell a superseded task's late completion (possible around - // a relaunch reconcile) apart from the live task's completion. + private unowned let q: QueueCoordinator + // id -> part index -> the task key that owns it. private var inFlight: [String: [Int: String]] = [:] - // The uploads whose in-flight set we rebuild from the daemon now. Refill is - // blocked until the rebuild lands. Thus a stale snapshot can never - // double-enqueue. The token makes overlapping reconciles safe: only the - // latest reconcile may apply its snapshot. An earlier snapshot could miss - // tasks enqueued after it was taken. To apply it would re-enqueue their - // part indexes. - private var reconcileToken: [String: UUID] = [:] - private var cooldownUntil: [String: [Int: Double]] = [:] // epoch ms - private var transientAttempts: [String: [Int: Int]] = [:] - private var expiryArmed: Set = [] - // The in-flight bytes per part index. They feed the byte-weighted - // aggregate progress. + // id -> part index -> bytes sent by its live task. Feeds the byte-weighted + // progress. private var partSent: [String: [Int: Int64]] = [:] - // A cache of the stored manifests, refreshed on every load. The progress - // path reads it. Thus didSendBodyData never touches the disk. - private var manifests: [String: ChunkedManifest] = [:] + // id -> part index -> when its delayed task begins (epoch ms), until it + // begins. When every part in the window waits, the row shows the earliest + // as nextAttemptAt. + private var partBeginAt: [String: [Int: Double]] = [:] - private static let progressThrottle: TimeInterval = 0.5 // seconds, per upload - private let progressLock = NSLock() - private var lastProgressAt: [String: TimeInterval] = [:] - - init(uploader: RNBackgroundUpload) { - self.uploader = uploader + init(_ coordinator: QueueCoordinator) { + q = coordinator } - private func nowMs() -> Double { Date().timeIntervalSince1970 * 1000 } - - // MARK: - Entry points (module methods) + // MARK: - Called by QueueCoordinator - /// Starts, or resumes, a chunked upload. The durable manifest makes the call - /// idempotent. A first call takes ownership of the source file (an O(1) - /// rename into the library's directory) and saves the manifest BEFORE any - /// task is enqueued. A new call with the same id reconciles instead. The - /// same parts resume: the stored headers are replaced, and accepted parts - /// are skipped. Different parts recreate the upload, per the design's rule - /// (see ChunkedManifest.reconciled). Crash recovery, resume after a stop, - /// and resume with fresh auth are all this same call. - /// - /// Every rejection-type validation runs BEFORE the source is consumed. The - /// parse throws first, and a reconcile never touches the source (`path` is - /// ignored once a manifest exists). One rejection is possible after the - /// move: the manifest save can fail. That leaves the blob adoptable. A - /// retry with the same id finds the blob at the blob path and proceeds (see - /// takeOwnership). - func startUpload(_ options: [String: Any], - resolve: @escaping (String) -> Void, - reject: @escaping (String) -> Void) { - queue.async { - do { - let incoming = try ChunkedManifest.parse(options, createdAt: self.nowMs()) - let id = incoming.id - let manifest: ChunkedManifest - if let existing = ChunkedStore.load(id) { - // "Running" per the design's recreate rule: not stalled (no - // journaled terminal error or cancel that awaits this resume) and - // not past its deadline. Everything else rejects a different parts - // array. That includes part tasks live with the daemon, and - // finished-but-unacked. - let running = !existing.stalled && !existing.isExpired(self.nowMs()) - manifest = try existing.reconciled( - with: incoming, running: running, blobSize: ChunkedStore.blobSize(id)) - if manifest.incarnation != existing.incarnation { - // This is a recreate. The in-flight byte counts belong to the - // replaced parts. reconcileLocked below cancels the old - // incarnation's tasks, and does not adopt them. enqueuePart - // sweeps its temp files. - self.partSent[id] = nil - } - } else { - guard let path = options["path"] as? String else { - throw ChunkedManifest.ParseError(message: "Missing 'path'") - } - try self.takeOwnership(path: path, id: id) - // The same rule as recreate, and as Android's validatedForCreate: - // the parts must tile [0, blob size) exactly. A partial or - // overlapping cover would silently upload wrong bytes. This throws - // BEFORE the manifest is saved and before anything is enqueued. - // Thus the moved blob stays adoptable by a corrected retry with the - // same id (see takeOwnership). - let blobSize = ChunkedStore.blobSize(id) - guard ChunkedManifest.tilesExactly(incoming.parts, size: blobSize) else { - throw ChunkedManifest.ParseError( - message: "chunked upload '\(id)' parts must tile exactly [0, \(blobSize))") - } - manifest = incoming - } - try ChunkedStore.save(manifest) - self.manifests[id] = manifest - // A fresh call gets a fresh retry budget. The persisted per-part - // rejection counts reset in the parts rebuild above (reconciled or - // parse). - self.transientAttempts[id] = nil - self.cooldownUntil[id] = nil - self.reconcileLocked(id, resumedByStart: true) - resolve(id) - } catch { - reject(error.localizedDescription) - } + /// queued -> running, then fill the window. + func start(_ id: String) { + guard var e = q.index.entry(id), e.isChunked, e.state == .queued || e.state == .running, + !q.settings.paused else { return } + if e.state == .queued { + e.state = .running + e.nextAttemptAt = nil + q.commit(e) } + refill(id) } - /// Rebuilds every stored upload's in-flight set from the daemon, and then - /// refills. Called when the sessions are created or recreated: an app - /// relaunch, a JS reload, or the background-wake path through - /// `RNBackgroundUpload.shared`. - func reconcileAll() { - // Claim the system's background completion handlers BEFORE the reconcile - // is queued. After a relaunch, the replayed didCompleteWithError callbacks - // run unowned and never refill. Thus this chain is the only refill. - // Nothing else stops urlSessionDidFinishEvents from handing the system - // its handler, and the app its suspension, before one new part task is - // enqueued. The risk is largest exactly when every enqueued part finished - // while the app was dead: zero daemon tasks left, no future wake, and a - // silent stall. The claim provably precedes any drain: a relaunch reaches - // this point inside the init of `shared`, and the AppDelegate hook - // finishes that init before it stores the handler. - RNBackgroundUpload.deferBackgroundCompletionHandlers() - queue.async { - let group = DispatchGroup() - for manifest in ChunkedStore.all() { - self.manifests[manifest.id] = manifest - group.enter() - self.reconcileLocked(manifest.id, resumedByStart: false) { group.leave() } - } - group.notify(queue: self.queue) { - // Every upload's post-reconcile refill has resumed its tasks. The - // handlers can drain now. - RNBackgroundUpload.releaseBackgroundCompletionHandlers() - } - } + /// Forgets the window. The caller cancels the tasks. + func stop(_ id: String) { + inFlight[id] = nil + partSent[id] = nil + partBeginAt[id] = nil } - /// Cancels a chunked upload. It journals one 'cancelled' (user) terminal - /// and stalls the upload. The manifest and the bytes are kept. Thus the - /// next startUpload resumes. Completion receives nil when the id has no - /// manifest (not a chunked upload). It receives false when nothing runs - /// (the upload is already terminal). - func cancel(_ id: String, completion: @escaping (Bool?) -> Void) { - queue.async { - guard let manifest = self.latest(id) else { completion(nil); return } - if manifest.stalled || manifest.allAccepted { completion(false); return } - var entry = JournaledEvent( - eventId: UUID().uuidString, id: id, type: "cancelled", timestamp: self.nowMs()) - entry.cancelReason = "user" - self.stall(id, entry: entry) - completion(true) + /// Relaunch: adopt the daemon's live part tasks of the current + /// incarnation, one per part index. Cancel the rest (a replaced plan, an + /// accepted part, a duplicate: concurrent PUTs of one partNum are unsafe on + /// the server). Then refill. + func reconcile(_ id: String, tasks: [LiveTask]) { + guard let e = q.index.entry(id) else { return } + var live: [Int: String] = [:] + for t in tasks { + let keep = t.incarnation == e.incarnation && e.parts.indices.contains(t.part) + && !e.parts[t.part].accepted && live[t.part] == nil + if keep { + live[t.part] = t.task.key + q.liveTasks[t.task.key] = (id, t.task) + if let begin = t.task.beginAt.map({ $0.timeIntervalSince1970 * 1000 }), begin > q.now() { + partBeginAt[id, default: [:]][t.part] = begin + } + } else { + q.taskMap.setPurpose(.superseded, forKey: t.task.key, id: id) + t.task.cancel() + } } - } - - /// An explicit release. It cancels the in-flight part tasks, with no - /// terminal event: the consumer lets go, and awaits no outcome. It deletes - /// the manifest, the moved bytes, and all part temp files. - func remove(_ id: String, completion: @escaping () -> Void) { - queue.async { - if self.latest(id) != nil { self.cancelTasks(for: id) } - ChunkedStore.remove(id) - self.clearState(id) - completion() + // A pending part with a TaskMap key but no live task finished while the + // app was dead, and its completion may replay now. Hold its slot for the + // grace, so no second PUT of that part starts, and its part file stays. + var held: [Int: String] = [:] + for key in q.taskMap.keys(where: { $0.id == id && $0.incarnation == e.incarnation + && ($0.purpose ?? .attempt) == .attempt }) { + guard let i = q.taskMap.meta(forKey: key)?.partIndex, e.parts.indices.contains(i), + !e.parts[i].accepted, live[i] == nil, held[i] == nil else { continue } + held[i] = key } - } - - /// The one moment when the library may delete a chunked upload's bytes: the - /// consumer acknowledged its 'completed' terminal event. - func releaseCompleted(_ ids: [String], completion: @escaping () -> Void) { - queue.async { - for id in ids { - ChunkedStore.remove(id) - self.clearState(id) - } - completion() + inFlight[id] = live.merging(held) { current, _ in current } + if !held.isEmpty { holdForReplay(id, held) } + // Part files of accepted parts with no live task are orphans. + for i in e.parts.indices where e.parts[i].accepted && live[i] == nil { + q.store.removePartFile(id, i) } + start(id) + updateWait(id) } - /// The chunked rows for getAllUploads: one aggregate row per manifest. The - /// part tasks are transport detail. bytesSent counts accepted parts only. - /// That is the durable number. - func snapshots(completion: @escaping ([[String: Any]]) -> Void) { - queue.async { - let rows = ChunkedStore.all().map { manifest -> [String: Any] in - let state: String - if manifest.allAccepted { - state = "completed" - } else if manifest.stalled { - state = "error" - } else if !(self.inFlight[manifest.id] ?? [:]).isEmpty { - state = "running" - } else { - state = "pending" - } - return ["id": manifest.id, - "state": state, - "bytesSent": manifest.acceptedBytes, - "totalBytes": manifest.totalBytes] + /// When the grace ends (every held replay came, or the timer fired), a + /// held slot whose replay never came is released: its key is pruned and + /// the part is sent again. + private func holdForReplay(_ id: String, _ held: [Int: String]) { + q.openGrace("part:" + id, keys: Set(held.values)) { [weak self] in + guard let self else { return } + var released = false + for (i, key) in held where self.inFlight[id]?[i] == key { + self.inFlight[id]?[i] = nil + self.q.taskMap.removeKey(key) + released = true } - completion(rows) + if released { self.refill(id) } } } - // MARK: - Delegate hooks (called by RNBackgroundUpload) + // MARK: - Delegate hooks - func partProgress(id: String, part: Int, incarnation: String?, sent: Int64) { - let now = Date().timeIntervalSince1970 - progressLock.lock() - if let last = lastProgressAt[id], now - last < Self.progressThrottle { - progressLock.unlock() + func partCompleted(id: String, part: Int, incarnation: String?, key: String, meta: TaskMap.Meta?, + completion c: TaskCompletion) { + let owned = inFlight[id]?[part] == key + if owned { + inFlight[id]?[part] = nil + partSent[id]?[part] = nil + partBeginAt[id]?[part] = nil + } + // Every path below may change the window; a settle or park makes this + // a no-op. + defer { updateWait(id) } + guard var e = q.index.entry(id), e.isChunked, !e.legacy else { + if owned { q.store.removePartFile(id, part) } return } - lastProgressAt[id] = now - progressLock.unlock() - queue.async { - // A removed or replaced incarnation's task must not feed the aggregate. - guard let manifest = self.manifests[id], manifest.incarnation == incarnation else { return } - self.partSent[id, default: [:]][part] = sent - self.emitAggregateProgress(id, manifest) + // A late callback from a replaced plan: its response is about ranges and + // urls this entry no longer describes. Write nothing from it. + guard incarnation == e.incarnation, e.parts.indices.contains(part) else { + if owned { refill(id) } + return } - } - - /// One part task finished (a foreground or background-wake delegate - /// callback). We evaluate the accept rules, update the manifest, delete the - /// temp file, and refill the window. It is synchronous on purpose: the - /// journal write for a terminal outcome must land before the delegate - /// callback returns. The simple-upload path obeys the same rule. - func handlePartCompletion(id: String, part: Int, incarnation: String?, taskKey: String, - statusCode: Int?, headers: [String: String], - body: String?, error: NSError?) { - queue.sync { - TaskMap.removeKey(taskKey) - let owned = inFlight[id]?[part] == taskKey - if owned { + let cancelled = RetryClassifier.isCancellation(c.error) + if cancelled, meta?.purpose == .pause || meta?.purpose == .superseded { return } + let accepted = c.error == nil + && c.statusCode.map { UploadOutcome.isAccepted($0, body: c.body, accept: e.accept) } == true + // A system cancel is not an attempt: no event. It retries below. + if !cancelled { + q.emitAttempt(e, requestId: meta?.requestId, attempt: meta?.attempt ?? e.attempts, completion: c, + partIndex: part, accepted: accepted) + } + e.lastRequestId = meta?.requestId ?? e.lastRequestId + e.lastUrl = e.parts[part].url + e.lastPartIndex = part + + // Accept first, whatever the entry's state: the server holds these bytes + // now. Losing the flag would re-send a part the server already has. + if accepted { + // A replay that lands after its slot was given to a new task: that + // task is a duplicate PUT, and it reads the part file. Stop it first. + if !owned, let other = inFlight[id]?[part] { + q.cancelTask(other, purpose: .superseded) inFlight[id]?[part] = nil partSent[id]?[part] = nil + partBeginAt[id]?[part] = nil } - guard var manifest = latest(id) else { - // The upload was removed (removeUpload, or a completed ack) while - // this task was in flight. There is nothing left to report. - if owned { ChunkedStore.removePartFile(id, part) } - return - } - // A late callback from a removed-then-recreated or replaced - // incarnation. Its response is about byte ranges and URLs that this - // manifest no longer describes. Thus nothing about it, the accept flag - // included, may be written into the current manifest. Its temp file has - // the old token in its name. The sweep removes it when the current plan - // next materializes this index. - guard incarnation == manifest.incarnation else { - if owned { refill(id) } - return - } - // A part index that the manifest does not know (corrupt task metadata) - // must not crash the delegate. Drop the task's outcome and let refill - // plan again. - guard manifest.parts.indices.contains(part) else { - if owned { refill(id) } - return - } - - // Accept evaluation comes first. The server holds these bytes now, - // regardless of a concurrent stall or a superseded task in the same - // incarnation. If we lose the flag, we re-send a part that the server - // already has. - if error == nil, let statusCode, - UploadOutcome.isAccepted(statusCode, body: body, accept: manifest.accept) { - manifest = updateManifest(id) { $0.withPartAccepted(part) } - ?? manifest.withPartAccepted(part) - transientAttempts[id]?[part] = nil - cooldownUntil[id]?[part] = nil - if owned { ChunkedStore.removePartFile(id, part) } - // A stalled upload keeps the flag but reports nothing more. The - // journaled terminal stands until the next startUpload resume. That - // resume finds all parts accepted and completes without a re-send. - guard !manifest.stalled else { return } - if manifest.allAccepted { - finalizeCompleted(id, manifest, reemit: false) - } else { - emitAggregateProgress(id, manifest) - if owned { refill(id) } - } - return - } - - // A superseded task's failure carries no policy weight. The live task - // for this part drives the retries. But an UNOWNED task with no live - // replacement is a relaunch replay that runs before reconcile rebuilds - // ownership. If we drop its deterministic HTTP rejection, the part gets - // a fresh retry budget on every system wake. So count it, and let it - // trip the budget. The in-flight reconcile does the re-enqueueing. - guard owned else { - if inFlight[id]?[part] == nil, !manifest.stalled, !manifest.parts[part].accepted, - error == nil, let code = statusCode, !ChunkedEngine.isTransientHttp(code) { - recordRejection(id, part: part, manifest: manifest, code: code, - headers: headers, body: body, scheduleRetryInBudget: false) - } - return - } - ChunkedStore.removePartFile(id, part) // the retry builds the file again - // This is a duplicate of a part that a superseded task already - // delivered. The part is settled, whatever this task's outcome was. Its - // failure must not burn retries. - if manifest.parts[part].accepted { - if !manifest.stalled { refill(id) } - return - } - // A terminal is already journaled (a cancel, or a sibling part's - // stall). Swallow the fallout. - guard !manifest.stalled else { return } - - if let error, error.domain == NSURLErrorDomain, error.code == NSURLErrorCancelled { - // A user cancel journals and stalls in cancel() before the tasks are - // torn down. Thus a cancel here, with no stall, comes from the - // system. Retry it like a transient failure. - scheduleTransientRetry(id, part: part) - return - } - - if manifest.isExpired(nowMs()) { - stall(id, entry: expiredEntry(id)) - return - } - - if let error { - if RNBackgroundUpload.errorKind(for: error) == "file", - !FileManager.default.fileExists(atPath: ChunkedStore.blobURL(id).path) { - stall(id, entry: errorEntry( - id: id, error: "chunked source blob missing", errorKind: "file", partIndex: part)) - } else { - // This includes a lost temp part file. The retry rebuilds it from - // the blob. - scheduleTransientRetry(id, part: part) - } - return + e = e.withPartAccepted(part) + q.commit(e, emit: false) + q.store.removePartFile(id, part) + guard e.state == .running else { return } + if e.allAccepted { + q.settle(id, .completed(RawResponseRecord(bodyTruncated: false))) + } else { + emitProgress(e) + refill(id) } + return + } - let code = statusCode ?? 0 - if ChunkedEngine.isTransientHttp(code) { - scheduleTransientRetry(id, part: part) - return + // A failure of a task this process does not own is a relaunch replay or + // a superseded duplicate. The live task, or the reconcile refill, drives + // the part. The part file stays for reuse. + guard owned, e.state == .running else { return } + q.index.upsert(e) + if e.parts[part].accepted { + refill(id) + return + } + if cancelled { + retryPart(e, part) + return + } + let blobExists = e.bodyPath.map { FileIO.exists(q.store.fileURL(id, $0)) } ?? false + let verdict = RetryClassifier.classify(RetryClassifier.Input( + statusCode: c.statusCode, body: c.body, error: c.error, accept: e.accept, policy: q.policy(e), + fileExists: blobExists, now: q.now(), expiresAt: e.expiresAt)) + switch verdict { + case .accepted: + break // handled above + case .transient: + retryPart(e, part) + case .auth: + if let g = meta?.headerGeneration, g < q.settings.headerGeneration { + _ = enqueuePart(id, part, delayMs: nil) + } else { + // Park the whole entry: the other parts would get the same answer. + q.park(e) } - recordRejection(id, part: part, manifest: manifest, code: code, - headers: headers, body: body, scheduleRetryInBudget: true) + case .terminalHttp: + q.settle(id, .error(OutcomeErrorRecord( + errorKind: "http", message: "HTTP \(c.statusCode ?? 0) on part \(part)", + response: q.response(c), partIndex: part))) + case .fileMissing: + q.settle(id, .fileError("the chunked source blob is missing", partIndex: part)) + case .expired: + q.settle(id, .expired) } } - /// The identity of a chunked part task, or nil for a simple upload's task. - /// taskDescription is primary. The persisted TaskMap entry, written before - /// the task first resumed, is the durable fallback. `incarnation` is the - /// manifest token that the task was created under. It is nil only for - /// corrupt metadata, and the consumers treat nil as a mismatch. - static func partRef(_ session: URLSession, _ task: URLSessionTask) - -> (id: String, part: Int, incarnation: String?)? { - if let ref = ChunkedEngine.parseTaskDescription(task.taskDescription) { return ref } - if let meta = TaskMap.meta(forKey: TaskMap.key(session, task)), let part = meta.partIndex { - return (meta.id, part, meta.incarnation) + /// A delayed part retry is about to start. Rebuild its request from the + /// entry's current headers, or cancel it when the entry moved on. + func partWillBegin(id: String, part: Int, incarnation: String?, key: String, + meta: TaskMap.Meta?) -> URLRequest? { + // Before the first reconcile nothing is owned yet; accept the task if the + // entry wants it. Reconcile then adopts or cancels it. + let ownedOrUnknown = !q.ready || inFlight[id]?[part] == key + guard let e = q.index.entry(id), e.isChunked, incarnation == e.incarnation, + e.parts.indices.contains(part), !e.parts[part].accepted, e.state == .running, + !q.settings.paused, ownedOrUnknown, let url = URL(string: e.parts[part].url) else { + q.taskMap.setPurpose(.superseded, forKey: key, id: id) + q.liveTasks[key] = nil + if inFlight[id]?[part] == key { + inFlight[id]?[part] = nil + partBeginAt[id]?[part] = nil + updateWait(id) + } + return nil } - return nil + partBeginAt[id]?[part] = nil + updateWait(id) + q.taskMap.setHeaderGeneration(q.settings.headerGeneration, forKey: key) + return q.buildRequest(e, url: url, requestId: meta?.requestId ?? UUID().uuidString, + partHeaders: e.parts[part].headers) } - // MARK: - Window (all on `queue`) - - /// Rebuilds inFlight for one upload from the daemon's live tasks, and then - /// refills. A task in the .completed or .canceling state is NOT live: its - /// delegate callback, replayed after a relaunch, settles it. A part with no - /// live task simply enqueues again. Accept evaluation absorbs a - /// completed-but-unreported duplicate. We never guess. - /// `completion` fires, on `queue`, when this reconcile has settled: the - /// refill ran, or a newer reconcile superseded this one. reconcileAll gates - /// the background completion handlers on it. - private func reconcileLocked(_ id: String, resumedByStart: Bool, - completion: (() -> Void)? = nil) { - let token = UUID() - reconcileToken[id] = token - enumerateAllTasks { tasks in - self.queue.async { - defer { completion?() } - guard self.reconcileToken[id] == token else { return } // superseded - let manifest = self.latest(id) - var live: [Int: String] = [:] - for (session, task) in tasks { - guard let ref = Self.partRef(session, task), ref.id == id, - task.state == .running || task.state == .suspended else { continue } - if ref.incarnation != manifest?.incarnation || live[ref.part] != nil { - // Never adopt a task from a replaced incarnation. Its bytes and - // URL belong to the old plan, and the token check in - // handlePartCompletion drops its late completion. Never adopt a - // second live task for one part index: concurrent PUTs of one - // partNum are verified unsafe on the server side. - task.cancel() - } else { - live[ref.part] = TaskMap.key(session, task) - } - } - self.inFlight[id] = live - self.reconcileToken[id] = nil - if let manifest { - // Temp files for accepted parts with no live task are orphans. - for index in manifest.parts.indices - where manifest.parts[index].accepted && live[index] == nil { - ChunkedStore.removePartFile(id, index) - } - } - self.refill(id, resumedByStart: resumedByStart) - } - } + func partProgress(id: String, part: Int, incarnation: String?, sent: Int64) { + guard let e = q.index.entry(id), e.incarnation == incarnation, e.state == .running else { return } + partSent[id, default: [:]][part] = sent + // A delayed part that began while the app was dead reports progress + // before any willBegin. + if partBeginAt[id]?.removeValue(forKey: part) != nil { updateWait(id) } + emitProgress(q.index.entry(id) ?? e) } - /// Fills the window back up to [ChunkedEngine.window] enqueued part tasks. - /// Called after every part completion (the background-wake refill that the - /// design's liveness rationale requires), after a retry cooldown, and at - /// the end of every reconcile. - private func refill(_ id: String, resumedByStart: Bool = false) { - guard reconcileToken[id] == nil, let manifest = latest(id) else { return } - // Stalled wins, even over all-accepted. The journaled terminal stands - // until an explicit startUpload resume. The resume clears the stall, - // lands here again, and completes without a re-send. - guard !manifest.stalled else { return } - if manifest.allAccepted { - finalizeCompleted(id, manifest, reemit: resumedByStart) + // MARK: - Window + + /// Fills the window back up. Called after every part completion (the + /// background-wake refill that keeps the upload moving while the app is + /// dead), at start, and at the end of every reconcile. + func refill(_ id: String) { + guard q.ready, !q.settings.paused, let e = q.index.entry(id), e.isChunked, + e.state == .running else { return } + if e.allAccepted { + q.settle(id, .completed(RawResponseRecord(bodyTruncated: false))) return } - let now = nowMs() - if manifest.isExpired(now) { - stall(id, entry: expiredEntry(id)) + if q.now() >= e.expiresAt { + q.settle(id, .expired) return } - armExpiryCheck(id, expiresAt: manifest.expiresAt) - // A blob shorter than a part's range can never finish. Report a terminal - // 'file' now, not a surprise when the window reaches the short part - // later. A retry cannot help, because the bytes are not there. Thus this - // stalls, and awaits removeUpload or a recreate whose tiling rule fits - // the real size. - let blobSize = ChunkedStore.blobSize(id) - if let short = manifest.parts.indices.first(where: { manifest.parts[$0].end > blobSize }) { - stall(id, entry: errorEntry( - id: id, - error: "source blob is \(blobSize) bytes; part \(short) needs " - + "[\(manifest.parts[short].start), \(manifest.parts[short].end))", - errorKind: "file", partIndex: short)) + // A blob shorter than a part can never finish: report it now. + let blobSize = e.bodyPath.flatMap { FileIO.size(q.store.fileURL(id, $0)) } ?? 0 + if let short = e.parts.indices.first(where: { e.parts[$0].end > blobSize }) { + q.settle(id, .fileError( + "source blob is \(blobSize) bytes; part \(short) needs " + + "[\(e.parts[short].start), \(e.parts[short].end))", partIndex: short)) return } let flight = Set((inFlight[id] ?? [:]).keys) - let cooling = Set((cooldownUntil[id] ?? [:]).filter { $0.value > now }.keys) - for index in ChunkedEngine.indexesToEnqueue( - pending: manifest.pendingIndexes(), inFlight: flight, cooling: cooling) { - if !enqueuePart(id, index, manifest) { return } // stalled inside + for index in ChunkedEngine.indexesToEnqueue(pending: e.pendingIndexes(), inFlight: flight) { + if !enqueuePart(id, index, delayMs: nil) { return } // settled or deferred inside } } - private func enqueuePart(_ id: String, _ index: Int, _ manifest: ChunkedManifest) -> Bool { - let part = manifest.parts[index] + // MARK: - Private + + /// One part task. Write-ahead: the attempt count is saved first; the + /// TaskMap entry is written before resume. Returns false when the entry + /// settled instead, or when the save failed: then no task exists and a + /// refill runs after a backoff. + private func enqueuePart(_ id: String, _ index: Int, delayMs: Int?) -> Bool { + guard var e = q.index.entry(id), e.parts.indices.contains(index) else { return false } + let part = e.parts[index] guard let url = URL(string: part.url) else { - stall(id, entry: errorEntry( - id: id, error: "part \(index) url is not a valid URL", errorKind: "unknown", - partIndex: index)) + q.settle(id, .error(OutcomeErrorRecord( + errorKind: "unknown", message: "part \(index) url is not valid", partIndex: index))) return false } - // A background session can upload only from a file. Thus each enqueued - // part gets a temp file that holds exactly its byte range. The transient - // disk usage stays at window × partSize, not a second full copy of the - // source. - let partFile: URL + let blob = e.bodyPath ?? ChunkedManifestV9.blobName + let file: URL do { - partFile = try ChunkedStore.writePartFile( - id: id, index: index, start: part.start, end: part.end, - incarnation: manifest.incarnation) + file = try q.store.writePartFile( + id: id, blob: blob, index: index, start: part.start, end: part.end, incarnation: e.incarnation) } catch { - stall(id, entry: errorEntry( - id: id, error: "cannot materialize part \(index): \(error.localizedDescription)", - errorKind: "file", partIndex: index)) + // Only a missing or short blob can never succeed. Any other failure + // (a full disk, protected data) may pass: build the part again later. + let blobSize = FileIO.size(q.store.fileURL(id, blob)) ?? 0 + if blobSize < part.end { + q.settle(id, .fileError("cannot build part \(index): \(error.localizedDescription)", partIndex: index)) + } else { + refillLater(e, part: part, delayMs: delayMs) + } return false } - var request = URLRequest(url: url) - request.httpMethod = "PUT" - // Unchanged, per the protocol-as-data rule. The library adds nothing. - for (key, value) in part.headers { - request.setValue(value, forHTTPHeaderField: key) + e.attempts += 1 + guard q.commitAhead(e, emit: false) else { + refillLater(e, part: part, delayMs: delayMs) + return false } - let session = uploader.session(wifiOnly: manifest.wifiOnly) - let task: URLSessionUploadTask + + let requestId = UUID().uuidString + let meta = TaskMap.Meta( + id: id, partIndex: index, incarnation: e.incarnation, attempt: e.attempts, + requestId: requestId, headerGeneration: q.settings.headerGeneration, + generation: e.generation, purpose: .attempt) + let task: UploadTask do { - task = try RNBackgroundUpload.uploadTask(session, request, fromFile: partFile) + task = try q.transport.upload( + q.buildRequest(e, url: url, requestId: requestId, partHeaders: part.headers), + fromFile: file, wifiOnly: q.settings.wifiOnly, + description: ChunkedEngine.taskDescription(id: id, part: index, incarnation: e.incarnation), + beginAt: delayMs.map { Date(timeIntervalSince1970: (q.now() + Double($0)) / 1000) }, + beforeResume: { key in self.q.taskMap.set(meta, forKey: key) }) } catch { - stall(id, entry: errorEntry( - id: id, error: "cannot enqueue part \(index): \(error.localizedDescription)", - errorKind: "file", partIndex: index)) + // The session could not open the part file. As for a failed part file + // build: a short blob can never succeed; otherwise try again later. + let blobSize = FileIO.size(q.store.fileURL(id, blob)) ?? 0 + if blobSize < part.end { + q.settle(id, .fileError("cannot read part \(index): \(error.localizedDescription)", partIndex: index)) + } else { + refillLater(e, part: part, delayMs: delayMs) + } return false } - task.taskDescription = ChunkedEngine.taskDescription( - id: id, part: index, incarnation: manifest.incarnation) - let key = TaskMap.key(session, task) - TaskMap.set(TaskMap.Meta(id: id, accept: nil, partIndex: index, - incarnation: manifest.incarnation), forKey: key) - inFlight[id, default: [:]][index] = key - task.resume() + inFlight[id, default: [:]][index] = task.key + q.liveTasks[task.key] = (id, task) + if let delayMs { + partBeginAt[id, default: [:]][index] = q.now() + Double(delayMs) + } else { + partBeginAt[id]?[index] = nil + } return true } - // MARK: - Terminal transitions (all on `queue`) - - /// Journals the terminal, marks the upload stalled, and cancels its - /// in-flight tasks. The stall is durable: relaunch reconciliation must not - /// resume the upload; only startUpload may. The manifest and the bytes are - /// kept. Every non-completed terminal leaves the consumer its recovery - /// options. - private func stall(_ id: String, entry: JournaledEvent) { - _ = updateManifest(id) { manifest in - var next = manifest - next.stalled = true - return next - } - cancelTasks(for: id) - partSent[id] = nil - cooldownUntil[id] = nil - RNBackgroundUpload.journalAndEmit(entry) + /// No task was made for a part (a failed save or part file). Fill the + /// window again after the wait it asked for, or a backoff, whichever is + /// longer. + private func refillLater(_ e: QueueEntry, part: QueueEntry.Part, delayMs: Int?) { + let backoff = RetryClassifier.backoffMs( + attempt: max(part.rejections, 1), policy: q.policy(e), random: q.random) + q.schedule(max(delayMs ?? 0, backoff)) { [weak self] in self?.refill(e.id) } } - private func finalizeCompleted(_ id: String, _ manifest: ChunkedManifest, reemit: Bool) { - for index in manifest.parts.indices { ChunkedStore.removePartFile(id, index) } - partSent[id] = nil - // A resume of a finished-but-unacked upload must not mint a second - // terminal event. Emit the journaled event again. Thus a live listener - // still hears it, with the eventId that the consumer will ack. - if let existing = EventJournal.unacknowledgedEntries() - .first(where: { $0.id == id && $0.type == "completed" }) { - if reemit { RNBackgroundUpload.emitEvent(existing) } + /// A transient part failure: the next task is created now with a + /// backoff delay, so it holds the part's window slot and the daemon starts + /// it on time even while the app is dead. + private func retryPart(_ e: QueueEntry, _ part: Int) { + var n = e + n.parts[part].rejections += 1 + let delay = RetryClassifier.backoffMs( + attempt: n.parts[part].rejections, policy: q.policy(n), random: q.random) + if q.now() + Double(delay) >= n.expiresAt { + q.settle(n.id, .expired) return } - // There are no response fields, because no single response represents N - // accepted parts. The blob is deleted only when this event is ACKED (see - // ackEvents). - RNBackgroundUpload.journalAndEmit( - JournaledEvent(eventId: UUID().uuidString, id: id, type: "completed", timestamp: nowMs())) - } - - // MARK: - Retry scheduling (all on `queue`) - - /// Counts one non-transient HTTP rejection against the budget of `part`. - /// The count lives in the manifest, persisted best-effort like the accepted - /// flag. Thus it survives process death and can trip across wakes. A resume - /// or a recreate resets it (ChunkedManifest.reconciled rebuilds the parts - /// from the incoming call). Over budget: journal the terminal 'http' and - /// stall. In budget: schedule the backoff retry when this callback owns the - /// part. For an unowned replay, the reconcile already in flight does the - /// re-enqueueing. - private func recordRejection(_ id: String, part: Int, manifest: ChunkedManifest, - code: Int, headers: [String: String], body: String?, - scheduleRetryInBudget: Bool) { - let count = (manifest.parts[part].rejections ?? 0) + 1 - _ = updateManifest(id) { $0.withPartRejections(part, count) } - if count > ChunkedEngine.partHttpRetries { - let (capped, truncated) = EventJournal.capBody(body) - var entry = errorEntry( - id: id, error: "HTTP \(code) on part \(part)", errorKind: "http", partIndex: part) - entry.responseCode = code - entry.responseBody = capped - entry.responseBodyTruncated = truncated - entry.responseHeaders = headers - stall(id, entry: entry) - } else if scheduleRetryInBudget { - scheduleRetry(id, part: part, attempt: count) - } - } - - private func scheduleTransientRetry(_ id: String, part: Int) { - let attempt = (transientAttempts[id]?[part] ?? 0) + 1 - transientAttempts[id, default: [:]][part] = attempt - scheduleRetry(id, part: part, attempt: attempt) - } - - private func scheduleRetry(_ id: String, part: Int, attempt: Int) { - let delayMs = ChunkedEngine.backoffMs(attempt: attempt) - cooldownUntil[id, default: [:]][part] = nowMs() + Double(delayMs) - queue.asyncAfter(deadline: .now() + .milliseconds(delayMs)) { [weak self] in - guard let self else { return } - self.cooldownUntil[id]?[part] = nil - self.refill(id) - } - } - - // Expiry is evaluated on every transition. But an upload whose tasks all - // wait (for connectivity, or for backoff) would pass its deadline silently - // while the app is alive. Thus we arm one timer at the deadline. When a - // resume extended expiresAt, the stale timer's refill is a no-op that arms - // the timer again. - private func armExpiryCheck(_ id: String, expiresAt: Double) { - guard !expiryArmed.contains(id) else { return } - expiryArmed.insert(id) - let delayMs = Int(min(max(expiresAt - nowMs(), 0) + 100, 7 * 24 * 3_600_000)) - queue.asyncAfter(deadline: .now() + .milliseconds(delayMs)) { [weak self] in - guard let self else { return } - self.expiryArmed.remove(id) - self.refill(id) - } - } - - // MARK: - Helpers - - // The stored copy is the truth. A reconcile can have replaced the headers - // or expiresAt. The cache exists for the progress path, and as a fallback - // when a read fails in flight. - private func latest(_ id: String) -> ChunkedManifest? { - guard let manifest = ChunkedStore.load(id) else { - manifests[id] = nil - return nil - } - manifests[id] = manifest - return manifest - } - - private func updateManifest( - _ id: String, _ transform: (ChunkedManifest) -> ChunkedManifest - ) -> ChunkedManifest? { - // Here the save is best-effort, unlike in startUpload. A lost accepted - // flag only causes a re-send of a part, and the server absorbs the - // duplicate through the accept rules. That is better than a failed upload - // that the server in fact took. - let next = ChunkedStore.update(id, transform) ?? manifests[id].map(transform) - if let next { manifests[id] = next } - return next - } - - private func takeOwnership(path: String, id: String) throws { - let source = URL(string: path) ?? URL(fileURLWithPath: path) - let blob = ChunkedStore.blobURL(id) - let fm = FileManager.default - guard fm.fileExists(atPath: source.path) else { - // A crash between the move and the manifest save leaves the bytes at - // the blob path with no manifest. Adopt the bytes. Do not fail the - // retry. - if fm.fileExists(atPath: blob.path) { return } - throw ChunkedManifest.ParseError( - message: "chunked source file does not exist: \(source.path)") - } - try fm.createDirectory(at: ChunkedStore.uploadDir(id), withIntermediateDirectories: true) - try? fm.removeItem(at: blob) - // This is an O(1) rename on the same volume. Across volumes, FileManager - // falls back to a copy. - try fm.moveItem(at: source, to: blob) - } - - private func emitAggregateProgress(_ id: String, _ manifest: ChunkedManifest) { - let total = manifest.totalBytes - guard total > 0 else { return } - let sent = min(manifest.acceptedBytes + (partSent[id]?.values.reduce(0, +) ?? 0), total) - RNBackgroundUpload.emitProgress(id: id, progress: 100.0 * Float(sent) / Float(total)) - } - - private func clearState(_ id: String) { - inFlight[id] = nil - reconcileToken[id] = nil // discards any pending reconcile snapshot - partSent[id] = nil - cooldownUntil[id] = nil - transientAttempts[id] = nil - manifests[id] = nil - // A removed-then-recreated id must be able to arm its own expiry - // deadline, which is possibly earlier. It must not wait out the stale - // timer. - expiryArmed.remove(id) - progressLock.lock() - lastProgressAt[id] = nil // without this, one entry per id stays forever - progressLock.unlock() - } - - private func cancelTasks(for id: String) { - enumerateAllTasks { tasks in - for (session, task) in tasks where Self.partRef(session, task)?.id == id { - task.cancel() - } - } - } - - // Always examine both sessions. A resume can change wifiOnly while earlier - // part tasks continue where they started. - private func enumerateAllTasks( - _ completion: @escaping ([(URLSession, URLSessionTask)]) -> Void - ) { - let sessions = [uploader.session(wifiOnly: false), uploader.session(wifiOnly: true)] - let group = DispatchGroup() - let lock = NSLock() - var collected: [(URLSession, URLSessionTask)] = [] - for session in sessions { - group.enter() - session.getAllTasks { tasks in - lock.lock() - collected.append(contentsOf: tasks.map { (session, $0) }) - lock.unlock() - group.leave() - } - } - group.notify(queue: .global()) { completion(collected) } + q.commit(n, emit: false) + _ = enqueuePart(n.id, part, delayMs: delay) } - private func expiredEntry(_ id: String) -> JournaledEvent { - errorEntry(id: id, error: "upload expired before every part was accepted", - errorKind: "expired") + /// Sets nextAttemptAt to the earliest begin date when every part in the + /// window is a delayed task that has not begun, and clears it otherwise. + /// The state stays running. Emits `state` only on a change. + private func updateWait(_ id: String) { + guard var e = q.index.entry(id), e.isChunked, e.state == .running else { return } + let flight = inFlight[id] ?? [:] + let waits = partBeginAt[id] ?? [:] + let next = !flight.isEmpty && flight.keys.allSatisfy { waits[$0] != nil } + ? flight.keys.compactMap { waits[$0] }.min() : nil + guard next != e.nextAttemptAt else { return } + e.nextAttemptAt = next + q.commit(e) } - private func errorEntry(id: String, error: String, errorKind: String, - partIndex: Int? = nil) -> JournaledEvent { - var entry = JournaledEvent( - eventId: UUID().uuidString, id: id, type: "error", timestamp: nowMs()) - entry.error = error - entry.errorKind = errorKind - entry.partIndex = partIndex - return entry + /// Byte-weighted: accepted parts plus what the live part tasks sent. The + /// row carries the same value. + private func emitProgress(_ e: QueueEntry) { + guard e.totalBytes > 0 else { return } + let sent = min(e.acceptedBytes + (partSent[e.id]?.values.reduce(0, +) ?? 0), e.totalBytes) + q.index.setBytes(e.id, sent) + q.emitProgress(e.id, sent: sent, total: e.totalBytes) } } diff --git a/ios/ChunkedEngine.swift b/ios/ChunkedEngine.swift index caf3007c..38b6dbdf 100644 --- a/ios/ChunkedEngine.swift +++ b/ios/ChunkedEngine.swift @@ -1,63 +1,34 @@ import Foundation -// The pure scheduling half of chunked execution: window arithmetic, the -// retry policy, backoff, and the part-task identity encoding. It is kept free -// of session state. Thus the highest-consequence invariants (at most WINDOW -// part tasks enqueued per upload, and never two for one part index) can be -// examined in one place. [ChunkedCoordinator] owns the session side. +// The pure half of task scheduling: the chunked window and the task identity +// encodings. Free of session state, so the high-consequence invariants (at +// most `window` part tasks per upload, never two for one part index) can be +// examined in one place. ChunkedCoordinator owns the session side. enum ChunkedEngine { // The number of part tasks of one upload enqueued with the daemon at one - // time. It is a library constant, not an option: if soak data argues for a - // different value, this constant changes, not the API. The window is also a - // liveness decision. A background session only progresses tasks that are - // already enqueued. Thus WINDOW tasks of runway let a multi-part upload - // proceed while the app is dead. Without them, the upload pays a - // rate-limited wake per part. + // time. A library constant, not an option. It is also a liveness decision: + // a background session only progresses tasks already enqueued, so `window` + // tasks of runway let an upload proceed while the app is dead. static let window = 3 - // A non-accepted, non-transient HTTP response is retried this many times - // for each part. Then it becomes a terminal error and stalls the upload. - // The number is small on purpose. A response that the server repeats (401, - // 400) will not change without a new startUpload. Only transient failures - // retry without a limit. - static let partHttpRetries = 3 - - private static let backoffBaseMs = 1_000 - private static let backoffCapMs = 60_000 - - // A 5xx means that the server fails, not that the request is wrong. Thus - // it retries like a transport failure: without a limit, within expiresAt. - static func isTransientHttp(_ code: Int) -> Bool { (500...599).contains(code) } - - /// Exponential backoff for transient failures: 1s, 2s, 4s, up to a 60s cap. - static func backoffMs(attempt: Int) -> Int { - min(backoffBaseMs << min(max(attempt - 1, 0), 6), backoffCapMs) - } - - /// The part indexes to enqueue now: pending (not accepted), not already - /// enqueued, and not cooling down after a failure, up to the window size. - /// It never returns an index in `inFlight`. That is the one-task-per-part - /// invariant. - static func indexesToEnqueue( - pending: [Int], inFlight: Set, cooling: Set, window: Int = window - ) -> [Int] { + /// The part indexes to enqueue now: pending (not accepted) and not already + /// in flight, up to the free window slots. It never returns an index in + /// `inFlight`: the one-task-per-part invariant. A part waiting out a + /// backoff is in flight (its delayed task holds the slot). + static func indexesToEnqueue(pending: [Int], inFlight: Set, window: Int = window) -> [Int] { let slots = window - inFlight.count guard slots > 0 else { return [] } - return Array(pending.filter { !inFlight.contains($0) && !cooling.contains($0) }.prefix(slots)) + return Array(pending.filter { !inFlight.contains($0) }.prefix(slots)) } - // MARK: - Part-task identity + // MARK: - Task identity - // A chunked part task must carry (uploadId, partIndex, incarnation) - // through the daemon. taskDescription is the primary carrier. It is a - // prefix plus JSON, so a consumer id that contains a delimiter survives. - // TaskMap holds the same triple as the durable fallback, per the DTS - // guidance that TaskMap documents. The incarnation is the manifest token - // that the task was created under. A callback whose token no longer matches - // the stored manifest's token is from a removed or replaced plan. It must - // not write into the current plan. - private static let descriptionPrefix = "rnbgu-chunk:" + // A task carries its owner through the daemon in taskDescription: a prefix + // plus JSON, so an id with a colon or a slash survives. TaskMap holds the + // same fields as the durable fallback. + private static let partPrefix = "rnbgu-chunk:" + private static let requestPrefix = "rnbgu-req:" private struct PartRef: Codable { let id: String @@ -65,17 +36,39 @@ enum ChunkedEngine { var inc: String? } + private struct RequestRef: Codable { + let id: String + let gen: Int + let att: Int + } + + /// A chunked part task: (id, part index, incarnation). static func taskDescription(id: String, part: Int, incarnation: String) -> String { - let data = (try? JSONEncoder().encode(PartRef(id: id, part: part, inc: incarnation))) ?? Data() - return descriptionPrefix + (String(data: data, encoding: .utf8) ?? "") + partPrefix + encode(PartRef(id: id, part: part, inc: incarnation)) } - static func parseTaskDescription( - _ description: String? - ) -> (id: String, part: Int, incarnation: String?)? { - guard let description, description.hasPrefix(descriptionPrefix) else { return nil } - let json = Data(description.dropFirst(descriptionPrefix.count).utf8) - guard let ref = try? JSONDecoder().decode(PartRef.self, from: json) else { return nil } + static func parsePartDescription(_ description: String?) -> (id: String, part: Int, incarnation: String?)? { + guard let ref: PartRef = decode(description, partPrefix) else { return nil } return (ref.id, ref.part, ref.inc) } + + /// A simple attempt: (id, entry generation, attempt ordinal). + static func taskDescription(id: String, attempt: Int, generation: Int) -> String { + requestPrefix + encode(RequestRef(id: id, gen: generation, att: attempt)) + } + + static func parseRequestDescription(_ description: String?) -> (id: String, generation: Int, attempt: Int)? { + guard let ref: RequestRef = decode(description, requestPrefix) else { return nil } + return (ref.id, ref.gen, ref.att) + } + + private static func encode(_ value: T) -> String { + let data = (try? JSONEncoder().encode(value)) ?? Data() + return String(data: data, encoding: .utf8) ?? "" + } + + private static func decode(_ description: String?, _ prefix: String) -> T? { + guard let description, description.hasPrefix(prefix) else { return nil } + return try? JSONDecoder().decode(T.self, from: Data(description.dropFirst(prefix.count).utf8)) + } } diff --git a/ios/ChunkedManifest.swift b/ios/ChunkedManifest.swift deleted file mode 100644 index 5375cd3c..00000000 --- a/ios/ChunkedManifest.swift +++ /dev/null @@ -1,404 +0,0 @@ -import Foundation - -/// The durable record of one chunked upload: the parts that the consumer -/// authored, and which of them the server has accepted. [ChunkedStore] saves -/// it at startUpload, BEFORE any task is enqueued. Thus a process that the -/// system relaunches (or a startUpload after a crash, a stop, or a reauth) -/// resumes from it without a call into JS. This manifest IS the resume -/// mechanism. -/// -/// The content is the same as the Android manifest, with two platform -/// differences: -/// - There is no sourcePath field. iOS moves the app container between -/// launches, so an absolute path would go stale. The moved bytes live at a -/// location derived from the id (ChunkedStore.blobURL). -/// - `stalled` is persisted. On Android, "stalled" only means that the worker -/// is not scheduled. iOS reconciles every upload on relaunch. Thus an -/// upload that journaled a terminal outcome needs a durable marker that -/// says: await an explicit startUpload resume, and do not refill. -struct ChunkedManifest: Codable { - /// One part, exactly as the consumer authored it. The library sends the - /// file bytes [start, end) as the body of a PUT to `url`, with `headers` - /// unchanged. It never derives or edits a protocol field. - struct Part: Codable { - let url: String - var headers: [String: String] - let start: Int64 - let end: Int64 // exclusive - var accepted: Bool = false - /// The non-transient HTTP rejections counted against this part's retry - /// budget (nil means 0). It is persisted so that the budget survives - /// process death. An in-memory count resets on every system wake. That - /// would let a deterministic 4xx upload the part again until expiresAt, - /// with no terminal ever journaled. The count resets when the part is - /// rebuilt from an incoming call. Resume and recreate both do that (see - /// [reconciled]). - var rejections: Int? - - var size: Int64 { end - start } - } - - let id: String - var parts: [Part] - var accept: [UploadOutcome.AcceptRule] - /// Epoch ms. After this time, the upload stops with errorKind 'expired'. - var expiresAt: Double - var wifiOnly: Bool - let createdAt: Double - var stalled: Bool = false - /// The identity of this parts plan. It rotates on a recreate (a startUpload - /// that replaced the parts wholesale). It never rotates on a resume. Part - /// tasks carry it in their identity. Thus a late delegate callback from a - /// removed or replaced incarnation can be told apart from the live plan's - /// callbacks and dropped. Its response is about byte ranges and URLs that - /// this manifest no longer describes. - var incarnation: String - - var totalBytes: Int64 { parts.reduce(0) { $0 + $1.size } } - var acceptedBytes: Int64 { parts.filter(\.accepted).reduce(0) { $0 + $1.size } } - - /// The server's auto-publish condition. It is the only thing that - /// 'completed' may mean. - var allAccepted: Bool { parts.allSatisfy(\.accepted) } - - func isExpired(_ nowMs: Double) -> Bool { nowMs >= expiresAt } - - func pendingIndexes() -> [Int] { parts.indices.filter { !parts[$0].accepted } } - - func withPartAccepted(_ index: Int) -> ChunkedManifest { - var next = self - next.parts[index].accepted = true - return next - } - - func withPartRejections(_ index: Int, _ count: Int) -> ChunkedManifest { - var next = self - next.parts[index].rejections = count - return next - } - - struct ReconcileError: LocalizedError { - let message: String - var errorDescription: String? { message } - } - - /// A new startUpload call with an existing id is one of two things. The - /// semantics are identical to Android's `ChunkedManifest.reconcile`. - /// - /// **Resume** — the incoming parts are the SAME array (identical count, - /// ranges, and urls). The headers, the accept rules, expiresAt, and wifiOnly - /// come from the new call. This is how fresh auth reaches stalled parts, - /// and how a salvage extends the deadline. The accepted part statuses, - /// createdAt, and the incarnation survive from this manifest. A resume is - /// permitted at any time, running or not. The stall clears, because a - /// resume is the whole point of the new call. - /// - /// **Recreate** — a DIFFERENT parts array: the consumer authored the upload - /// again, under a fresh server uploadId, after the old one died. The owned - /// bytes are kept. The parts are replaced wholesale. Every part status - /// resets to unsent. The headers, accept rules, and expiresAt come from the - /// new call. The new ranges must tile exactly [0, blobSize). A partial or - /// overlapping cover would silently upload wrong bytes. A recreate is - /// accepted only while the upload is NOT running (stalled on a terminal - /// error or cancel, or expired). A different parts array while part tasks - /// are live is a consumer bug, not a recreate, because the in-flight - /// requests belong to the old parts. The incarnation rotates to the - /// incoming manifest's fresh token. Thus late callbacks from the replaced - /// parts are dropped. - func reconciled(with incoming: ChunkedManifest, running: Bool, - blobSize: Int64) throws -> ChunkedManifest { - if samePartsAs(incoming) { - // Built from `incoming`, so the per-part rejection counts reset. A - // resume arrives with fresh headers and gets a fresh retry budget. - let mergedParts = incoming.parts.enumerated().map { i, new -> Part in - var part = new - part.accepted = parts[i].accepted - return part - } - return ChunkedManifest( - id: id, parts: mergedParts, accept: incoming.accept, expiresAt: incoming.expiresAt, - wifiOnly: incoming.wifiOnly, createdAt: createdAt, stalled: false, - incarnation: incarnation) - } - guard !running else { - throw ReconcileError(message: - "chunked upload '\(id)' is running; a different parts array is only accepted once it stops") - } - guard Self.tilesExactly(incoming.parts, size: blobSize) else { - throw ReconcileError(message: - "chunked upload '\(id)' recreate parts must tile exactly [0, \(blobSize))") - } - return ChunkedManifest( - id: id, parts: incoming.parts, accept: incoming.accept, expiresAt: incoming.expiresAt, - wifiOnly: incoming.wifiOnly, createdAt: createdAt, stalled: false, - incarnation: incoming.incarnation) - } - - private func samePartsAs(_ incoming: ChunkedManifest) -> Bool { - incoming.parts.count == parts.count && parts.indices.allSatisfy { i in - incoming.parts[i].url == parts[i].url - && incoming.parts[i].start == parts[i].start - && incoming.parts[i].end == parts[i].end - } - } - - /// Tells whether `parts` cover [0, size) exactly: no gap, no overlap, and - /// nothing past the end. It is order-independent, like everything else - /// about parts. - static func tilesExactly(_ parts: [Part], size: Int64) -> Bool { - guard !parts.isEmpty else { return false } - var cursor: Int64 = 0 - for part in parts.sorted(by: { $0.start < $1.start }) { - guard part.start == cursor, part.end > part.start else { return false } - cursor = part.end - } - return cursor == size - } - - struct ParseError: LocalizedError { - let message: String - var errorDescription: String? { message } - } - - /// Turns bridged options into a manifest. It throws on each field that the - /// engine relies on. JS validates first. Thus a throw here is a bug worth - /// surfacing, not UX. - static func parse(_ options: [String: Any], createdAt: Double) throws -> ChunkedManifest { - guard let id = options["id"] as? String, !id.isEmpty else { - throw ParseError(message: "Missing 'id'") - } - guard let rawParts = options["parts"] as? [[String: Any]], !rawParts.isEmpty else { - throw ParseError(message: "'parts' must be a non-empty array") - } - guard let expiresAt = (options["expiresAt"] as? NSNumber)?.doubleValue else { - throw ParseError(message: "Missing 'expiresAt'") - } - let parts = try rawParts.enumerated().map { i, raw -> Part in - guard let url = raw["url"] as? String else { - throw ParseError(message: "Missing 'parts[\(i)].url'") - } - guard let range = raw["range"] as? [String: Any], - let start = (range["start"] as? NSNumber)?.int64Value, - let end = (range["end"] as? NSNumber)?.int64Value, - start >= 0, start < end else { - throw ParseError(message: "Invalid 'parts[\(i)].range'") - } - return Part(url: url, headers: parseHeaders(raw["headers"]), start: start, end: end) - } - return ChunkedManifest( - id: id, - parts: parts, - accept: UploadOutcome.parseAcceptRules(options["accept"]), - expiresAt: expiresAt, - wifiOnly: (options["wifiOnly"] as? Bool) ?? false, - createdAt: createdAt, - incarnation: UUID().uuidString) - } - - // The same header coercion as the simple-upload path: strings and numbers - // only. Anything else is skipped. It is not interpolated onto the wire. - private static func parseHeaders(_ raw: Any?) -> [String: String] { - guard let headers = raw as? [String: Any] else { return [:] } - var result: [String: String] = [:] - for (key, value) in headers { - if let s = value as? String { - result[key] = s - } else if let n = value as? NSNumber { - result[key] = n.stringValue - } - } - return result - } -} - -/// A file-backed store: one directory per upload id. The directory holds -/// `manifest.json`, `blob` (the moved source bytes), and the in-flight part -/// temp files. It has the same durability pattern as [EventJournal]: a -/// synchronous serial queue, atomic writes, and corrupt files read as -/// absent. -enum ChunkedStore { - struct StoreError: LocalizedError { - let message: String - var errorDescription: String? { message } - } - - private static let queue = DispatchQueue(label: "ai.openspace.rnbgupload.chunkedstore") - - private static let dirURL: URL = { - let base = FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask)[0] - var dir = base.appendingPathComponent("RNFileUploaderChunked", isDirectory: true) - try? FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) - // This is device-local upload state. Keep it out of iCloud/iTunes - // backups. - var values = URLResourceValues() - values.isExcludedFromBackup = true - try? dir.setResourceValues(values) - return dir - }() - - // Upload ids come from the consumer. They can contain path separators or - // other filesystem-hostile characters. Thus the directory name is an - // encoding of the id, never the id itself. The id is read back from the - // manifest, not decoded from the name. - static func uploadDir(_ id: String) -> URL { - dirURL.appendingPathComponent( - Data(id.utf8).base64EncodedString() - .replacingOccurrences(of: "+", with: "-") - .replacingOccurrences(of: "/", with: "_") - .replacingOccurrences(of: "=", with: ""), - isDirectory: true) - } - - private static func manifestURL(_ id: String) -> URL { - uploadDir(id).appendingPathComponent("manifest.json") - } - - /// The location where startUpload moves the source file for this id. - static func blobURL(_ id: String) -> URL { - uploadDir(id).appendingPathComponent("blob") - } - - /// The temp file that holds exactly the byte range of part `index` while - /// that part is enqueued with the daemon (a background session can upload - /// only from a file). The name encodes the manifest incarnation and the - /// byte range. Thus a stale file from a previous incarnation or a different - /// plan can never be adopted by a size coincidence. Reuse checks the full - /// identity, not only the byte count. - static func partFileURL(_ id: String, _ index: Int, incarnation: String, - start: Int64, end: Int64) -> URL { - uploadDir(id).appendingPathComponent("part-\(index).\(incarnation).\(start)-\(end)") - } - - /// The size in bytes of the moved blob. It is 0 when the blob is missing. - static func blobSize(_ id: String) -> Int64 { - queue.sync { - (((try? FileManager.default.attributesOfItem(atPath: blobURL(id).path))?[.size] - as? NSNumber)?.int64Value) ?? 0 - } - } - - static func load(_ id: String) -> ChunkedManifest? { - queue.sync { read(manifestURL(id)) } - } - - /// Throws on a write failure. A manifest that did not persist must fail - /// the startUpload call. - static func save(_ manifest: ChunkedManifest) throws { - try queue.sync { - try FileManager.default.createDirectory( - at: uploadDir(manifest.id), withIntermediateDirectories: true) - let data = try JSONEncoder().encode(manifest) - try data.write(to: manifestURL(manifest.id), options: .atomic) - } - } - - /// An atomic read-modify-write. Thus a mark of one part as accepted can - /// never clobber a concurrent reconcile's fresh headers, or another part's - /// flag. It returns nil, and does not throw, when the manifest is gone or - /// the write failed. Callers that can proceed from memory do so. - static func update(_ id: String, _ transform: (ChunkedManifest) -> ChunkedManifest) -> ChunkedManifest? { - queue.sync { - guard let manifest = read(manifestURL(id)) else { return nil } - let next = transform(manifest) - guard let data = try? JSONEncoder().encode(next) else { return nil } - do { - try data.write(to: manifestURL(id), options: .atomic) - return next - } catch { - return nil - } - } - } - - /// Deletes the manifest, the moved bytes, AND all part temp files. It does - /// nothing for an unknown id (for example, a simple upload's id). - static func remove(_ id: String) { - queue.sync { try? FileManager.default.removeItem(at: uploadDir(id)) } - } - - static func all() -> [ChunkedManifest] { - queue.sync { - let dirs = (try? FileManager.default.contentsOfDirectory( - at: dirURL, includingPropertiesForKeys: nil)) ?? [] - return dirs.compactMap { read($0.appendingPathComponent("manifest.json")) } - } - } - - // Codable enforces the non-optional fields at decode time, unlike Gson. - // Thus a corrupt or field-renamed file simply reads as absent. - private static func read(_ url: URL) -> ChunkedManifest? { - guard let data = try? Data(contentsOf: url) else { return nil } - guard let m = try? JSONDecoder().decode(ChunkedManifest.self, from: data), - !m.parts.isEmpty else { return nil } - return m - } - - /// Writes bytes [start, end) of the blob into `dest`. It writes a tmp file - /// and renames it. Thus a partial write can never be mistaken for a - /// finished part file. It throws when the blob is missing or shorter than - /// `end`. For the caller that is a 'file' terminal, because a retry can - /// never succeed. - static func writePartFile(id: String, index: Int, start: Int64, end: Int64, - incarnation: String) throws -> URL { - try queue.sync { - let dest = partFileURL(id, index, incarnation: incarnation, start: start, end: end) - // Sweep the other files of this index first. A temp file left by a - // replaced incarnation or plan must not stay and leak disk. It cannot - // be reused, because the identity is in the name, but it can pile up. - removePartFilesLocked(id, index, keeping: dest) - // An existing file with exactly this identity and size is a finished - // copy from a previous enqueue of this part. Reuse it. Size alone is - // not trusted. The name carries the incarnation and the range that - // produced the file. - if let size = try? FileManager.default.attributesOfItem(atPath: dest.path)[.size] as? NSNumber, - size.int64Value == end - start { - return dest - } - let blob = blobURL(id) - let blobSize = ((try FileManager.default.attributesOfItem(atPath: blob.path)[.size] - as? NSNumber)?.int64Value) ?? 0 - guard blobSize >= end else { - throw StoreError( - message: "source blob is \(blobSize) bytes; part \(index) needs [\(start), \(end))") - } - let tmp = uploadDir(id).appendingPathComponent("part-\(index).tmp") - FileManager.default.createFile(atPath: tmp.path, contents: nil) - let reader = try FileHandle(forReadingFrom: blob) - defer { try? reader.close() } - let writer = try FileHandle(forWritingTo: tmp) - defer { try? writer.close() } - try reader.seek(toOffset: UInt64(start)) - var remaining = end - start - while remaining > 0 { - let chunk = Int(min(remaining, 1 << 20)) - guard let data = try reader.read(upToCount: chunk), !data.isEmpty else { - throw StoreError(message: "short read building part \(index)") - } - try writer.write(contentsOf: data) - remaining -= Int64(data.count) - } - try? FileManager.default.removeItem(at: dest) - try FileManager.default.moveItem(at: tmp, to: dest) - return dest - } - } - - /// Removes every file for part `index`: the current incarnation's file, - /// stale files, and half-written tmp files. They all share the - /// `part-.` prefix. - static func removePartFile(_ id: String, _ index: Int) { - queue.sync { removePartFilesLocked(id, index, keeping: nil) } - } - - // Must run on `queue`. The `part-.` prefix cannot collide across - // indexes ("part-1." is not a prefix of "part-12.<...>"). - private static func removePartFilesLocked(_ id: String, _ index: Int, keeping: URL?) { - let files = (try? FileManager.default.contentsOfDirectory( - at: uploadDir(id), includingPropertiesForKeys: nil)) ?? [] - for file in files - where file.lastPathComponent.hasPrefix("part-\(index).") - && file.lastPathComponent != keeping?.lastPathComponent { - try? FileManager.default.removeItem(at: file) - } - } -} diff --git a/ios/ChunkedManifestV9.swift b/ios/ChunkedManifestV9.swift new file mode 100644 index 00000000..b7bac607 --- /dev/null +++ b/ios/ChunkedManifestV9.swift @@ -0,0 +1,35 @@ +import Foundation + +/// The v9 chunked manifest (`manifest.json`), kept only to read the files a +/// v9 build left behind. Decode only; v10 never writes it. A same-id enqueue +/// with the same parts adopts its accepted flags, incarnation and blob. The +/// legacy row reads expiresAt and the byte counts. Other v9 keys are ignored. +struct ChunkedManifestV9: Codable, Equatable { + struct Part: Codable, Equatable { + let url: String + let start: Int64 + let end: Int64 + var accepted: Bool + + var size: Int64 { end - start } + } + + let id: String + var parts: [Part] + var expiresAt: Double + var incarnation: String + + /// v9 wrote the moved bytes at this fixed name. + static let blobName = "blob" + + var totalBytes: Int64 { parts.reduce(0) { $0 + $1.size } } + var acceptedBytes: Int64 { parts.filter(\.accepted).reduce(0) { $0 + $1.size } } + + /// Same count, and the same url and range at each index. + func sameParts(as other: [QueueEntry.Part]) -> Bool { + parts.count == other.count && parts.indices.allSatisfy { + parts[$0].url == other[$0].url && parts[$0].start == other[$0].start + && parts[$0].end == other[$0].end + } + } +} diff --git a/ios/EnqueueParser.swift b/ios/EnqueueParser.swift new file mode 100644 index 00000000..6e5474d5 --- /dev/null +++ b/ios/EnqueueParser.swift @@ -0,0 +1,223 @@ +import Foundation + +/// What enqueue() received, validated. A throw here rejects E_INVALID: input +/// native cannot send. +struct ParsedEnqueue { + enum Body { + case none + case data(json: String) + case form([FormField]) + case file(path: String) + case parts(file: String) + } + + struct FormField: Equatable { + let name: String + let contentType: String + let string: String? + let path: String? + let fileName: String? + } + + let id: String + let key: String + let varsJSON: String + let url: String? + let method: String + let headers: [String: String] + let body: Body + let parts: [QueueEntry.Part] + let accept: [UploadOutcome.AcceptRule] + let expiresAt: Double + let retry: RetryOverride? + /// Body identity for the same-id rules. + let fingerprint: String +} + +struct ParseError: LocalizedError { + let message: String + var errorDescription: String? { message } +} + +enum EnqueueParser { + static let methods: Set = ["POST", "PUT", "PATCH", "DELETE", "GET"] + + /// Parses the bridged `{ id, key, varsJson, descriptor }`. `varsJson` and + /// `descriptor.dataJson` are JSON text, because React Native on iOS drops + /// object keys whose value is null. dataJson "null" is the JSON body null. + /// NSNull counts as absent for the other optional fields. + static func parse(_ raw: [String: Any]) throws -> ParsedEnqueue { + guard let id = raw["id"] as? String, !id.isEmpty else { throw ParseError(message: "missing 'id'") } + guard let key = raw["key"] as? String, !key.isEmpty else { throw ParseError(message: "missing 'key'") } + guard let d = raw["descriptor"] as? [String: Any] else { + throw ParseError(message: "missing 'descriptor'") + } + guard let varsJSON = raw["varsJson"] as? String, JSONText.parse(varsJSON) != nil else { + throw ParseError(message: "'varsJson' must be JSON text") + } + // An object `data` would have lost its null-valued keys on the way here. + guard d["data"] == nil else { throw ParseError(message: "'data' must cross as 'dataJson'") } + let method = ((present(d["method"]) as? String) ?? "POST").uppercased() + guard methods.contains(method) else { throw ParseError(message: "unknown method '\(method)'") } + guard let expiresAt = (present(d["expiresAt"]) as? NSNumber)?.doubleValue else { + throw ParseError(message: "missing 'expiresAt'") + } + + let url = present(d["url"]) as? String + if let url { try requireURL(url, "url") } + + var kinds: [ParsedEnqueue.Body] = [] + if let text = present(d["dataJson"]) { + guard let json = text as? String, JSONText.parse(json) != nil else { + throw ParseError(message: "'dataJson' must be JSON text") + } + kinds.append(.data(json: json)) + } + if let form = present(d["form"]) { kinds.append(.form(try parseForm(form))) } + let file = present(d["file"]) as? String + var parts: [QueueEntry.Part] = [] + if let rawParts = present(d["parts"]) { + guard let file else { throw ParseError(message: "'parts' requires 'file'") } + parts = try parseParts(rawParts) + kinds.append(.parts(file: file)) + } else if let file { + kinds.append(.file(path: file)) + } + guard kinds.count <= 1 else { throw ParseError(message: "more than one body kind") } + guard url != nil || !parts.isEmpty else { throw ParseError(message: "'url' is required unless 'parts' is set") } + let body = kinds.first ?? .none + if method == "GET", !kinds.isEmpty { throw ParseError(message: "a GET request cannot have a body") } + + return ParsedEnqueue( + id: id, key: key, varsJSON: varsJSON, url: url, + method: method, headers: try headers(present(d["headers"])), body: body, parts: parts, + accept: UploadOutcome.parseAcceptRules(present(d["accept"])), expiresAt: expiresAt, + retry: RetryOverride.parse(present(d["retry"])), + fingerprint: fingerprint(body, parts: parts, url: url, method: method)) + } + + /// Strings and numbers become headers. Anything else is skipped, never + /// interpolated onto the wire ("" in an Authorization header). + /// Throws on a name that is not an HTTP token, or a value with CR, LF or + /// NUL: URLRequest drops those without a word. The message names the + /// header, never its value. + static func headers(_ raw: Any?, field: String = "headers") throws -> [String: String] { + guard let headers = raw as? [String: Any] else { return [:] } + var result: [String: String] = [:] + for (k, v) in headers { + let value: String + if let s = v as? String { + value = s + } else if let n = v as? NSNumber { + value = n.stringValue + } else { + continue + } + guard isToken(k) else { throw ParseError(message: "'\(field)' has an invalid header name") } + guard !hasLineBreakOrNUL(value) else { + throw ParseError(message: "'\(field)' header '\(k)' has a line break or NUL in its value") + } + result[k] = value + } + return result + } + + // RFC 9110 token: the characters a header name may use. + private static let tokenChars = CharacterSet(charactersIn: "!#$%&'*+-.^_`|~") + .union(CharacterSet(charactersIn: "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")) + + // By scalar: "\r\n" is one Character, so a Character compare misses it. + private static func hasLineBreakOrNUL(_ s: String) -> Bool { + s.unicodeScalars.contains { $0 == "\r" || $0 == "\n" || $0 == "\0" } + } + + private static func isToken(_ name: String) -> Bool { + !name.isEmpty && name.unicodeScalars.allSatisfy { tokenChars.contains($0) } + } + + /// Body identity, and the url and method: a different url or method is a + /// different body. data: canonical JSON, so key order does not matter. + /// form: the fields as sent; paths compare as strings, because the caller + /// may have deleted the source after the first mutate() resolved. file: the + /// path as sent. parts: url and range of each part; the `file` path is not + /// part of it, because the moved blob is the body. + static func fingerprint(_ body: ParsedEnqueue.Body, parts: [QueueEntry.Part], url: String?, + method: String) -> String { + bodyFingerprint(body, parts: parts) + "|" + JSONText.sha256(method + " " + (url ?? "")) + } + + private static func bodyFingerprint(_ body: ParsedEnqueue.Body, parts: [QueueEntry.Part]) -> String { + switch body { + case .none: + return "none" + case .data(let json): + // Canonical text, so key order does not matter. + let canonical = JSONText.parse(json).flatMap { JSONText.encode($0) } ?? json + return "data:" + JSONText.sha256(canonical) + case .form(let fields): + let text = fields.map { f in + [f.name, f.contentType, f.string.map { "s:" + $0 } ?? "", f.path.map { "p:" + $0 } ?? "", + f.fileName ?? ""].map { $0.replacingOccurrences(of: "|", with: "||") }.joined(separator: "|") + }.joined(separator: "\n") + return "form:" + JSONText.sha256(text) + case .file(let path): + return "file:" + path + case .parts: + return "parts:" + JSONText.sha256(parts.map { "\($0.url)|\($0.start)|\($0.end)" }.joined(separator: "\n")) + } + } + + private static func present(_ value: Any?) -> Any? { + value is NSNull ? nil : value + } + + /// http or https with a host. The message leaves the URL out: its query + /// may hold a token. + private static func requireURL(_ s: String, _ field: String) throws { + guard let u = URL(string: s), let scheme = u.scheme?.lowercased(), + scheme == "http" || scheme == "https", let host = u.host, !host.isEmpty else { + throw ParseError(message: "'\(field)' must be an http or https URL") + } + } + + private static func parseForm(_ raw: Any) throws -> [ParsedEnqueue.FormField] { + guard let fields = raw as? [[String: Any]], !fields.isEmpty else { + throw ParseError(message: "'form' must be a non-empty array") + } + return try fields.enumerated().map { i, f in + guard let name = f["name"] as? String, let contentType = f["contentType"] as? String else { + throw ParseError(message: "'form[\(i)]' needs name and contentType") + } + // The content type goes into the multipart part header as it is. + guard !hasLineBreakOrNUL(contentType) else { + throw ParseError(message: "'form[\(i)].contentType' has a line break") + } + let string = present(f["string"]) as? String + let path = present(f["path"]) as? String + guard (string == nil) != (path == nil) else { + throw ParseError(message: "'form[\(i)]' must set exactly one of string, path") + } + return ParsedEnqueue.FormField( + name: name, contentType: contentType, string: string, path: path, + fileName: present(f["fileName"]) as? String) + } + } + + private static func parseParts(_ raw: Any) throws -> [QueueEntry.Part] { + guard let parts = raw as? [[String: Any]], !parts.isEmpty else { + throw ParseError(message: "'parts' must be a non-empty array") + } + return try parts.enumerated().map { i, p in + guard let url = p["url"] as? String else { throw ParseError(message: "missing 'parts[\(i)].url'") } + try requireURL(url, "parts[\(i)].url") + guard let range = p["range"] as? [String: Any], + let start = (range["start"] as? NSNumber)?.int64Value, + let end = (range["end"] as? NSNumber)?.int64Value, + start >= 0, start < end else { + throw ParseError(message: "invalid 'parts[\(i)].range'") + } + return QueueEntry.Part(url: url, headers: try headers(present(p["headers"]), field: "parts[\(i)].headers"), start: start, + end: end, accepted: false, rejections: 0) + } + } +} diff --git a/ios/EventJournal.swift b/ios/EventJournal.swift index 3011acd8..46a0af89 100644 --- a/ios/EventJournal.swift +++ b/ios/EventJournal.swift @@ -1,134 +1,266 @@ import Foundation -// A terminal upload outcome, persisted before it is emitted to JS. -struct JournaledEvent: Codable { +/// The last response of a request, as the journal keeps it. +struct RawResponseRecord: Codable, Equatable { + var status: Int? + var headers: [String: String]? + var body: String? + var bodyTruncated: Bool + + var bridged: [String: Any] { + var m: [String: Any] = ["bodyTruncated": bodyTruncated] + if let status { m["status"] = status } + if let headers { m["headers"] = headers } + if let body { m["body"] = body } + return m + } +} + +struct OutcomeErrorRecord: Codable, Equatable { + var errorKind: String // http | network | file | expired | unknown + var message: String + var response: RawResponseRecord? + var partIndex: Int? + + var bridged: [String: Any] { + var m: [String: Any] = ["errorKind": errorKind, "message": message] + if let response { m["response"] = response.bridged } + if let partIndex { m["partIndex"] = partIndex } + return m + } +} + +/// A terminal outcome, in the SettledEvent shape. Journaled before it is +/// emitted; deleted when JS acknowledges it. +struct JournaledEvent: Codable, Equatable { + enum Kind: String, Codable { case completed, error, cancelled } + let eventId: String - let id: String // upload id - var type: String // completed | error | cancelled - let timestamp: Double // epoch ms - var responseCode: Int? - var responseBody: String? - var responseBodyTruncated: Bool? - var responseHeaders: [String: String]? - var error: String? - var errorKind: String? // http | network | file | expired | unknown - var cancelReason: String? // user | system - var partIndex: Int? // chunked uploads only: the failing part, when known - - // Bridge-friendly dictionary (nil fields omitted so nothing becomes NSNull). + let id: String + let key: String + let varsJSON: String + let at: Double + var attempts: Int + var requestId: String? + /// 0 when journaled; +1 on every emit and every getUnacknowledgedEvents. + var deliveries: Int + var bytesSent: Int64 + var totalBytes: Int64 + var url: String + var method: String + var partIndex: Int? + /// The entry generation this outcome settled. An ack forgets the entry + /// only when it still matches. + var generation: Int + var kind: Kind + var response: RawResponseRecord? + var error: OutcomeErrorRecord? + var cancelReason: String? + + /// The SettledEvent dictionary. Nil fields are omitted, so nothing becomes + /// NSNull; `vars` is the decoded object (NSNull for JS null). var bridged: [String: Any] { - var m: [String: Any] = ["eventId": eventId, "id": id, "type": type, "timestamp": timestamp] - if let responseCode { m["responseCode"] = responseCode } - if let responseBody { m["responseBody"] = responseBody } - if let responseBodyTruncated { m["responseBodyTruncated"] = responseBodyTruncated } - if let responseHeaders { m["responseHeaders"] = responseHeaders } - if let error { m["error"] = error } - if let errorKind { m["errorKind"] = errorKind } - if let cancelReason { m["cancelReason"] = cancelReason } + var m: [String: Any] = [ + "eventId": eventId, "id": id, "key": key, "vars": JSONText.decode(varsJSON), "at": at, + "attempts": attempts, "deliveries": deliveries, "state": kind.rawValue, + "bytesSent": bytesSent, "totalBytes": totalBytes, "url": url, "method": method, + "kind": kind.rawValue, + ] + if let requestId { m["requestId"] = requestId } if let partIndex { m["partIndex"] = partIndex } + switch kind { + case .completed: + m["response"] = (response ?? RawResponseRecord(bodyTruncated: false)).bridged + case .error: + m["error"] = (error ?? OutcomeErrorRecord(errorKind: "unknown", message: "")).bridged + case .cancelled: + m["cancelReason"] = cancelReason ?? "user" + } return m } } -// Durable record of terminal upload events (completed / error / cancelled). -// Written BEFORE the event is emitted to JS, deleted only when JS acknowledges, -// so an outcome that fires while JS is dead survives to the next launch. -// -// Synchronous (serial queue) — deliberately NOT an actor. The URLSession delegate -// is synchronous and must journal an outcome BEFORE emitting it; an actor would -// force that ordering to become async and racy. -// -// One JSON file per event: Data.write(atomically:) is its own tmp+rename, so a -// crash mid-write can't corrupt other entries, and separate files avoid a shared -// mutable file across processes. -enum EventJournal { - static let maxBodyChars = 64 * 1024 - static let maxEntries = 1000 - - private static let queue = DispatchQueue(label: "ai.openspace.rnbgupload.journal") - - // Char-count cap (a byte-accurate split could cut a surrogate pair). Single - // source of truth so the journaled body and the live-emitted body match. - static func capBody(_ body: String?) -> (String?, Bool) { - guard let body, body.count > maxBodyChars else { return (body, false) } - return (String(body.prefix(maxBodyChars)), true) - } - - // Computed once: creating the dir and re-setting the backup flag on every - // append/read/ack call is wasteful. - private static let dirURL: URL = { - let base = FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask)[0] - var dir = base.appendingPathComponent("RNFileUploaderEvents", isDirectory: true) - try? FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) - // Transient device-local state; keep it out of iCloud/iTunes backups. - var values = URLResourceValues() - values.isExcludedFromBackup = true - try? dir.setResourceValues(values) - return dir - }() - - static func append(_ event: JournaledEvent) { +/// The v9 journal entry, kept only for the one-time import as legacy rows. +struct JournaledEventV9: Codable, Equatable { + let eventId: String + let id: String + var type: String // completed | error | cancelled + let timestamp: Double + var responseCode: Int? + var errorKind: String? + var cancelReason: String? + var partIndex: Int? +} + +/// Durable record of terminal outcomes: `RNFileUploaderEvents/.json`, +/// one file per event, each written tmp + fsync + rename. Written BEFORE the +/// event is emitted and deleted only when JS acknowledges it, so an outcome +/// that fires while JS is dead survives to the next launch. +/// +/// Synchronous on a serial queue, not an actor: the delegate must journal an +/// outcome before it emits, and an actor would make that ordering async. +final class EventJournal { + static let shared = EventJournal( + root: FileIO.applicationSupport().appendingPathComponent("RNFileUploaderEvents", isDirectory: true)) + + /// The settled response body cap, in UTF-8 bytes. + static let maxBodyBytes = 1_048_576 + /// The attempt event body cap, in characters. + static let maxAttemptBodyChars = 4_096 + /// Runaway guard: past this count the oldest files a row does not name + /// are dropped. + let maxEntries: Int + + let root: URL + private let queue = DispatchQueue(label: "ai.openspace.rnbgupload.journal") + + init(root: URL, maxEntries: Int = 1000) { + self.root = root + self.maxEntries = maxEntries + FileIO.makeLocalDirectory(root) + } + + /// Returns false when the write failed. `keeping` holds the eventIds that + /// rows still name; the prune never deletes them, because an entry whose + /// outcome file is gone cannot be acked. + @discardableResult + func append(_ event: JournaledEvent, keeping: Set = []) -> Bool { queue.sync { - var e = event - let (body, truncated) = capBody(e.responseBody) - if truncated { - e.responseBody = body - e.responseBodyTruncated = true - } - // A journal write must never throw into the caller: the delegate calls this - // right after a completed upload, and a propagated failure could misfire the - // error path. Dropping one entry is the lesser evil. - guard let data = try? JSONEncoder().encode(e) else { return } do { - try data.write(to: dirURL.appendingPathComponent("\(e.eventId).json"), options: .atomic) + try write(event) } catch { NSLog("[RNFileUploader] journal append failed: \(error.localizedDescription)") - return + return false } - pruneToMax() + pruneToMax(keeping: keeping.union([event.eventId])) + return true } } - static func unacknowledged() -> [[String: Any]] { - unacknowledgedEntries().map { $0.bridged } + func load(_ eventId: String) -> JournaledEvent? { + queue.sync { read(url(eventId)) } } - static func unacknowledgedEntries() -> [JournaledEvent] { + /// deliveries += 1 on each event, persisted. Returns the updated events in + /// the order of `ids`; unknown ids are skipped. Every emit and every + /// getUnacknowledgedEvents goes through this. + func markDelivered(_ ids: [String]) -> [JournaledEvent] { queue.sync { - let files = (try? FileManager.default.contentsOfDirectory(at: dirURL, includingPropertiesForKeys: nil)) ?? [] - return files - .filter { $0.pathExtension == "json" } - .compactMap { url -> JournaledEvent? in - guard let data = try? Data(contentsOf: url) else { return nil } - return try? JSONDecoder().decode(JournaledEvent.self, from: data) + ids.compactMap { id in + guard var e = read(url(id)) else { return nil } + e.deliveries += 1 + do { + try write(e) + } catch { + NSLog("[RNFileUploader] journal update failed: \(error.localizedDescription)") } - .sorted { $0.timestamp < $1.timestamp } + return e + } } } - static func ack(_ eventIds: [String]) { + /// Every v10 event, oldest first. v9-shaped files are left for the import. + func unacknowledged() -> [JournaledEvent] { + queue.sync { jsonFiles().compactMap(read).sorted { ($0.at, $0.eventId) < ($1.at, $1.eventId) } } + } + + func unacknowledgedForId(_ id: String) -> [JournaledEvent] { + unacknowledged().filter { $0.id == id } + } + + /// Deletes the files. Unknown ids are ignored, so ack is idempotent. + func ack(_ ids: [String]) { + queue.sync { + for id in ids { try? FileManager.default.removeItem(at: url(id)) } + } + } + + /// cancel() on a settled entry: its unacked outcomes go with it. Throws at + /// the first file that cannot be deleted; a file already gone is not an + /// error. + func removeForId(_ id: String) throws { + let ids = unacknowledgedForId(id).map(\.eventId) + try queue.sync { + for eventId in ids { + let file = url(eventId) + guard FileIO.exists(file) else { continue } + try FileManager.default.removeItem(at: file) + } + } + } + + /// v9 entries: files that decode as the v9 shape (have `type` and + /// `timestamp`, no `kind`). + func legacyEvents() -> [JournaledEventV9] { queue.sync { - for id in eventIds { - try? FileManager.default.removeItem(at: dirURL.appendingPathComponent("\(id).json")) + jsonFiles().compactMap { file -> JournaledEventV9? in + guard read(file) == nil, let data = try? Data(contentsOf: file) else { return nil } + return try? JSONDecoder().decode(JournaledEventV9.self, from: data) } } } - // Runaway guard: assumes JS drains via ack on each boot, but bounds the - // directory if that loop breaks or hasn't been adopted. Drops the oldest by - // file modification time (no parsing). Caller already holds `queue`. - private static func pruneToMax() { + func removeLegacy(_ eventId: String) { + queue.sync { _ = try? FileManager.default.removeItem(at: url(eventId)) } + } + + /// Decodes a response body capped at `cap` bytes. A cut that splits a + /// UTF-8 sequence backs off to the last whole character. + static func decodeBody(_ data: Data, cap: Int = maxBodyBytes, truncated: Bool = false) + -> (body: String, truncated: Bool) { + guard data.count > cap || truncated else { + return (String(decoding: data, as: UTF8.self), false) + } + var cut = data.prefix(cap) + for _ in 0..<4 { + if let s = String(data: cut, encoding: .utf8) { return (s, true) } + cut = cut.dropLast() + } + return (String(decoding: data.prefix(cap), as: UTF8.self), true) + } + + /// A character cap, for the 4 KB attempt body. + static func capChars(_ s: String?, _ max: Int) -> (String?, Bool) { + guard let s, s.count > max else { return (s, false) } + return (String(s.prefix(max)), true) + } + + // MARK: - Private (callers hold `queue`) + + private func url(_ eventId: String) -> URL { + // eventId is a UUID the library minted, but keep a hostile one inside root. + root.appendingPathComponent(eventId.replacingOccurrences(of: "/", with: "_") + ".json") + } + + private func write(_ event: JournaledEvent) throws { + try FileIO.writeAtomically(try JSONEncoder().encode(event), to: url(event.eventId)) + } + + private func read(_ file: URL) -> JournaledEvent? { + guard let data = try? Data(contentsOf: file) else { return nil } + return try? JSONDecoder().decode(JournaledEvent.self, from: data) + } + + private func jsonFiles() -> [URL] { + ((try? FileManager.default.contentsOfDirectory(at: root, includingPropertiesForKeys: nil)) ?? []) + .filter { $0.pathExtension == "json" } + } + + // Drops the oldest unnamed files by modification time, without parsing. + // When rows name more than maxEntries events, the count stays above it. + private func pruneToMax(keeping: Set) { let key: URLResourceKey = .contentModificationDateKey guard let files = try? FileManager.default.contentsOfDirectory( - at: dirURL, includingPropertiesForKeys: [key]) else { return } + at: root, includingPropertiesForKeys: [key]) else { return } let jsons = files.filter { $0.pathExtension == "json" } guard jsons.count > maxEntries else { return } - let sorted = jsons.sorted { + let kept = Set(keeping.map { url($0).lastPathComponent }) + let candidates = jsons.filter { !kept.contains($0.lastPathComponent) }.sorted { let a = (try? $0.resourceValues(forKeys: [key]).contentModificationDate) ?? .distantPast let b = (try? $1.resourceValues(forKeys: [key]).contentModificationDate) ?? .distantPast return a < b } - for f in sorted.prefix(jsons.count - maxEntries) { + for f in candidates.prefix(jsons.count - maxEntries) { try? FileManager.default.removeItem(at: f) } } diff --git a/ios/Events.swift b/ios/Events.swift new file mode 100644 index 00000000..fbc45921 --- /dev/null +++ b/ios/Events.swift @@ -0,0 +1,51 @@ +import Foundation + +/// Builds the live `attempt` event: one HTTP attempt before the library +/// interprets it for retry. `outcome` is 'completed' for a 2xx or a matching +/// accept rule. Any other response is 'error' with errorKind 'http'. A +/// transport failure is 'error' with its kind. A cancel of any kind (pause, +/// cancel, supersede, the system) is not an attempt: the caller emits none. +enum AttemptEvent { + struct Input { + var id: String + var key: String + var requestId: String + var attempt: Int + var url: String + var method: String + var partIndex: Int? + var statusCode: Int? + var headers: [String: String] + var body: String? + var error: NSError? + var accepted: Bool + var at: Double + } + + static func build(_ i: Input) -> [String: Any] { + var m: [String: Any] = [ + "id": i.id, "key": i.key, "requestId": i.requestId, "attempt": i.attempt, + "url": i.url, "method": i.method, "at": i.at, + ] + if let partIndex = i.partIndex { m["partIndex"] = partIndex } + if let code = i.statusCode, i.error == nil { + m["httpCode"] = code + m["responseHeaders"] = i.headers + let (body, truncated) = EventJournal.capChars(i.body ?? "", EventJournal.maxAttemptBodyChars) + m["responseBody"] = body ?? "" + m["responseBodyTruncated"] = truncated + } + if let error = i.error { + m["outcome"] = "error" + m["errorKind"] = RetryClassifier.errorKind(for: error) + m["errorMessage"] = error.localizedDescription + } else if i.accepted { + m["outcome"] = "completed" + } else { + m["outcome"] = "error" + m["errorKind"] = "http" + m["errorMessage"] = "HTTP \(i.statusCode ?? 0)" + } + return m + } +} diff --git a/ios/FileIO.swift b/ios/FileIO.swift new file mode 100644 index 00000000..e41690ec --- /dev/null +++ b/ios/FileIO.swift @@ -0,0 +1,65 @@ +import Foundation + +// Small file helpers shared by the store, the journal, the task map and body +// staging. Every durable write is tmp + fsync + rename, so a reader never sees +// a half-written file: after a crash there is either the old file or the new +// one, plus at most a stray `.tmp` that the next write replaces. +enum FileIO { + struct IOError: LocalizedError { + let message: String + var errorDescription: String? { message } + } + + static func tmpURL(for url: URL) -> URL { + url.deletingLastPathComponent().appendingPathComponent(url.lastPathComponent + ".tmp") + } + + /// Writes `data` to `url.tmp`, flushes it to disk, then renames it onto `url`. + static func writeAtomically(_ data: Data, to url: URL) throws { + let tmp = tmpURL(for: url) + try writeSynced(data, to: tmp) + try rename(tmp, onto: url) + } + + /// Writes and fsyncs a file in place. Callers rename it afterwards. + static func writeSynced(_ data: Data, to url: URL) throws { + let fm = FileManager.default + try? fm.removeItem(at: url) + guard fm.createFile(atPath: url.path, contents: nil) else { + throw IOError(message: "cannot create \(url.lastPathComponent)") + } + let handle = try FileHandle(forWritingTo: url) + defer { try? handle.close() } + try handle.write(contentsOf: data) + try handle.synchronize() + } + + /// POSIX rename: atomic, and it replaces an existing destination. + static func rename(_ from: URL, onto to: URL) throws { + guard Foundation.rename(from.path, to.path) == 0 else { + throw IOError(message: "rename \(from.lastPathComponent) -> \(to.lastPathComponent) failed: errno \(errno)") + } + } + + static func size(_ url: URL) -> Int64? { + ((try? FileManager.default.attributesOfItem(atPath: url.path))?[.size] as? NSNumber)?.int64Value + } + + static func exists(_ url: URL) -> Bool { + FileManager.default.fileExists(atPath: url.path) + } + + /// Creates a directory that holds device-local upload state and keeps it + /// out of iCloud and iTunes backups. + static func makeLocalDirectory(_ url: URL) { + try? FileManager.default.createDirectory(at: url, withIntermediateDirectories: true) + var dir = url + var values = URLResourceValues() + values.isExcludedFromBackup = true + try? dir.setResourceValues(values) + } + + static func applicationSupport() -> URL { + FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask)[0] + } +} diff --git a/ios/JSONText.swift b/ios/JSONText.swift new file mode 100644 index 00000000..a6624d81 --- /dev/null +++ b/ios/JSONText.swift @@ -0,0 +1,37 @@ +import CryptoKit +import Foundation + +// JSON text for values that come over the bridge (NSDictionary, NSArray, +// NSString, NSNumber, NSNull). `vars`, the descriptor and a `data` body are +// persisted as text and decoded again when a row or an event is built. +enum JSONText { + /// Canonical JSON text: keys sorted, fragments allowed. The same value with + /// its keys in another order gives the same text. Nil when the value is not + /// JSON (a NaN, a non-string key). The check runs first because + /// JSONSerialization raises an Obj-C exception, not a Swift error, on an + /// invalid object. + static func encode(_ value: Any?) -> String? { + let v: Any = value ?? NSNull() + guard JSONSerialization.isValidJSONObject([v]), + let data = try? JSONSerialization.data( + withJSONObject: v, options: [.sortedKeys, .fragmentsAllowed, .withoutEscapingSlashes]) + else { return nil } + return String(data: data, encoding: .utf8) + } + + /// The bridged object for stored text. NSNull (JS null) when the text is + /// "null" or does not parse. + static func decode(_ text: String) -> Any { + parse(text) ?? NSNull() + } + + /// The parsed value, or nil when the text is not JSON. "null" parses to + /// NSNull, which is a value. + static func parse(_ text: String) -> Any? { + try? JSONSerialization.jsonObject(with: Data(text.utf8), options: [.fragmentsAllowed]) + } + + static func sha256(_ text: String) -> String { + SHA256.hash(data: Data(text.utf8)).map { String(format: "%02x", $0) }.joined() + } +} diff --git a/ios/LegacyImport.swift b/ios/LegacyImport.swift new file mode 100644 index 00000000..3003b22d --- /dev/null +++ b/ios/LegacyImport.swift @@ -0,0 +1,49 @@ +import Foundation + +/// The pure half of the first-launch v9 import: which legacy rows to create. +/// +/// Each v9 journal entry becomes a read-only settled row with key "legacy" +/// and id = the v9 upload id. Nothing is delivered: Diana reads the rows, +/// marks those transfers terminal, and cancels them. A legacy row reports +/// 0/0 bytes, as on Android. When the id also has a v9 chunked manifest, the +/// blob stays in the directory until cancel(id) or a same-id enqueue. +/// +/// A manifest with no journal entry makes no row. It stays dormant until a +/// same-id enqueue adopts it (a legacy row would be cancelled by Diana, and +/// the capture re-send needs the bytes). +enum LegacyImport { + static let key = "legacy" + static let fingerprint = "legacy" + + static func plan(events: [JournaledEventV9], manifests: [String: ChunkedManifestV9]) -> [QueueEntry] { + // One row per id: the latest v9 outcome wins. + var latest: [String: JournaledEventV9] = [:] + for e in events where latest[e.id].map({ $0.timestamp <= e.timestamp }) ?? true { + latest[e.id] = e + } + return latest.values.sorted { ($0.timestamp, $0.id) < ($1.timestamp, $1.id) }.compactMap { e in + guard let state = state(e.type) else { return nil } + let manifest = manifests[e.id] + return QueueEntry( + id: e.id, key: key, varsJSON: "null", url: nil, method: "POST", + accept: [], retry: nil, bodyKind: .none, + bodyPath: manifest == nil ? nil : ChunkedManifestV9.blobName, + bodyContentType: nil, forceContentType: false, bodyFingerprint: fingerprint, parts: [], + incarnation: manifest?.incarnation ?? UUID().uuidString, headers: [:], headerGeneration: 0, + state: state, authParked: false, generation: 1, attempts: 0, + bytesSent: 0, totalBytes: 0, + expiresAt: manifest?.expiresAt ?? e.timestamp, nextAttemptAt: nil, settledEventId: nil, + lastRequestId: nil, lastUrl: nil, lastPartIndex: e.partIndex, legacy: true, + createdAt: e.timestamp, updatedAt: e.timestamp) + } + } + + static func state(_ v9Type: String) -> QueueEntry.State? { + switch v9Type { + case "completed": return .completed + case "error": return .error + case "cancelled": return .cancelled + default: return nil + } + } +} diff --git a/ios/Package.swift b/ios/Package.swift new file mode 100644 index 00000000..ab3711e2 --- /dev/null +++ b/ios/Package.swift @@ -0,0 +1,46 @@ +// swift-tools-version:5.9 +// Host-side unit tests for the pure half of the iOS module: `cd ios && swift test`. +// The CocoaPods build ignores this file (see exclude_files in the podspec). +// RNBackgroundUpload.swift and the .mm import React and UIKit, so they are +// not part of this package. Neither is the Obj-C exception helper that +// RNBackgroundUpload.swift calls; the tests fake the transport instead. +import PackageDescription + +let package = Package( + name: "RNBGUCore", + platforms: [.macOS(.v12)], + targets: [ + .target( + name: "RNBGUCore", + path: ".", + exclude: ["Tests", "RNBackgroundUpload.swift", "RNFileUploader.h", "RNFileUploader.mm", + "RNBGUCatchException.h", "RNBGUCatchException.m", ".gitignore"], + sources: [ + "BodyStaging.swift", + "ChunkedCoordinator.swift", + "ChunkedEngine.swift", + "ChunkedManifestV9.swift", + "EnqueueParser.swift", + "EventJournal.swift", + "Events.swift", + "FileIO.swift", + "JSONText.swift", + "LegacyImport.swift", + "ProgressThrottle.swift", + "QueueCoordinator.swift", + "QueueCoordinator+Enqueue.swift", + "QueueCoordinator+Outcomes.swift", + "QueueCoordinator+Reconcile.swift", + "QueueCoordinator+Simple.swift", + "QueueEntry.swift", + "QueueSettings.swift", + "QueueStore.swift", + "RequestIndex.swift", + "RetryClassifier.swift", + "TaskMap.swift", + "Transport.swift", + "UploadOutcome.swift", + ]), + .testTarget(name: "RNBGUCoreTests", dependencies: ["RNBGUCore"], path: "Tests"), + ] +) diff --git a/ios/ProgressThrottle.swift b/ios/ProgressThrottle.swift new file mode 100644 index 00000000..b024ba8a --- /dev/null +++ b/ios/ProgressThrottle.swift @@ -0,0 +1,39 @@ +import Foundation + +/// Per-id progress throttle: one event per second while the app is in the +/// foreground, one per 10 minutes in the background. settle() forces a +/// trailing edge past it. It runs on the URLSession delegate queue, before +/// the hop onto the coordinator queue, so a chatty task never floods that +/// queue. Lock-guarded. +final class ProgressThrottle { + static let foregroundMs = 1_000.0 + static let backgroundMs = 600_000.0 + + private let lock = NSLock() + private var last: [String: Double] = [:] + private var foreground = false + + /// Set from the UIApplication notifications. Reading applicationState + /// needs the main thread; the delegate queue is not it. + var isForeground: Bool { + get { lock.lock(); defer { lock.unlock() }; return foreground } + set { lock.lock(); foreground = newValue; lock.unlock() } + } + + /// true when an event for `id` may go now; records it. + func shouldEmit(_ id: String, now: Double) -> Bool { + lock.lock() + defer { lock.unlock() } + let interval = foreground ? Self.foregroundMs : Self.backgroundMs + if let t = last[id], now - t < interval { return false } + last[id] = now + return true + } + + /// The next event for `id` passes. Called at issue and at settle. + func reset(_ id: String) { + lock.lock() + last[id] = nil + lock.unlock() + } +} diff --git a/ios/QueueCoordinator+Enqueue.swift b/ios/QueueCoordinator+Enqueue.swift new file mode 100644 index 00000000..3a852eaf --- /dev/null +++ b/ios/QueueCoordinator+Enqueue.swift @@ -0,0 +1,192 @@ +import Foundation + +// enqueue() and the same-id rules (spec 5.4). Runs on the coordinator queue. +// Every reject path leaves the previous entry and its body as they were: +// a new body is staged under a fresh name, and the old one is deleted only +// after the new entry.json landed. +extension QueueCoordinator { + + /// Returns the id, and whether to issue it after the resolve. + func enqueueLocked(_ raw: [String: Any]) throws -> (id: String, issue: Bool) { + let p: ParsedEnqueue + do { + p = try EnqueueParser.parse(raw) + } catch { + throw EnqueueError.invalid("enqueue: \(error.localizedDescription)") + } + if let existing = index.entry(p.id) { + if existing.legacy { + // A legacy row over v9 chunked bytes: create() adopts the manifest + // (same parts resume their accepted parts) or keeps the blob. It + // writes entry.json over the legacy row. + if case .parts = p.body, store.loadV9Manifest(p.id) != nil { return try create(p) } + return try enqueueExisting(existing, p) + } + // A completed entry whose ack landed but whose forget did not (a crash + // between the two) is gone for JS. Start over. + if existing.state == .completed, settledEvent(existing) == nil { + forget(existing.id, dropEvents: true) + } else { + return try enqueueExisting(existing, p) + } + } + return try create(p) + } + + /// Rule 2: no entry has the id. + private func create(_ p: ParsedEnqueue) throws -> (id: String, issue: Bool) { + let dir = store.dir(p.id) + var fallback = store.adoptableBlob(p.id) + var adopted: ChunkedManifestV9? + if case .parts = p.body, let manifest = store.loadV9Manifest(p.id), + FileIO.exists(store.fileURL(p.id, ChunkedManifestV9.blobName)) { + fallback = ChunkedManifestV9.blobName + // A dormant v9 upload with the same parts resumes: its blob and its + // accepted parts are the body. The source path is ignored, as in v9. + if manifest.sameParts(as: p.parts) { adopted = manifest } + } + + let staged: StagedBody + if adopted != nil { + let size = FileIO.size(store.fileURL(p.id, ChunkedManifestV9.blobName)) ?? 0 + try mapStaging { try BodyStaging.requireTiling(p.parts, size: size) } + staged = StagedBody(kind: .parts, relativePath: ChunkedManifestV9.blobName, contentType: nil, + forceContentType: false, totalBytes: size, adopted: true) + } else { + staged = try mapStaging { + try BodyStaging.stage(p.body, parts: p.parts, into: dir, fallbackBlob: fallback) + } + } + + var e = QueueEntry.created(from: p, staged: staged, headerGeneration: settings.headerGeneration, + paused: settings.paused, now: now()) + if let manifest = adopted { + e.incarnation = manifest.incarnation + for i in e.parts.indices { e.parts[i].accepted = manifest.parts[i].accepted } + e.bytesSent = e.acceptedBytes + } + try saveOrDiscard(e, staged: staged) + store.removeV9Manifest(p.id) + store.sweep(e) + publish(e) + armExpiry(e) + return (p.id, !settings.paused) + } + + private func enqueueExisting(_ existing: QueueEntry, _ p: ParsedEnqueue) throws + -> (id: String, issue: Bool) { + let paused = settings.paused + if existing.bodyFingerprint == p.fingerprint && !existing.legacy { + switch existing.state { + case .completed: + // Rule 7: re-emit the journaled outcome. Do not run again. With no + // listener yet, the drain delivers it. + if sink?.canDeliver() == true, let eventId = existing.settledEventId, + let event = redeliver(eventId) { + sink?.emitSettled(event.bridged) + } + return (p.id, false) + + case .error, .cancelled: + // Rule 3 on a settled entry, and rule 6 for a cancelled one: reopen + // under a fresh generation. The old outcome's ack no longer forgets it. + var n = existing.resumed(with: p, resetBudget: true, now: now()) + n.generation += 1 + n.state = paused ? .paused : .queued + n.settledEventId = nil + n.authParked = false + n.nextAttemptAt = nil + n.bytesSent = n.isChunked ? n.acceptedBytes : 0 + try saveOrThrow(n) + publish(n) + armExpiry(n) + return (p.id, !paused) + + case .awaitingAuth: + // Fresh headers came with the call: leave the parking spot. Same + // generation, so attempts keep counting. + var n = existing.resumed(with: p, resetBudget: false, now: now()) + n.authParked = false + n.state = paused ? .paused : .queued + try saveOrThrow(n) + publish(n) + armExpiry(n) + return (p.id, !paused) + + case .queued, .running, .paused: + // The in-flight task keeps its request. A retry waiting in the daemon + // picks up the new headers in willBeginDelayedRequest. + let n = existing.resumed(with: p, resetBudget: false, now: now()) + try saveOrThrow(n) + publish(n) + armExpiry(n) + if ready, !paused, n.state == .queued, !n.isChunked, let at = n.nextAttemptAt, at > now() { + // A simple retry waiting out its backoff: the caller asks again, + // so retry now. The waiting attempt never ran: keep its ordinal. + cancelTasks(n.id, purpose: .superseded) + issue(n.id, delayMs: nil, advanceAttempt: false) + } + return (p.id, false) + } + } + + // Rules 4 and 5: a different body. + guard existing.state != .running else { throw EnqueueError.running(p.id) } + // A delayed retry task may wait in the daemon. Mark it superseded first, + // so its NSURLErrorCancelled is dropped. + cancelTasks(existing.id, purpose: .superseded) + chunked.stop(existing.id) + // A chunked replace may keep the current blob when the caller already + // deleted its source (the part-404 recreate over moved bytes). + var fallback: String? + if let path = existing.bodyPath, existing.isChunked || existing.legacy, + path == ChunkedManifestV9.blobName || path.hasPrefix(BodyStaging.blobPrefix) { + fallback = path + } + let staged = try mapStaging { + try BodyStaging.stage(p.body, parts: p.parts, into: store.dir(p.id), fallbackBlob: fallback) + } + var n = existing.replaced(with: p, staged: staged, now: now()) + n.headerGeneration = settings.headerGeneration + n.state = paused ? .paused : .queued + try saveOrDiscard(n, staged: staged) + store.removeV9Manifest(p.id) + store.sweep(n) // deletes the old body + publish(n) + armExpiry(n) + return (p.id, !paused) + } + + // MARK: - Helpers + + private func saveOrThrow(_ e: QueueEntry) throws { + do { + try store.save(e) + } catch { + throw EnqueueError.storage("enqueue: cannot save '\(e.id)': \(error.localizedDescription)") + } + } + + /// A failed save deletes the body staged for it, unless that body is an + /// adopted file the store already owned. + private func saveOrDiscard(_ e: QueueEntry, staged: StagedBody) throws { + do { + try store.save(e) + } catch { + if !staged.adopted { try? FileManager.default.removeItem(at: store.fileURL(e.id, staged.relativePath)) } + throw EnqueueError.storage("enqueue: cannot save '\(e.id)': \(error.localizedDescription)") + } + } + + private func mapStaging(_ body: () throws -> T) throws -> T { + do { + return try body() + } catch StagingError.fileMissing(let path) { + throw EnqueueError.fileMissing(path) + } catch StagingError.invalid(let message) { + throw EnqueueError.invalid("enqueue: \(message)") + } catch StagingError.io(let message) { + throw EnqueueError.storage("enqueue: \(message)") + } + } +} diff --git a/ios/QueueCoordinator+Outcomes.swift b/ios/QueueCoordinator+Outcomes.swift new file mode 100644 index 00000000..9684185f --- /dev/null +++ b/ios/QueueCoordinator+Outcomes.swift @@ -0,0 +1,264 @@ +import Foundation + +// Terminal outcomes: settle, ack, forget, and the repair paths for a settled +// row whose journal file is missing. Runs on the coordinator queue. +extension QueueCoordinator { + /// First wait before a failed journal write is tried again. It doubles up + /// to `journalRetryMaxMs`. + static let journalRetryMs = 5_000 + static let journalRetryMaxMs = 600_000 + + /// The one terminal path. Journals the outcome, cancels the entry's + /// remaining tasks, emits the trailing progress, saves the settled row, + /// emits `state`, then emits `settled` with deliveries 1 when a listener + /// exists. With no listener the outcome stays at deliveries 0 for the + /// drain. + /// + /// `requireJournal` (a user cancel): a failed journal write returns false + /// and changes nothing. No task stops, no row moves, nothing is emitted, + /// and nothing is kept in memory, so the caller can reject and JS can call + /// again. + /// + /// Otherwise a failed journal write still settles and emits: the request + /// already ran, so a retry would send it twice. The event stays in memory, + /// a timer retries the write, and ack finds the row by its settledEventId. + @discardableResult + func settle(_ id: String, _ outcome: Outcome, requireJournal: Bool = false) -> Bool { + guard var e = index.entry(id) else { return true } + switch outcome { + case .completed: e.bytesSent = e.totalBytes + // A simple entry keeps the live bytes of its last attempt. + default: if e.isChunked { e.bytesSent = e.acceptedBytes } + } + + var event = JournaledEvent( + eventId: UUID().uuidString, id: id, key: e.key, varsJSON: e.varsJSON, at: now(), + attempts: e.attempts, requestId: e.lastRequestId, deliveries: 0, bytesSent: e.bytesSent, + totalBytes: e.totalBytes, url: e.targetURL, method: e.method, partIndex: e.lastPartIndex, + generation: e.generation, kind: .completed) + switch outcome { + case .completed(let response): + event.kind = .completed + event.response = response + e.state = .completed + case .error(let error): + event.kind = .error + event.error = error + event.partIndex = error.partIndex ?? e.lastPartIndex + e.state = .error + case .cancelled(let reason): + event.kind = .cancelled + event.cancelReason = reason + event.partIndex = nil + e.state = .cancelled + } + let journaled = journal.append(event, keeping: referencedEventIds().union([event.eventId])) + if !journaled && requireJournal { return false } + cancelTasks(id, purpose: .superseded) + chunked.stop(id) + emitProgress(id, sent: e.bytesSent, total: e.totalBytes) + e.settledEventId = event.eventId + e.nextAttemptAt = nil + e.authParked = false + commit(e) + // Checked after the append. A drain that ran before this line already + // set the flag; one that runs after reads the journal and counts it. + let live = sink?.canDeliver() == true + if journaled { + if live { + var delivered = journal.markDelivered([event.eventId]).first ?? event + delivered.deliveries = max(delivered.deliveries, 1) + sink?.emitSettled(delivered.bridged) + } + } else { + event.deliveries = live ? 1 : 0 + pendingJournal[event.eventId] = event + retryJournal(event.eventId, delayMs: Self.journalRetryMs) + if live { sink?.emitSettled(event.bridged) } + } + disarmExpiry(id) + throttle.reset(id) + return true + } + + /// A settle whose journal write landed but whose entry.json save failed + /// leaves a live row on disk for an outcome that already happened. This + /// moves the row to that outcome, with no emit: the journal drain + /// delivers it. Returns the settled row, or nil when `event` is not the + /// outcome of this row's generation. + @discardableResult + func applyJournaled(_ event: JournaledEvent, to e: QueueEntry) -> QueueEntry? { + guard e.isLive, !e.legacy, event.id == e.id, event.generation == e.generation else { return nil } + var n = e + switch event.kind { + case .completed: n.state = .completed + case .error: n.state = .error + case .cancelled: n.state = .cancelled + } + n.settledEventId = event.eventId + n.bytesSent = event.bytesSent + n.nextAttemptAt = nil + n.authParked = false + cancelTasks(e.id, purpose: .superseded) + chunked.stop(e.id) + disarmExpiry(e.id) + commit(n, emit: false) + return n + } + + /// Relaunch, before any task is matched: every live row whose own + /// generation already has an unacked outcome takes that outcome, so it is + /// never sent again. + func repairLostSettles() { + let byId = Dictionary(grouping: journal.unacknowledged(), by: \.id) + for e in index.entries() where e.isLive && !e.legacy { + guard let event = byId[e.id]?.last(where: { $0.generation == e.generation }) else { continue } + applyJournaled(event, to: e) + } + } + + /// Deletes the row and the bytes. `dropEvents` also deletes the id's + /// unacked outcomes (cancel on a settled entry). + func forget(_ id: String, dropEvents: Bool) { + cancelTasks(id, purpose: .superseded) + chunked.stop(id) + store.remove(id) + index.remove(id) + if dropEvents { + try? journal.removeForId(id) + pendingJournal = pendingJournal.filter { $0.value.id != id } + } + disarmExpiry(id) + throttle.reset(id) + } + + /// cancel() on a settled or legacy entry: the row, its bytes and its + /// unacked outcomes, all or none. The directory is set aside first (one + /// rename), then the outcome files are deleted. When a delete fails, the + /// directory goes back and this throws; nothing in memory changed. Limit: + /// with two or more outcome files, a failure after the first delete leaves + /// the ones already deleted gone. + func forgetWithEvents(_ id: String) throws { + let aside = try store.setAside(id) + do { + try journal.removeForId(id) + } catch { + if let aside { + do { + try store.restore(aside, id) + } catch let restore { + NSLog("[RNFileUploader] cannot restore \(id) after a failed cancel: \(restore.localizedDescription)") + } + } + throw error + } + pendingJournal = pendingJournal.filter { $0.value.id != id } + cancelTasks(id, purpose: .superseded) + chunked.stop(id) + index.remove(id) + disarmExpiry(id) + throttle.reset(id) + if let aside { store.discard(aside) } + } + + /// Launch, before the index loads: a set-aside directory is a + /// `forgetWithEvents` that did not finish. It finishes it when the + /// outcome files can go, and puts the row back when they cannot. When the + /// id has a directory again, the forget already passed its journal step + /// (only the discard failed), so it only discards. + func finishSetAsideForgets() { + for (aside, id) in store.setAsideDirectories() { + guard let id, !FileIO.exists(store.dir(id)) else { + store.discard(aside) + continue + } + do { + try journal.removeForId(id) + store.discard(aside) + } catch { + try? store.restore(aside, id) + } + } + } + + /// One ack. The event comes from the journal, or from memory when its + /// write failed. When neither has it (pruned, or a crash after the file + /// went), the row that names the eventId still settles. + func ackLocked(_ eventId: String) { + let event = journal.load(eventId) ?? pendingJournal[eventId] + pendingJournal[eventId] = nil + journal.ack([eventId]) + var owner = event.flatMap { index.entry($0.id) } + ?? index.entries().first { $0.settledEventId == eventId } + // An ack can run before the relaunch repair: settle a live row first. + if let event, let e = owner, let settled = applyJournaled(event, to: e) { owner = settled } + guard let e = owner, e.isSettled, !e.legacy, e.state != .error else { return } + if let event { + guard event.kind != .error, event.generation == e.generation else { return } + } else { + guard e.settledEventId == eventId else { return } + } + forget(e.id, dropEvents: false) + } + + /// Every unacked outcome, oldest first, the in-memory ones included. Each + /// return counts as a delivery. + func unacknowledgedLocked() -> [JournaledEvent] { + let stored = journal.markDelivered(journal.unacknowledged().map(\.eventId)) + for (eventId, var event) in pendingJournal { + event.deliveries += 1 + pendingJournal[eventId] = event + } + return (stored + pendingJournal.values).sorted { ($0.at, $0.eventId) < ($1.at, $1.eventId) } + } + + /// A re-emit (same-id rule 7). Counts a delivery. + func redeliver(_ eventId: String) -> JournaledEvent? { + if let event = journal.markDelivered([eventId]).first { return event } + guard var event = pendingJournal[eventId] else { return nil } + event.deliveries += 1 + pendingJournal[eventId] = event + return event + } + + /// The settled outcome of `e`, from the journal or from memory. + func settledEvent(_ e: QueueEntry) -> JournaledEvent? { + e.settledEventId.flatMap { journal.load($0) ?? pendingJournal[$0] } + } + + /// Relaunch repair: a completed or cancelled row whose outcome file is + /// gone can never be acked. That happens after a crash between the ack's + /// delete and the forget, or when the journal write failed and the process + /// died. Forget the row and its bytes. An error row stays, as it does + /// after an ack, until cancel() or a same-id enqueue. + func sweepOrphanedOutcomes() { + for e in index.entries() where e.isSettled && !e.legacy && e.state != .error { + guard let eventId = e.settledEventId, pendingJournal[eventId] == nil, + journal.load(eventId) == nil else { continue } + forget(e.id, dropEvents: false) + } + } + + /// Every eventId a row still names. The journal never prunes these. + func referencedEventIds() -> Set { + Set(index.entries().compactMap(\.settledEventId)) + } + + /// Tries a failed journal write again, with doubling waits, while the + /// entry still names the event. An outcome the entry no longer names (an + /// ack, a cancel, a reopen) is dropped from memory. + private func retryJournal(_ eventId: String, delayMs: Int) { + schedule(delayMs) { [weak self] in + guard let self, let event = self.pendingJournal[eventId] else { return } + guard self.index.entry(event.id)?.settledEventId == eventId else { + self.pendingJournal[eventId] = nil + return + } + if self.journal.append(event, keeping: self.referencedEventIds()) { + self.pendingJournal[eventId] = nil + } else { + self.retryJournal(eventId, delayMs: min(delayMs * 2, Self.journalRetryMaxMs)) + } + } + } +} diff --git a/ios/QueueCoordinator+Reconcile.swift b/ios/QueueCoordinator+Reconcile.swift new file mode 100644 index 00000000..206ed7b4 --- /dev/null +++ b/ios/QueueCoordinator+Reconcile.swift @@ -0,0 +1,184 @@ +import Foundation + +// Relaunch: matching the daemon's surviving tasks to the stored entries, and +// the one-time v9 import. Runs at `shared` init: an app launch, a JS reload, +// or the AppDelegate background wake. +extension QueueCoordinator { + /// How long a running entry with no live task waits for a completion the + /// daemon may still replay before it re-issues. + static let graceMs = 10_000 + + /// `completion` runs on the queue once every entry has its tasks again and + /// every grace wait has ended. The caller holds the background completion + /// handlers until then, so the system cannot suspend the app before the + /// refill enqueues new tasks, or in the middle of a grace wait. + func reconcileAll(completion: @escaping () -> Void) { + queue.async { + self.transport.allTasks { tasks in + self.queue.async { + self.reconcile(tasks) + self.whenGraceEnds(completion) + } + } + } + } + + /// One open grace wait: the TaskMap keys whose completion may still + /// replay, and what to do when the wait ends. + struct Grace { + let token: UUID + var keys: Set + let resolve: () -> Void + } + + /// Runs `block` now when no grace wait is open, or when the last one ends. + func whenGraceEnds(_ block: @escaping () -> Void) { + if graces.isEmpty { + block() + } else { + afterGrace.append(block) + } + } + + /// Opens a grace wait for `keys`. It ends when the completion of every key + /// was handled, or after `graceMs`, whichever comes first. Then `resolve` + /// runs. Does nothing when a wait with this name is open. + func openGrace(_ name: String, keys: Set, resolve: @escaping () -> Void) { + guard graces[name] == nil else { return } + let token = UUID() + graces[name] = Grace(token: token, keys: keys, resolve: resolve) + schedule(Self.graceMs) { [weak self] in self?.endGrace(name, token: token) } + } + + /// A completion was handled for `key`. A wait with no key left ends now, + /// so the background completion handler does not wait out the timer. + func replayHandled(_ key: String) { + for (name, grace) in graces where grace.keys.contains(key) { + graces[name]?.keys.remove(key) + if graces[name]?.keys.isEmpty == true { endGrace(name, token: grace.token) } + } + } + + /// Closes one wait, runs its resolve, and releases the held blocks when no + /// wait is open. The token stops the timer of a wait that already ended + /// from closing a newer wait with the same name. + private func endGrace(_ name: String, token: UUID) { + guard let grace = graces[name], grace.token == token else { return } + graces[name] = nil + grace.resolve() + guard graces.isEmpty else { return } + let blocks = afterGrace + afterGrace = [] + blocks.forEach { $0() } + } + + func reconcile(_ tasks: [UploadTask]) { + ready = true + repairLostSettles() + sweepOrphanedOutcomes() + var simpleLive: Set = [] + var partTasks: [String: [ChunkedCoordinator.LiveTask]] = [:] + let liveKeys = Set(tasks.filter(\.isLive).map(\.key)) + + for task in tasks where task.isLive { + let owner = TaskOwner.resolve(description: task.taskDescription, + meta: taskMap.meta(forKey: task.key)) + let entry = owner.flatMap { index.entry($0.id) } + let runnable = entry.map { !$0.legacy && ($0.state == .queued || $0.state == .running) } ?? false + switch owner { + case .part(let id, let part, let incarnation)? where runnable && entry?.isChunked == true: + partTasks[id, default: []].append(.init(task: task, part: part, incarnation: incarnation)) + case .request(let id, let generation, let attempt)? + where runnable && entry?.isChunked == false && entry?.generation == generation + && entry?.attempts == attempt && !simpleLive.contains(id): + simpleLive.insert(id) + liveTasks[task.key] = (id, task) + default: + // No v10 owner (a v9 task), an older generation or attempt, a + // duplicate, or an entry that must have no task. Drop its callback. + taskMap.setPurpose(.superseded, forKey: task.key, id: owner?.id ?? "") + task.cancel() + } + } + // A key whose id has no entry belongs to nothing. A live task keeps its + // key until its cancel callback, which reads the purpose. + taskMap.removeAll { key, meta in !liveKeys.contains(key) && index.entry(meta.id) == nil } + + for e in index.entries() where e.isLive && !e.legacy { + armExpiry(e) + guard e.state == .queued || e.state == .running else { continue } + if e.isChunked { + chunked.reconcile(e.id, tasks: partTasks[e.id] ?? []) + } else if !simpleLive.contains(e.id) { + withoutTask(e) + } + } + } + + /// A queued or running entry with no live task. Either the daemon finished + /// the task while we were dead and will replay its completion now, or the + /// task never reached the daemon (a crash between the save and resume), or + /// it was lost. The TaskMap tells them apart: its key goes when a + /// completion is handled. + private func withoutTask(_ e: QueueEntry) { + let pending = taskMap.keys(where: { + $0.id == e.id && $0.generation == e.generation && $0.attempt == e.attempts + }) + guard !pending.isEmpty else { + // No task was ever made for a waiting attempt: it never ran, so it + // keeps its ordinal and request id. + reissue(e, advanceAttempt: false) + return + } + openGrace(e.id, keys: Set(pending)) { [weak self] in + guard let self, let current = self.index.entry(e.id), current.state == e.state, + current.generation == e.generation, current.attempts == e.attempts, + !self.liveTasks.values.contains(where: { $0.id == e.id }) else { return } + // No replay moved the entry: the task is lost. Its key would send + // every later launch through this wait again. + self.taskMap.removeAll { _, m in + m.id == e.id && m.generation == e.generation && m.attempt == e.attempts + } + // It may have run, so the next attempt gets a new ordinal: a late + // replay of this one is then dropped as stale. + self.reissue(current, advanceAttempt: true) + } + } + + /// Issues again, keeping what is left of a wait. + private func reissue(_ e: QueueEntry, advanceAttempt: Bool) { + var n = e + n.state = .queued + index.upsert(n) + let remaining = e.nextAttemptAt.map { Int($0 - now()) }.flatMap { $0 > 0 ? $0 : nil } + issue(n.id, delayMs: remaining, advanceAttempt: advanceAttempt) + } + + /// First v10 launch, from init: disk only, before any session exists. + /// Each v9 journal entry becomes a legacy row; nothing is emitted. v9 + /// manifests with no journal entry stay dormant. v9 task metadata is + /// dropped; reconcile cancels the v9 tasks. The marker is written last, so + /// a crash mid-import runs it again. + func importLegacyIfNeeded() { + guard !store.isImported() else { return } + let events = journal.legacyEvents() + let manifests = store.allV9Manifests() + for e in LegacyImport.plan(events: events, manifests: manifests) { + if let existing = index.entry(e.id), !existing.legacy { continue } + do { + try store.save(e) + } catch { + NSLog("[RNFileUploader] v9 import: cannot save \(e.id): \(error.localizedDescription)") + return + } + index.upsert(e) + } + for event in events { journal.removeLegacy(event.eventId) } + taskMap.removeAll { _, meta in meta.generation == nil } + do { + try store.markImported() + } catch { + NSLog("[RNFileUploader] v9 import: cannot write the marker: \(error.localizedDescription)") + } + } +} diff --git a/ios/QueueCoordinator+Simple.swift b/ios/QueueCoordinator+Simple.swift new file mode 100644 index 00000000..17c6d35c --- /dev/null +++ b/ios/QueueCoordinator+Simple.swift @@ -0,0 +1,293 @@ +import Foundation + +// Simple (one-body) entries: one task per attempt, and the URLSession +// delegate hooks. A chunked entry routes to ChunkedCoordinator from each hook. +extension QueueCoordinator { + + /// Starts the next attempt of a queued entry. With `delayMs` the task is + /// created now with earliestBeginDate, so nsurlsessiond starts it on time + /// whether the app lives or not; the row stays queued with nextAttemptAt. + /// Write-ahead: the attempt ordinal and request id are saved before the + /// task exists. A failed save creates no task and tries again later. + /// + /// `advanceAttempt: false` re-creates a waiting attempt that never ran (a + /// session move, or a delayed task that never reached the daemon). It keeps + /// the ordinal and request id. It applies only while the entry holds such + /// an attempt (nextAttemptAt set); otherwise a new attempt is minted. + func issue(_ id: String, delayMs: Int? = nil, advanceAttempt: Bool = true) { + guard var e = index.entry(id), !e.legacy, e.state == .queued, !settings.paused else { return } + let t = now() + if t >= e.expiresAt { + settle(id, .expired) + return + } + guard ready else { + // Reconcile has not matched the daemon's tasks yet. Keep the queued + // state and the wait on disk; reconcile issues it. A wait follows an + // attempt that ran (a completion that landed before reconcile), so + // mint its successor now: reconcile keeps a waiting attempt's ordinal, + // and must not reuse the one that ran. + if let delayMs { + if advanceAttempt { + e.attempts += 1 + e.lastRequestId = UUID().uuidString + } + e.nextAttemptAt = t + Double(delayMs) + } + commit(e) + return + } + if e.isChunked { + chunked.start(id) + return + } + guard let body = store.bodyURL(e), FileIO.exists(body) else { + settle(id, .fileError("the staged body is missing")) + return + } + guard let urlString = e.url, let url = URL(string: urlString) else { + settle(id, .error(OutcomeErrorRecord(errorKind: "unknown", message: "the url is not valid"))) + return + } + + let before = e + let reuse = !advanceAttempt && e.nextAttemptAt != nil && e.attempts > 0 && e.lastRequestId != nil + let requestId = reuse ? e.lastRequestId! : UUID().uuidString + if !reuse { + e.attempts += 1 + e.bytesSent = 0 // a new attempt sends from byte 0 + } + e.lastRequestId = requestId + e.lastUrl = urlString + e.lastPartIndex = nil + let beginAt = delayMs.map { t + Double($0) } + if let beginAt { + e.nextAttemptAt = beginAt + } else { + e.state = .running + e.nextAttemptAt = nil + } + guard commitAhead(e) else { + deferIssue(before, delayMs: delayMs) + return + } + + let meta = TaskMap.Meta( + id: id, attempt: e.attempts, requestId: requestId, + headerGeneration: settings.headerGeneration, generation: e.generation, purpose: .attempt) + let task: UploadTask + do { + task = try transport.upload( + buildRequest(e, url: url, requestId: requestId), fromFile: body, wifiOnly: settings.wifiOnly, + description: ChunkedEngine.taskDescription(id: id, attempt: e.attempts, generation: e.generation), + beginAt: beginAt.map { Date(timeIntervalSince1970: $0 / 1000) }, + beforeResume: { key in self.taskMap.set(meta, forKey: key) }) + } catch { + // The session could not open the body. Gone since the check above: it + // can never send. Still there: it is unreadable for now (data + // protection while the device is locked), so try again later. + guard FileIO.exists(body) else { + settle(id, .fileError("cannot read the staged body: \(error.localizedDescription)")) + return + } + var waiting = e + waiting.state = .queued + waiting.nextAttemptAt = nil + commit(waiting) + deferIssue(waiting, delayMs: delayMs) + return + } + liveTasks[task.key] = (id, task) + throttle.reset(id) + } + + /// The attempt could not be written ahead (disk full, protected data). + /// The entry stays as the disk has it, queued in memory, with no task. + /// Issue again after the wait it asked for, or a backoff, whichever is + /// longer, unless something else moved the entry first. + private func deferIssue(_ e: QueueEntry, delayMs: Int?) { + let backoff = RetryClassifier.backoffMs(attempt: max(e.attempts, 1), policy: policy(e), random: random) + let generation = e.generation + let attempts = e.attempts + schedule(max(delayMs ?? 0, backoff)) { [weak self] in + guard let self, let current = self.index.entry(e.id), current.generation == generation, + current.attempts == attempts else { return } + self.issue(e.id) + } + } + + // MARK: - Delegate hooks + + /// didCompleteWithError. Synchronous, so the journal write for a terminal + /// lands before the delegate callback returns (a background wake may + /// suspend the app right after). + func taskCompleted(_ c: TaskCompletion) { + queue.sync { taskCompletedLocked(c) } + } + + func taskCompletedLocked(_ c: TaskCompletion) { + let meta = taskMap.meta(forKey: c.key) + taskMap.removeKey(c.key) + liveTasks[c.key] = nil + // Runs after the completion moved the entry, so a grace that ends here + // sees the new state. + defer { replayHandled(c.key) } + guard let owner = TaskOwner.resolve(description: c.description, meta: meta) else { return } + if case .part(let id, let part, let incarnation) = owner { + chunked.partCompleted(id: id, part: part, incarnation: incarnation, key: c.key, meta: meta, + completion: c) + return + } + // Only the current attempt of the current generation may drive the entry. + guard case .request(let id, let generation, let attempt) = owner, + var e = index.entry(id), !e.legacy, !e.isChunked, + e.generation == generation, e.attempts == attempt else { return } + + let cancelled = RetryClassifier.isCancellation(c.error) + if cancelled, meta?.purpose == .pause || meta?.purpose == .superseded { return } + let accepted = c.error == nil + && c.statusCode.map { UploadOutcome.isAccepted($0, body: c.body, accept: e.accept) } == true + // A replaced task that finished before its cancel took effect. Its + // replacement drives the entry, unless this one landed. + if meta?.purpose == .superseded && !accepted { return } + // A cancel the library did not ask for (the system, a force-quit) is not + // an attempt: no event. It retries below. + if !cancelled { + emitAttempt(e, requestId: meta?.requestId ?? e.lastRequestId, attempt: attempt, completion: c, + partIndex: nil, accepted: accepted) + } + + guard e.state == .running || e.state == .queued else { + // A pause raced this completion. An accepted response did land, so + // settle it: a resume must not send it twice. + if accepted && e.state == .paused { settle(id, .completed(response(c))) } + return + } + // A system cancel is a transient failure, never a 'cancelled' outcome. + if cancelled { + scheduleRetry(e) + return + } + let fileExists = store.bodyURL(e).map(FileIO.exists) ?? false + let verdict = RetryClassifier.classify(RetryClassifier.Input( + statusCode: c.statusCode, body: c.body, error: c.error, accept: e.accept, policy: policy(e), + fileExists: fileExists, now: now(), expiresAt: e.expiresAt)) + switch verdict { + case .accepted: + settle(id, .completed(response(c))) + case .transient: + scheduleRetry(e) + case .auth: + if let g = meta?.headerGeneration, g < settings.headerGeneration { + // Issued under older headers. updateHeaders() already merged the new + // ones into the entry, so re-issue at once. + e.state = .queued + index.upsert(e) + issue(id) + } else { + park(e) + } + case .terminalHttp: + settle(id, .error(OutcomeErrorRecord( + errorKind: "http", message: "HTTP \(c.statusCode ?? 0)", response: response(c)))) + case .fileMissing: + settle(id, .fileError("the staged body is missing")) + case .expired: + settle(id, .expired) + } + } + + /// willBeginDelayedRequest: a delayed retry is about to start while the + /// app is alive. Returns the request rebuilt from the entry's current + /// headers (an updateHeaders during the backoff reaches the retry), or nil + /// to cancel a task whose entry moved on. + func taskWillBegin(key: String, description: String?) -> URLRequest? { + queue.sync { + let meta = taskMap.meta(forKey: key) + guard let owner = TaskOwner.resolve(description: description, meta: meta) else { + taskMap.setPurpose(.superseded, forKey: key, id: "") + return nil + } + if case .part(let id, let part, let incarnation) = owner { + return chunked.partWillBegin(id: id, part: part, incarnation: incarnation, key: key, meta: meta) + } + // A task the library already replaced (a session move keeps the + // attempt ordinal, so the ordinal alone cannot tell them apart). + guard meta?.purpose != .superseded, + case .request(let id, let generation, let attempt) = owner, + var e = index.entry(id), !e.isChunked, e.generation == generation, + e.attempts == attempt, e.state == .queued || e.state == .running, !settings.paused, + let url = e.url.flatMap(URL.init(string:)) else { + taskMap.setPurpose(.superseded, forKey: key, id: owner.id) + liveTasks[key] = nil + return nil + } + if e.state == .queued { + e.state = .running + e.nextAttemptAt = nil + commit(e) + } + taskMap.setHeaderGeneration(settings.headerGeneration, forKey: key) + return buildRequest(e, url: url, requestId: meta?.requestId ?? e.lastRequestId ?? UUID().uuidString) + } + } + + /// didSendBodyData. The throttle runs here, on the delegate queue, before + /// the hop. The first event after an issue always passes, which also moves + /// a delayed retry that began while the app was dead to running. + func taskProgress(key: String, description: String?, sent: Int64, expected: Int64) { + let meta = taskMap.meta(forKey: key) + guard meta?.purpose != .superseded, let owner = TaskOwner.resolve(description: description, meta: meta), + throttle.shouldEmit(owner.id, now: now()) else { return } + queue.async { + switch owner { + case .part(let id, let part, let incarnation): + self.chunked.partProgress(id: id, part: part, incarnation: incarnation, sent: sent) + case .request(let id, let generation, let attempt): + guard var e = self.index.entry(id), e.generation == generation, e.attempts == attempt, + e.state == .queued || e.state == .running else { return } + if e.state == .queued { + e.state = .running + e.nextAttemptAt = nil + self.commit(e) + } + self.index.setBytes(id, sent) + self.emitProgress(id, sent: sent, total: expected > 0 ? expected : e.totalBytes) + } + } + } + + // MARK: - Retry and auth + + /// Backoff from the attempt count. A wait that would pass expiresAt + /// settles 'expired' now instead of scheduling. + func scheduleRetry(_ e: QueueEntry) { + let delay = RetryClassifier.backoffMs(attempt: max(e.attempts, 1), policy: policy(e), random: random) + if now() + Double(delay) >= e.expiresAt { + settle(e.id, .expired) + return + } + var n = e + n.state = .queued + index.upsert(n) + issue(n.id, delayMs: delay) + } + + /// awaiting-auth: no task, one `state` event. updateHeaders() resumes it. + func park(_ e: QueueEntry) { + cancelTasks(e.id, purpose: .superseded) + chunked.stop(e.id) + var n = e + n.state = .awaitingAuth + n.authParked = true + n.nextAttemptAt = nil + commit(n) + // The timer may have passed while the entry ran. + armExpiry(n) + } + + func response(_ c: TaskCompletion) -> RawResponseRecord { + RawResponseRecord(status: c.statusCode, headers: c.headers, body: c.body ?? "", + bodyTruncated: c.bodyTruncated) + } +} diff --git a/ios/QueueCoordinator.swift b/ios/QueueCoordinator.swift new file mode 100644 index 00000000..93360c71 --- /dev/null +++ b/ios/QueueCoordinator.swift @@ -0,0 +1,434 @@ +import Foundation + +/// A rejection that crosses to JS with a code. +struct EnqueueError: Error { + let code: String + let message: String + + static func storage(_ message: String) -> EnqueueError { EnqueueError(code: "E_STORAGE", message: message) } + /// Input native cannot send: a bad URL scheme, header, body kind or tiling. + static func invalid(_ message: String) -> EnqueueError { EnqueueError(code: "E_INVALID", message: message) } + static func running(_ id: String) -> EnqueueError { + EnqueueError(code: "E_RUNNING", message: "enqueue: '\(id)' is running; a different body is accepted once it stops") + } + static func fileMissing(_ path: String) -> EnqueueError { + EnqueueError(code: "E_FILE_MISSING", message: "enqueue: file does not exist: \(path)") + } +} + +/// A terminal outcome, before it is journaled. +enum Outcome { + case completed(RawResponseRecord) + case error(OutcomeErrorRecord) + case cancelled(reason: String) + + static let expired = Outcome.error(OutcomeErrorRecord( + errorKind: "expired", message: "expiresAt passed before the request completed")) + + static func fileError(_ message: String, partIndex: Int? = nil) -> Outcome { + .error(OutcomeErrorRecord(errorKind: "file", message: message, partIndex: partIndex)) + } +} + +/// The queue's owner. It holds the store, the settings, the in-memory index, +/// the journal and the task map, schedules every entry, and makes every +/// emit. Every state change runs on one serial queue, which the chunked +/// coordinator shares. Rules it keeps: +/// - Write-ahead: an entry, its body and each attempt ordinal are on disk +/// before a task exists. +/// - Journal before emit: a terminal is a journal file before any emit. +/// - Store first, then index, then the `state` event. +/// - The module queue never waits on this queue, except the synchronous +/// delegate hops, which never wait on JS. +/// +/// The files next to this one extend it: enqueue and the same-id rules, +/// simple attempts, outcomes (settle, ack, forget), and relaunch +/// reconciliation. +final class QueueCoordinator { + static let queueLabel = "ai.openspace.rnbgupload.queue" + + let queue: DispatchQueue + let store: QueueStore + let journal: EventJournal + let taskMap: TaskMap + let transport: Transport + weak var sink: EventSink? + let index = RequestIndex() + let throttle = ProgressThrottle() + + let now: () -> Double + let random: () -> Double + /// Runs `block` on `queue` after `delayMs`. Injected so tests control time. + let schedule: (_ delayMs: Int, _ block: @escaping () -> Void) -> Void + + var settings: QueueSettings + /// false until the first reconcile has matched the daemon's tasks to the + /// entries. Until then nothing issues: a task made now could duplicate one + /// the daemon already holds. Reconcile issues whatever waited. + var ready = false + /// Every task this process created or adopted, by TaskMap key. + var liveTasks: [String: (id: String, task: UploadTask)] = [:] + var expiryTokens: [String: UUID] = [:] + /// Open grace waits (a completion that may still replay), by name. While + /// any is open, `afterGrace` holds the background completion handler + /// release. + var graces: [String: Grace] = [:] + var afterGrace: [() -> Void] = [] + /// Settle outcomes (never a user cancel) whose journal write failed, by + /// eventId. They were emitted live; a timer retries the write while the + /// entry still names them. + var pendingJournal: [String: JournaledEvent] = [:] + lazy var chunked = ChunkedCoordinator(self) + + init(store: QueueStore, journal: EventJournal, taskMap: TaskMap, transport: Transport, + sink: EventSink?, queue: DispatchQueue = DispatchQueue(label: QueueCoordinator.queueLabel), + now: @escaping () -> Double = { Date().timeIntervalSince1970 * 1000 }, + random: @escaping () -> Double = { Double.random(in: 0..<1) }, + schedule: ((Int, @escaping () -> Void) -> Void)? = nil) { + self.queue = queue + self.store = store + self.journal = journal + self.taskMap = taskMap + self.transport = transport + self.sink = sink + self.now = now + self.random = random + self.schedule = schedule ?? { ms, block in + queue.asyncAfter(deadline: .now() + .milliseconds(ms), execute: block) + } + settings = store.loadSettings() + finishSetAsideForgets() + // Synchronous, before any session exists, so getRequests() is warm by + // the time JS can call it, legacy rows included. + index.load(store.all()) + importLegacyIfNeeded() + } + + // MARK: - Module methods + + func configure(_ options: [String: Any]) { + queue.async { + var next = self.settings + next.apply(configure: options) + _ = self.saveSettings(next) + } + } + + func enqueue(_ raw: [String: Any], resolve: @escaping (String) -> Void, + reject: @escaping (String, String) -> Void) { + queue.async { + let result: (id: String, issue: Bool) + do { + result = try self.enqueueLocked(raw) + } catch let e as EnqueueError { + reject(e.code, e.message) + return + } catch { + reject("E_STORAGE", error.localizedDescription) + return + } + resolve(result.id) + if result.issue { self.issue(result.id) } + } + } + + /// Whole-queue pause. Cancels every task with purpose "pause", so its + /// NSURLErrorCancelled produces no outcome and no attempt event. A single + /// body restarts from byte 0 on resume; a chunked upload keeps its + /// accepted parts. + func pause(resolve: @escaping () -> Void, reject: @escaping (String, String) -> Void) { + queue.async { + var next = self.settings + next.paused = true + guard self.saveSettings(next) else { + reject("E_STORAGE", "pause: cannot save the queue settings") + return + } + for e in self.index.entries() where !e.legacy + && [.queued, .running, .awaitingAuth].contains(e.state) { + self.cancelTasks(e.id, purpose: .pause) + self.chunked.stop(e.id) + var n = e + n.state = .paused + n.nextAttemptAt = nil + self.commit(n) + } + resolve() + } + } + + func resume(resolve: @escaping () -> Void, reject: @escaping (String, String) -> Void) { + queue.async { + var next = self.settings + next.paused = false + guard self.saveSettings(next) else { + reject("E_STORAGE", "resume: cannot save the queue settings") + return + } + for e in self.index.entries() where e.state == .paused && !e.legacy { + // A pause does not expire an entry; the resume does. + if self.now() >= e.expiresAt { + self.settle(e.id, .expired) + continue + } + var n = e + n.state = e.authParked ? .awaitingAuth : .queued + self.commit(n) + if n.state == .queued { self.issue(n.id) } + } + resolve() + } + } + + /// Live entry: journal 'cancelled' (user); forgotten after its ack. + /// Settled entry: forgotten now, with its unacked outcomes. Unknown: no-op. + /// When the journal or the store cannot be written, rejects E_STORAGE and + /// changes nothing: the entry keeps running (or stays settled), and JS may + /// call again. + func cancel(_ id: String, resolve: @escaping () -> Void, reject: @escaping (String, String) -> Void) { + queue.async { + guard let e = self.index.entry(id) else { return resolve() } + if e.isLive && !e.legacy { + guard self.settle(id, .cancelled(reason: "user"), requireJournal: true) else { + return reject("E_STORAGE", "cancel: cannot journal the outcome of '\(id)'; nothing changed") + } + } else { + do { + try self.forgetWithEvents(id) + } catch { + return reject("E_STORAGE", "cancel: cannot delete '\(id)': \(error.localizedDescription)") + } + } + resolve() + } + } + + /// Persisted. Queued and future entries use the session it picks. A queued + /// entry whose retry waits in the daemon moves to the new session; a + /// running task finishes where it started (session config is fixed). + func setWifiOnly(_ enabled: Bool, resolve: @escaping () -> Void, + reject: @escaping (String, String) -> Void) { + queue.async { + var next = self.settings + let changed = next.wifiOnly != enabled + next.wifiOnly = enabled + guard self.saveSettings(next) else { + reject("E_STORAGE", "setWifiOnly: cannot save the queue settings") + return + } + if changed && self.ready { + for e in self.index.entries() where e.state == .queued && !e.isChunked && !e.legacy { + let remaining = e.nextAttemptAt.map { Int($0 - self.now()) } + self.cancelTasks(e.id, purpose: .superseded) + // The waiting attempt never ran: keep its ordinal and request id. + self.issue(e.id, delayMs: remaining.flatMap { $0 > 0 ? $0 : nil }, advanceAttempt: false) + } + } + resolve() + } + } + + /// Merges the patch into every entry not yet forgotten (names match + /// without regard to case) and bumps the header generation. Parked entries + /// go back to queued and issue. Resolves after the loop. + func updateHeaders(_ patch: [String: Any], resolve: @escaping () -> Void, + reject: @escaping (String, String) -> Void) { + queue.async { + let headers: [String: String] + do { + headers = try EnqueueParser.headers(patch, field: "patch") + } catch { + reject("E_INVALID", "updateHeaders: \(error.localizedDescription)") + return + } + var next = self.settings + next.headerGeneration += 1 + guard self.saveSettings(next) else { + reject("E_STORAGE", "updateHeaders: cannot save the queue settings") + return + } + for e in self.index.entries() where !e.legacy { + var n = e + n.headers = HeaderMerge.merge(e.headers, headers) + // A part's own header of the same name would win over the entry's. + for i in n.parts.indices { + n.parts[i].headers = HeaderMerge.replaceExisting(n.parts[i].headers, headers) + } + n.headerGeneration = self.settings.headerGeneration + if n.state == .awaitingAuth { + n.authParked = false + n.state = self.settings.paused ? .paused : .queued + self.commit(n) + self.issue(n.id) + } else { + n.authParked = false + self.commit(n, emit: false) + } + } + resolve() + } + } + + /// Synchronous: the in-memory index, under its lock. Called from the JS + /// thread; never touches this queue or the disk. + func rows() -> [[String: Any]] { + index.rows() + } + + /// Every unacked outcome, oldest first. Each return counts as a delivery. + func unacknowledgedEvents(resolve: @escaping ([[String: Any]]) -> Void) { + queue.async { + // JS drains after it attaches its listener. From here on, a settle is + // emitted live. + self.sink?.listenerReady() + resolve(self.unacknowledgedLocked().map(\.bridged)) + } + } + + /// Removes the outcomes. An acked 'completed' or 'cancelled' of the + /// entry's current generation forgets the entry: row and bytes. An 'error' + /// keeps both until cancel() or a same-id enqueue. Unknown ids are ignored. + func ack(_ eventIds: [String], resolve: @escaping () -> Void) { + queue.async { + for eventId in eventIds { self.ackLocked(eventId) } + resolve() + } + } + + // MARK: - Transitions (on `queue`) + + /// Store, then index, then the `state` event. A failed save is logged and + /// the index still moves: the in-memory state is what runs, and the next + /// save of this entry writes it. Returns whether the save landed. + @discardableResult + func commit(_ entry: QueueEntry, emit: Bool = true) -> Bool { + let (e, saved) = save(entry) + publish(e, emit: emit) + return saved + } + + /// Write-ahead for an attempt. Publishes only when the save landed, so a + /// failed save leaves the index at what the disk holds. On false the caller + /// creates no task. + func commitAhead(_ entry: QueueEntry, emit: Bool = true) -> Bool { + let (e, saved) = save(entry) + if saved { publish(e, emit: emit) } + return saved + } + + private func save(_ entry: QueueEntry) -> (QueueEntry, Bool) { + var e = entry + e.updatedAt = now() + do { + try store.save(e) + return (e, true) + } catch { + NSLog("[RNFileUploader] cannot save entry \(e.id): \(error.localizedDescription)") + return (e, false) + } + } + + /// Index, then the `state` event, for an entry already saved. + func publish(_ entry: QueueEntry, emit: Bool = true) { + index.upsert(entry) + guard emit, let row = index.row(entry.id) else { return } + sink?.emitState(row) + } + + /// Records why, then cancels every task this process holds for `id`. + func cancelTasks(_ id: String, purpose: TaskMap.Purpose) { + for (key, owner) in liveTasks where owner.id == id { + cancelTask(key, purpose: purpose) + } + } + + /// One task, by TaskMap key. + func cancelTask(_ key: String, purpose: TaskMap.Purpose) { + guard let owner = liveTasks[key] else { return } + taskMap.setPurpose(purpose, forKey: key, id: owner.id) + owner.task.cancel() + liveTasks[key] = nil + } + + func emitProgress(_ id: String, sent: Int64, total: Int64) { + sink?.emitProgress(["id": id, "bytesSent": sent, "totalBytes": total]) + } + + func policy(_ e: QueueEntry) -> RetryPolicy { + RetryPolicy.resolve([settings.retry, e.retry]) + } + + /// The request for one attempt: the entry's method and headers, part + /// headers over them, the staged body's Content-Type when the headers set + /// none (a multipart body always uses its own), and a fresh X-Request-Id. + func buildRequest(_ e: QueueEntry, url: URL, requestId: String, + partHeaders: [String: String] = [:]) -> URLRequest { + var request = URLRequest(url: url) + request.httpMethod = e.method + let headers = HeaderMerge.merge(e.headers, partHeaders) + for (name, value) in headers { request.setValue(value, forHTTPHeaderField: name) } + if let contentType = e.bodyContentType, + e.forceContentType || HeaderMerge.value("Content-Type", in: headers) == nil { + request.setValue(contentType, forHTTPHeaderField: "Content-Type") + } + request.setValue(requestId, forHTTPHeaderField: "X-Request-Id") + return request + } + + /// One HTTP attempt that ended with a response or a transport error. A + /// cancel of any kind is not an attempt and never gets here. + func emitAttempt(_ e: QueueEntry, requestId: String?, attempt: Int, completion c: TaskCompletion, + partIndex: Int?, accepted: Bool) { + let url = c.url ?? partIndex.flatMap { e.parts.indices.contains($0) ? e.parts[$0].url : nil } + ?? e.url ?? "" + sink?.emitAttempt(AttemptEvent.build(AttemptEvent.Input( + id: e.id, key: e.key, requestId: requestId ?? "", attempt: attempt, url: url, + method: e.method, partIndex: partIndex, statusCode: c.statusCode, headers: c.headers, + body: c.body, error: c.error, accepted: accepted, at: now()))) + } + + // MARK: - Expiry + + /// One in-process timer per live entry at expiresAt + 100 ms. A later arm + /// replaces the token, so a resume that moved expiresAt makes the old timer + /// a no-op. Long waits re-arm daily. It settles a queued or awaiting-auth + /// entry. It leaves a paused one to resume(), and a running one to the + /// result of its attempt: a real response keeps its own error kind, and a + /// transient one becomes 'expired' (scheduleRetry, retryPart). An entry + /// that parks after that is armed again by park(). + func armExpiry(_ e: QueueEntry) { + guard e.isLive, !e.legacy else { return } + let token = UUID() + expiryTokens[e.id] = token + let delay = Int(min(max(e.expiresAt - now(), 0) + 100, 86_400_000)) + schedule(delay) { [weak self] in + guard let self, self.expiryTokens[e.id] == token else { return } + self.expiryTokens[e.id] = nil + guard let current = self.index.entry(e.id), current.isLive, !current.legacy else { return } + guard self.now() >= current.expiresAt else { + self.armExpiry(current) + return + } + switch current.state { + case .paused, .running: return + default: self.settle(current.id, .expired) + } + } + } + + func disarmExpiry(_ id: String) { + expiryTokens[id] = nil + } + + // Assigns only when the write landed. + func saveSettings(_ next: QueueSettings) -> Bool { + do { + try store.saveSettings(next) + settings = next + return true + } catch { + NSLog("[RNFileUploader] cannot save settings: \(error.localizedDescription)") + return false + } + } +} diff --git a/ios/QueueEntry.swift b/ios/QueueEntry.swift new file mode 100644 index 00000000..7ebc2437 --- /dev/null +++ b/ios/QueueEntry.swift @@ -0,0 +1,218 @@ +import Foundation + +/// One durable queue entry: what JS sent at enqueue(), the staged body, and +/// where the entry is in its life. Saved as `entry.json` in the entry's +/// directory (see QueueStore). A pure model: no I/O here. +/// +/// It generalizes the v9 chunked manifest. A chunked entry keeps the v9 +/// fields (parts, accepted flags, incarnation) and gains the queue fields. +struct QueueEntry: Codable, Equatable { + enum State: String, Codable { + case queued, running, awaitingAuth = "awaiting-auth", paused, completed, error, cancelled + } + + enum BodyKind: String, Codable { case none, data, form, file, parts } + + /// One part of a chunked upload, exactly as the consumer authored it. The + /// library sends the file bytes [start, end) to `url`. + struct Part: Codable, Equatable { + let url: String + var headers: [String: String] + let start: Int64 + let end: Int64 // exclusive + var accepted: Bool + /// Failed attempts of this part since its last success or resume. Drives + /// the backoff exponent. Persisted, so a relaunch does not reset it. + var rejections: Int + + var size: Int64 { end - start } + } + + let id: String + var key: String + var varsJSON: String + /// nil only for a chunked entry whose descriptor has no url. + var url: String? + var method: String + var accept: [UploadOutcome.AcceptRule] + var retry: RetryOverride? + var bodyKind: BodyKind + /// File name inside the entry directory: "body-" or a blob name. + /// nil for a legacy row with no bytes. + var bodyPath: String? + /// Content-Type the staged body needs (JSON or multipart). + var bodyContentType: String? + /// true for a multipart body: its boundary is ours, so our Content-Type wins. + var forceContentType: Bool + /// The body identity for the same-id rules. See EnqueueParser.fingerprint. + var bodyFingerprint: String + var parts: [Part] + /// The parts-plan identity. Rotates when the body is replaced. Part tasks + /// carry it, so a late callback from a replaced plan is dropped. + var incarnation: String + /// The merged request headers. updateHeaders() patches them. + var headers: [String: String] + /// settings.headerGeneration at the last header write. + var headerGeneration: Int + var state: State + /// true while parked on a 401/403. Survives a pause. + var authParked: Bool + /// Bumps when a settled entry reopens or its body is replaced. An ack + /// forgets the entry only when the event's generation matches. + var generation: Int + /// HTTP attempts issued, parts included. The current simple attempt's + /// ordinal equals this value. + var attempts: Int + var bytesSent: Int64 + var totalBytes: Int64 + var expiresAt: Double // epoch ms + var nextAttemptAt: Double? // epoch ms, set while a delayed retry waits + var settledEventId: String? + var lastRequestId: String? + var lastUrl: String? + var lastPartIndex: Int? + /// An imported v9 journal row: read-only, never scheduled. + var legacy: Bool + let createdAt: Double + var updatedAt: Double +} + +extension QueueEntry { + var isLive: Bool { + switch state { + case .queued, .running, .awaitingAuth, .paused: return true + case .completed, .error, .cancelled: return false + } + } + + var isSettled: Bool { !isLive } + var isChunked: Bool { bodyKind == .parts } + + var acceptedBytes: Int64 { parts.filter(\.accepted).reduce(0) { $0 + $1.size } } + var allAccepted: Bool { !parts.isEmpty && parts.allSatisfy(\.accepted) } + func pendingIndexes() -> [Int] { parts.indices.filter { !parts[$0].accepted } } + + /// The url a SettledEvent reports when no attempt has run yet. + var targetURL: String { lastUrl ?? url ?? parts.first?.url ?? "" } + + func withPartAccepted(_ index: Int) -> QueueEntry { + var next = self + next.parts[index].accepted = true + next.parts[index].rejections = 0 + next.bytesSent = next.acceptedBytes + return next + } + + /// true when `parts` cover [0, size) exactly: no gap, no overlap, nothing + /// past the end. + static func tilesExactly(_ parts: [Part], size: Int64) -> Bool { + guard !parts.isEmpty else { return false } + var cursor: Int64 = 0 + for part in parts.sorted(by: { $0.start < $1.start }) { + guard part.start == cursor, part.end > part.start else { return false } + cursor = part.end + } + return cursor == size + } + + /// Same parts = same count, and the same url and range at each index. + static func sameParts(_ a: [Part], _ b: [Part]) -> Bool { + a.count == b.count && a.indices.allSatisfy { + a[$0].url == b[$0].url && a[$0].start == b[$0].start && a[$0].end == b[$0].end + } + } + + /// Rule 2: a new entry. + static func created(from p: ParsedEnqueue, staged: StagedBody, headerGeneration: Int, + paused: Bool, now: Double, createdAt: Double? = nil) -> QueueEntry { + QueueEntry( + id: p.id, key: p.key, varsJSON: p.varsJSON, url: p.url, method: p.method, accept: p.accept, retry: p.retry, + bodyKind: staged.kind, bodyPath: staged.relativePath, + bodyContentType: staged.contentType, forceContentType: staged.forceContentType, + bodyFingerprint: p.fingerprint, parts: p.parts, incarnation: UUID().uuidString, + headers: p.headers, headerGeneration: headerGeneration, + state: paused ? .paused : .queued, authParked: false, generation: 1, attempts: 0, + bytesSent: 0, totalBytes: staged.totalBytes, expiresAt: p.expiresAt, + nextAttemptAt: nil, settledEventId: nil, lastRequestId: nil, lastUrl: nil, + lastPartIndex: nil, legacy: false, createdAt: createdAt ?? now, updatedAt: now) + } + + /// Rule 3, same body: the new vars, headers, expiresAt and descriptor + /// fields replace the stored ones. The body, url, method, accepted parts, + /// generation and createdAt stay (a different url or method is a different + /// body). `resetBudget` (a reopen, which starts a new generation) also + /// resets attempts and part rejections: attempts count one generation. + func resumed(with p: ParsedEnqueue, resetBudget: Bool, now: Double) -> QueueEntry { + var next = self + next.key = p.key + next.varsJSON = p.varsJSON + next.headers = p.headers + next.expiresAt = p.expiresAt + next.accept = p.accept + next.retry = p.retry + // Same parts by definition of "same body"; the incoming ones carry the + // new per-part headers. + if isChunked, p.parts.count == parts.count { + next.parts = p.parts.enumerated().map { i, part in + var merged = part + merged.accepted = parts[i].accepted + merged.rejections = resetBudget ? 0 : parts[i].rejections + return merged + } + } + if resetBudget { next.attempts = 0 } + next.updatedAt = now + return next + } + + /// Rule 4, different body: a new body and plan, a new generation, state + /// queued. The caller moves it to paused when the queue is paused. + func replaced(with p: ParsedEnqueue, staged: StagedBody, now: Double) -> QueueEntry { + var next = QueueEntry.created(from: p, staged: staged, headerGeneration: headerGeneration, + paused: false, now: now, createdAt: createdAt) + next.generation = generation + 1 + return next + } + + /// The RequestRow dictionary. `vars` is the decoded object (the index + /// caches it). nextAttemptAt is omitted when nil, so nothing becomes NSNull. + func row(vars: Any) -> [String: Any] { + var r: [String: Any] = [ + "id": id, + "key": key, + "vars": vars, + "state": state.rawValue, + "bytesSent": state == .completed ? totalBytes : bytesSent, + "totalBytes": totalBytes, + "attempts": attempts, + "updatedAt": updatedAt, + ] + if let nextAttemptAt { r["nextAttemptAt"] = nextAttemptAt } + return r + } +} + +/// Header names match without regard to case, as in the JS layer. +enum HeaderMerge { + /// `over` wins. A name in `base` that `over` sets in another case is + /// dropped, so the request never carries both spellings. + static func merge(_ base: [String: String], _ over: [String: String]) -> [String: String] { + let overridden = Set(over.keys.map { $0.lowercased() }) + var result = base.filter { !overridden.contains($0.key.lowercased()) } + for (k, v) in over { result[k] = v } + return result + } + + /// Replaces only the names `base` already carries, in any case. A part's + /// own header (say Authorization) takes the patched value; a name the + /// part does not carry is left to the entry headers. + static func replaceExisting(_ base: [String: String], _ patch: [String: String]) -> [String: String] { + let carried = Set(base.keys.map { $0.lowercased() }) + return merge(base, patch.filter { carried.contains($0.key.lowercased()) }) + } + + static func value(_ name: String, in headers: [String: String]) -> String? { + let lower = name.lowercased() + return headers.first { $0.key.lowercased() == lower }?.value + } +} diff --git a/ios/QueueSettings.swift b/ios/QueueSettings.swift new file mode 100644 index 00000000..6249af68 --- /dev/null +++ b/ios/QueueSettings.swift @@ -0,0 +1,74 @@ +import Foundation + +/// The retry policy after every override is applied. Defaults are the spec's +/// table: base 1 s, max 2 h, jitter 0.2, exempt [404]. +struct RetryPolicy: Codable, Equatable { + var baseMs: Double + var maxMs: Double + var jitter: Double + var exempt: [Int] + + static let defaults = RetryPolicy(baseMs: 1_000, maxMs: 7_200_000, jitter: 0.2, exempt: [404]) + + /// Applies the overrides in order. A later override wins, field by field. + static func resolve(_ overrides: [RetryOverride?]) -> RetryPolicy { + var p = defaults + for o in overrides.compactMap({ $0 }) { + if let v = o.baseMs { p.baseMs = v } + if let v = o.maxMs { p.maxMs = v } + if let v = o.jitter { p.jitter = v } + if let v = o.exempt { p.exempt = v } + } + return p + } +} + +/// A partial retry policy, as `configure({ retry })` or `descriptor.retry` +/// sends it: `{ backoff?: { baseMs, maxMs, jitter }, terminalHttp?: { exempt } }`. +/// Each field is optional, because JS checks names, not presence. +struct RetryOverride: Codable, Equatable { + var baseMs: Double? + var maxMs: Double? + var jitter: Double? + var exempt: [Int]? + + static func parse(_ raw: Any?) -> RetryOverride? { + guard let r = raw as? [String: Any] else { return nil } + let backoff = r["backoff"] as? [String: Any] + let terminal = r["terminalHttp"] as? [String: Any] + let o = RetryOverride( + baseMs: (backoff?["baseMs"] as? NSNumber)?.doubleValue, + maxMs: (backoff?["maxMs"] as? NSNumber)?.doubleValue, + jitter: (backoff?["jitter"] as? NSNumber)?.doubleValue, + exempt: (terminal?["exempt"] as? [NSNumber])?.map(\.intValue)) + return o == RetryOverride() ? nil : o + } +} + +/// Queue-wide settings, persisted as `settings.json` in the queue directory. +struct QueueSettings: Codable, Equatable { + var wifiOnly = false + var paused = false + /// Bumped by every updateHeaders(). An attempt records the value it was + /// issued under; a 401/403 from an older value re-issues instead of parking. + var headerGeneration = 0 + var retry: RetryOverride? + + init() {} + + // Every field is optional on decode, so a settings file from an older build + // (or a newer one) still loads. + init(from decoder: Decoder) throws { + let c = try decoder.container(keyedBy: CodingKeys.self) + wifiOnly = try c.decodeIfPresent(Bool.self, forKey: .wifiOnly) ?? false + paused = try c.decodeIfPresent(Bool.self, forKey: .paused) ?? false + headerGeneration = try c.decodeIfPresent(Int.self, forKey: .headerGeneration) ?? 0 + retry = try c.decodeIfPresent(RetryOverride.self, forKey: .retry) + } + + /// configure(options). iOS reads `retry` only: JS turns lifetimeMs into + /// each entry's expiresAt, and the Android notification keys are Android's. + mutating func apply(configure options: [String: Any]) { + retry = RetryOverride.parse(options["retry"]) + } +} diff --git a/ios/QueueStore.swift b/ios/QueueStore.swift new file mode 100644 index 00000000..3d3382eb --- /dev/null +++ b/ios/QueueStore.swift @@ -0,0 +1,290 @@ +import Foundation + +/// The durable queue: one directory per entry id under +/// `Application Support/RNFileUploaderChunked/`. It generalizes the v9 +/// chunked store in place, so v9 bytes and manifests survive the upgrade. +/// +/// An entry directory holds: +/// - `entry.json`: the QueueEntry (v10). +/// - `body-` or `blob-` / `blob`: the staged body. +/// - `part-..-`: a chunked part while in flight. +/// - `manifest.json`: a v9 manifest, until a same-id enqueue adopts it. +/// +/// Next to the directories: `settings.json` and the `v10-imported` marker. +/// Writes are tmp + fsync + rename. A corrupt or half-written file reads as +/// absent. Synchronous on a private serial queue, like the v9 store. +final class QueueStore { + static let shared = QueueStore( + root: FileIO.applicationSupport().appendingPathComponent("RNFileUploaderChunked", isDirectory: true)) + + let root: URL + private let queue = DispatchQueue(label: "ai.openspace.rnbgupload.store") + + static let entryName = "entry.json" + static let manifestName = "manifest.json" + private static let settingsName = "settings.json" + private static let importedMarker = "v10-imported" + /// Id directory names are base64url, which never starts with ".". + private static let asidePrefix = ".forget-" + + init(root: URL) { + self.root = root + FileIO.makeLocalDirectory(root) + } + + // MARK: - Paths + + /// Ids come from the consumer and may hold path separators. The directory + /// name is url-safe base64 of the id, never the id itself. + func dir(_ id: String) -> URL { + root.appendingPathComponent( + Data(id.utf8).base64EncodedString() + .replacingOccurrences(of: "+", with: "-") + .replacingOccurrences(of: "/", with: "_") + .replacingOccurrences(of: "=", with: ""), + isDirectory: true) + } + + func fileURL(_ id: String, _ relative: String) -> URL { + dir(id).appendingPathComponent(relative) + } + + /// The staged body of an entry, or nil for a legacy row with no bytes. + func bodyURL(_ entry: QueueEntry) -> URL? { + entry.bodyPath.map { fileURL(entry.id, $0) } + } + + // MARK: - Entries + + /// Throws on a write failure. An entry that did not persist must fail the + /// enqueue. + func save(_ entry: QueueEntry) throws { + try queue.sync { + try FileManager.default.createDirectory(at: dir(entry.id), withIntermediateDirectories: true) + try FileIO.writeAtomically(Self.encode(entry), to: fileURL(entry.id, Self.entryName)) + } + } + + /// Never reads a `.tmp`. A corrupt file reads as nil. + func load(_ id: String) -> QueueEntry? { + queue.sync { Self.read(fileURL(id, Self.entryName)) } + } + + func all() -> [QueueEntry] { + queue.sync { + subdirectories().compactMap { Self.read($0.appendingPathComponent(Self.entryName)) } + } + } + + /// Deletes the id directory: entry, body, blob, part files, and a v9 + /// manifest. + func remove(_ id: String) { + queue.sync { _ = try? FileManager.default.removeItem(at: dir(id)) } + } + + // MARK: - Forget in steps (cancel on a settled entry) + + /// Step one of a forget that must not half-happen: renames the id + /// directory to a hidden name in `root`. One rename, so either the row is + /// gone from `all()` or nothing moved. Throws when the rename fails. + /// Returns nil when the id has no directory. + func setAside(_ id: String) throws -> URL? { + try queue.sync { + let d = dir(id) + guard FileIO.exists(d) else { return nil } + let aside = root.appendingPathComponent( + Self.asidePrefix + d.lastPathComponent + "-" + UUID().uuidString, isDirectory: true) + try FileIO.rename(d, onto: aside) + return aside + } + } + + /// Undoes `setAside`. + func restore(_ aside: URL, _ id: String) throws { + try queue.sync { try FileIO.rename(aside, onto: dir(id)) } + } + + /// Deletes a set-aside directory. A failure leaves it for the next launch. + func discard(_ aside: URL) { + queue.sync { _ = try? FileManager.default.removeItem(at: aside) } + } + + /// Set-aside directories a crash left, with the id their entry names (nil + /// when entry.json is unreadable). + func setAsideDirectories() -> [(aside: URL, id: String?)] { + queue.sync { + let items = (try? FileManager.default.contentsOfDirectory(at: root, includingPropertiesForKeys: nil)) ?? [] + return items.filter { $0.lastPathComponent.hasPrefix(Self.asidePrefix) } + .map { ($0, Self.read($0.appendingPathComponent(Self.entryName))?.id) } + } + } + + /// Deletes files the entry does not reference: an old body after a + /// replace, a body staged by an enqueue that crashed before its save, + /// `.tmp` leftovers, and part files of another incarnation. + func sweep(_ entry: QueueEntry) { + queue.sync { + for file in files(in: dir(entry.id)) { + let name = file.lastPathComponent + let isBody = name.hasPrefix(BodyStaging.bodyPrefix) || name.hasPrefix(BodyStaging.blobPrefix) + || name == ChunkedManifestV9.blobName + let stalePart = name.hasPrefix("part-") && !name.contains(".\(entry.incarnation).") + if (isBody && name != entry.bodyPath) || name.hasSuffix(".tmp") || stalePart { + try? FileManager.default.removeItem(at: file) + } + } + } + } + + /// A blob in a directory with no entry.json: the bytes a crash left + /// between the move and the first save. A same-id retry adopts them. + func adoptableBlob(_ id: String) -> String? { + queue.sync { + let d = dir(id) + guard !FileIO.exists(d.appendingPathComponent(Self.entryName)) else { return nil } + return files(in: d).map(\.lastPathComponent) + .filter { $0 == ChunkedManifestV9.blobName || $0.hasPrefix(BodyStaging.blobPrefix) } + .sorted().first + } + } + + // MARK: - Settings and the import marker + + func loadSettings() -> QueueSettings { + queue.sync { + guard let data = try? Data(contentsOf: root.appendingPathComponent(Self.settingsName)), + let s = try? JSONDecoder().decode(QueueSettings.self, from: data) else { return QueueSettings() } + return s + } + } + + func saveSettings(_ settings: QueueSettings) throws { + try queue.sync { + try FileIO.writeAtomically( + try JSONEncoder().encode(settings), to: root.appendingPathComponent(Self.settingsName)) + } + } + + func isImported() -> Bool { + queue.sync { FileIO.exists(root.appendingPathComponent(Self.importedMarker)) } + } + + func markImported() throws { + try queue.sync { + try FileIO.writeAtomically(Data(), to: root.appendingPathComponent(Self.importedMarker)) + } + } + + // MARK: - v9 manifests + + /// A v9 `manifest.json` in the id directory, with or without an entry. + func loadV9Manifest(_ id: String) -> ChunkedManifestV9? { + queue.sync { Self.readManifest(fileURL(id, Self.manifestName)) } + } + + /// Every v9 manifest, keyed by id. + func allV9Manifests() -> [String: ChunkedManifestV9] { + queue.sync { + var result: [String: ChunkedManifestV9] = [:] + for d in subdirectories() { + if let m = Self.readManifest(d.appendingPathComponent(Self.manifestName)) { result[m.id] = m } + } + return result + } + } + + func removeV9Manifest(_ id: String) { + queue.sync { _ = try? FileManager.default.removeItem(at: fileURL(id, Self.manifestName)) } + } + + // MARK: - Part files (carried over from v9) + + /// The temp file that holds exactly the byte range of part `index` while + /// that part is enqueued (a background session uploads only from a file). + /// The name carries the incarnation and the range, so a stale file from + /// another plan is never adopted by a size coincidence. + func partFileURL(_ id: String, _ index: Int, incarnation: String, start: Int64, end: Int64) -> URL { + dir(id).appendingPathComponent("part-\(index).\(incarnation).\(start)-\(end)") + } + + /// Writes bytes [start, end) of `blob` into the part file, tmp + rename. + /// Reuses a finished file with the same identity and size. Throws when the + /// blob is missing or shorter than `end`: a 'file' terminal for the caller. + func writePartFile(id: String, blob: String, index: Int, start: Int64, end: Int64, + incarnation: String) throws -> URL { + try queue.sync { + let dest = partFileURL(id, index, incarnation: incarnation, start: start, end: end) + removePartFilesLocked(id, index, keeping: dest) + if FileIO.size(dest) == end - start { return dest } + let blobURL = fileURL(id, blob) + let blobSize = FileIO.size(blobURL) ?? 0 + guard blobSize >= end else { + throw FileIO.IOError(message: "source blob is \(blobSize) bytes; part \(index) needs [\(start), \(end))") + } + let tmp = dir(id).appendingPathComponent("part-\(index).tmp") + try? FileManager.default.removeItem(at: tmp) + FileManager.default.createFile(atPath: tmp.path, contents: nil) + let reader = try FileHandle(forReadingFrom: blobURL) + defer { try? reader.close() } + let writer = try FileHandle(forWritingTo: tmp) + defer { try? writer.close() } + try reader.seek(toOffset: UInt64(start)) + var remaining = end - start + while remaining > 0 { + let chunk = Int(min(remaining, 1 << 20)) + guard let data = try reader.read(upToCount: chunk), !data.isEmpty else { + throw FileIO.IOError(message: "short read building part \(index)") + } + try writer.write(contentsOf: data) + remaining -= Int64(data.count) + } + try FileIO.rename(tmp, onto: dest) + return dest + } + } + + /// Removes every file of part `index`: current, stale, and tmp. + func removePartFile(_ id: String, _ index: Int) { + queue.sync { removePartFilesLocked(id, index, keeping: nil) } + } + + // MARK: - Private (callers hold `queue`) + + // The "part-." prefix cannot collide across indexes ("part-1." is not a + // prefix of "part-12."). + private func removePartFilesLocked(_ id: String, _ index: Int, keeping: URL?) { + for file in files(in: dir(id)) + where file.lastPathComponent.hasPrefix("part-\(index).") + && file.lastPathComponent != keeping?.lastPathComponent { + try? FileManager.default.removeItem(at: file) + } + } + + private func subdirectories() -> [URL] { + let items = (try? FileManager.default.contentsOfDirectory( + at: root, includingPropertiesForKeys: [.isDirectoryKey], options: [.skipsHiddenFiles])) ?? [] + return items.filter { (try? $0.resourceValues(forKeys: [.isDirectoryKey]).isDirectory) == true } + } + + private func files(in dir: URL) -> [URL] { + (try? FileManager.default.contentsOfDirectory(at: dir, includingPropertiesForKeys: nil)) ?? [] + } + + static func encode(_ entry: QueueEntry) throws -> Data { + let encoder = JSONEncoder() + encoder.outputFormatting = [.sortedKeys] + return try encoder.encode(entry) + } + + private static func read(_ url: URL) -> QueueEntry? { + guard let data = try? Data(contentsOf: url) else { return nil } + return try? JSONDecoder().decode(QueueEntry.self, from: data) + } + + private static func readManifest(_ url: URL) -> ChunkedManifestV9? { + guard let data = try? Data(contentsOf: url), + let m = try? JSONDecoder().decode(ChunkedManifestV9.self, from: data), + !m.parts.isEmpty else { return nil } + return m + } +} diff --git a/ios/RNBackgroundUpload.swift b/ios/RNBackgroundUpload.swift index cd967f8a..f25a480d 100644 --- a/ios/RNBackgroundUpload.swift +++ b/ios/RNBackgroundUpload.swift @@ -1,536 +1,207 @@ import Foundation import React +import UIKit -// Live events destined for JS. The TurboModule shell (RNFileUploader.mm) adopts -// this and forwards to the codegen-generated emitters. The delegate is nil -// whenever JS isn't around (headless relaunch, before the module is created, -// after a reload tears the old one down) — terminal outcomes are journaled -// before we ever get here, so dropping a live event is always safe. +// Live events destined for JS. The TurboModule shell (RNFileUploader.mm) +// adopts this and forwards to the codegen emitters. The delegate is nil +// whenever JS is not around (a headless relaunch, before the module exists, +// after a reload tears the old one down). Every terminal is journaled before +// it gets here, so dropping a live event is always safe. @objc public protocol RNFileUploaderEventDelegate { + func emitState(_ body: [String: Any]) func emitProgress(_ body: [String: Any]) - func emitCompleted(_ body: [String: Any]) - func emitError(_ body: [String: Any]) - func emitCancelled(_ body: [String: Any]) + func emitAttempt(_ body: [String: Any]) + func emitSettled(_ body: [String: Any]) } -// Background HTTP file uploader (iOS). Uploads run on a background URLSession so -// they continue while the app is suspended and complete/relaunch when terminated -// by the system. Terminal outcomes are journaled before being emitted, so JS can -// recover them even if it was dead when they fired. +// The process-wide owner of the two background URLSessions and their +// delegate. It adapts URLSession callbacks and the TurboModule methods to the +// QueueCoordinator, which holds every queue rule. // -// State that must be consistent for the whole process is STATIC: the background -// sessions, the in-flight response buffers, the user-cancel set, and the event -// delegate. The TurboModule instance comes and goes with the JS runtime while the -// URLSession delegate stays pinned to this object, so keeping that state static -// (rather than on the module) is what keeps cancel attribution and response -// assembly correct across a reload — and guarantees we never create two -// background sessions with the same identifier. +// The TurboModule instance comes and goes with the JS runtime; this object +// lives for the whole process. That keeps one delegate per background session +// identifier, and the response buffers and task ownership stay correct +// across a reload. @objc(RNBackgroundUpload) public class RNBackgroundUpload: NSObject, URLSessionDataDelegate { - // The instance that owns the URLSession delegate callbacks. Created on first - // access — by the TurboModule, or by the AppDelegate's - // handleEventsForBackgroundURLSession hook, whichever happens first. That - // second path is load-bearing: on a system relaunch there may be no JS at all, - // and touching `shared` is what recreates the sessions so nsurlsessiond can - // deliver the delegate events it has queued for us. + // Created on first access: by the TurboModule, or by the AppDelegate's + // handleEventsForBackgroundURLSession hook, whichever comes first. The + // second path matters: on a system relaunch there may be no JS at all, and + // touching `shared` recreates the sessions so nsurlsessiond can deliver the + // events it queued. @objc public static let shared = RNBackgroundUpload() - private static let backgroundSessionId = "ReactNativeBackgroundUpload" - private static let wifiOnlySessionId = "ReactNativeBackgroundUpload_WifiOnly" - private static let progressThrottle: TimeInterval = 0.5 // seconds, per upload - - private static let lock = NSLock() - private static var responsesData: [String: NSMutableData] = [:] // sessionId:taskId -> body - private static var lastProgressAt: [String: TimeInterval] = [:] // uploadId -> time - private static var userCancelledIds = Set() - // The ids that removeUpload is releasing now. The cancellation of their - // tasks is an explicit release, not an outcome that the consumer awaits. - // Thus no terminal event is journaled. This matches Android, whose - // removeUpload cancels work with no user-cancel mark. - private static var removedIds = Set() - // The consumer-supplied ids whose check-and-create is in flight, mapped to - // the promises of the concurrent same-id calls. The existence check - // enumerates the session tasks asynchronously. Without this claim, two - // concurrent calls could both see "no task" and enqueue duplicates. The id - // is claimed synchronously, under `lock`, BEFORE the enumeration is - // dispatched. The map entry drains when the first caller's create-or-find - // lands, and every parked call gets that caller's outcome. - private static var creationsInFlight: - [String: [(resolve: RCTPromiseResolveBlock, reject: RCTPromiseRejectBlock)]] = [:] - - private static var backgroundSession: URLSession? - private static var wifiOnlySession: URLSession? - - // Deliberately its own lock, not `lock`: creating `shared` acquires `lock` to - // build the sessions, so guarding the delegate with the same lock would risk a - // deadlock between "ensure shared exists" and "set the delegate". + private let transport: SessionTransport + private let sink: DelegateSink + private let coordinator: QueueCoordinator + + private let bufferLock = NSLock() + private var responses: [String: ResponseBuffer] = [:] // TaskMap key -> body so far + + // Its own lock, not bufferLock: creating `shared` must never wait on the + // lock that guards the delegate. private static let delegateLock = NSLock() private static weak var eventDelegate: RNFileUploaderEventDelegate? + // true from the registered module's first journal drain (its JS listener + // is attached by then) until that module goes away. Guarded by + // delegateLock. A settle with no listener is journaled at deliveries 0. + private static var listening = false - // AppDelegate stores the system-provided completion handler here (per session - // id) so the app can be relaunched to finish uploads after termination. + // AppDelegate stores the system completion handler here per session id. private static let bgHandlerLock = NSLock() private static var bgCompletionHandlers: [String: () -> Void] = [:] - // Relaunch ordering: while the chunked coordinator reconciles (deferrals - // > 0), urlSessionDidFinishEvents must NOT hand the system its completion - // handler. The system could suspend the app before the post-reconcile - // refill enqueues a new part task. That would leave zero daemon tasks and + // While a relaunch reconcile runs (deferrals > 0), urlSessionDidFinishEvents + // must not hand the system its handler: the system could suspend the app + // before the refill enqueues the next tasks, leaving zero daemon tasks and // no future wake. A session that finishes its events in that window parks - // its id here. The release drains it. + // its id here; the release drains it. private static var bgHandlerDeferrals = 0 private static var bgSessionsAwaitingDrain: Set = [] - // Owns the chunked-upload window and the manifests. It is implicitly - // unwrapped only because it needs `self` (for the sessions) and is assigned - // before init returns. It is never nil after that. - private var chunked: ChunkedCoordinator! - public override init() { + let transport = SessionTransport() + let sink = DelegateSink() + self.transport = transport + self.sink = sink + // The coordinator exists, and its index is loaded from disk, before any + // session can deliver a callback. + coordinator = QueueCoordinator( + store: .shared, journal: .shared, taskMap: .shared, transport: transport, sink: sink) super.init() - // Recreate the sessions as early as possible so delegate events queued by - // nsurlsessiond from a previous launch are delivered to this process. - _ = session(wifiOnly: false) - _ = session(wifiOnly: true) - chunked = ChunkedCoordinator(uploader: self) - // Relaunch reconciliation: match the daemon's surviving tasks against - // the stored manifests, and refill each upload's window. It runs on the - // coordinator queue. Thus nothing here re-enters the initialization of - // `shared`. - chunked.reconcileAll() + transport.createSessions(delegate: self) + observeAppState() + // Claim the completion-handler deferral BEFORE the reconcile is queued. + // The release waits for the reconcile and for every grace wait it opens. + // A relaunch reaches here inside the init of `shared`, and the AppDelegate + // hook finishes that init before it stores the handler, so the claim + // always precedes any drain. + RNBackgroundUpload.deferBackgroundCompletionHandlers() + coordinator.reconcileAll { RNBackgroundUpload.releaseBackgroundCompletionHandlers() } } // MARK: - Event delegate @objc public static func setEventDelegate(_ delegate: RNFileUploaderEventDelegate) { - // Force the singleton (and therefore the sessions) into existence before - // taking the lock — see the note on delegateLock. + // Force the singleton (and the sessions) into existence before the lock. _ = shared delegateLock.lock() eventDelegate = delegate + // A new module has no JS listener until its own drain. + listening = false delegateLock.unlock() } - /// Deregisters a delegate, but only if it is still the registered one. - /// - /// React Native dispatches `invalidate` asynchronously and gives up waiting - /// after 10s, so a slow call can let the replacement module register itself - /// before the outgoing module's `invalidate` actually runs. Clearing - /// unconditionally there would null out the live delegate and silently stop - /// every event for the rest of the process. + /// Deregisters a delegate only if it is still the registered one. React + /// Native runs invalidate asynchronously and stops waiting after 10 s, so + /// the replacement module can register first. Clearing unconditionally + /// would stop every event for the rest of the process. @objc public static func clearEventDelegate(_ delegate: RNFileUploaderEventDelegate) { delegateLock.lock() defer { delegateLock.unlock() } - if eventDelegate === delegate { eventDelegate = nil } + if eventDelegate === delegate { + eventDelegate = nil + listening = false + } } - private static var currentDelegate: RNFileUploaderEventDelegate? { + fileprivate static var currentDelegate: RNFileUploaderEventDelegate? { delegateLock.lock() defer { delegateLock.unlock() } return eventDelegate } - // Journal-before-emit, the library's one terminal-event path. The write is - // durable. The emit is best-effort, because JS can be dead. The - // simple-upload delegate handling and the chunked coordinator share it. - static func journalAndEmit(_ event: JournaledEvent) { - EventJournal.append(event) - emitEvent(event) + fileprivate static var canDeliver: Bool { + delegateLock.lock() + defer { delegateLock.unlock() } + return listening && eventDelegate != nil } - /// Emits WITHOUT a journal write. Use it to deliver again an event that is - /// already in the journal (a resume of a finished-but-unacked upload). - static func emitEvent(_ event: JournaledEvent) { - let body = event.bridged - let delegate = currentDelegate - switch event.type { - case "completed": delegate?.emitCompleted(body) - case "cancelled": delegate?.emitCancelled(body) - default: delegate?.emitError(body) - } + fileprivate static func markListening() { + delegateLock.lock() + defer { delegateLock.unlock() } + if eventDelegate != nil { listening = true } } - static func emitProgress(id: String, progress: Float) { - currentDelegate?.emitProgress(["id": id, "progress": progress]) - } + // MARK: - Module methods (called from the TurboModule shell) - // MARK: - Sessions - - // Internal, not private: the chunked coordinator enqueues part tasks on the - // same two sessions. - func session(wifiOnly: Bool) -> URLSession { - RNBackgroundUpload.lock.lock() - defer { RNBackgroundUpload.lock.unlock() } - if wifiOnly { - if let s = RNBackgroundUpload.wifiOnlySession { return s } - let s = makeSession(identifier: RNBackgroundUpload.wifiOnlySessionId, wifiOnly: true) - RNBackgroundUpload.wifiOnlySession = s - return s - } else { - if let s = RNBackgroundUpload.backgroundSession { return s } - let s = makeSession(identifier: RNBackgroundUpload.backgroundSessionId, wifiOnly: false) - RNBackgroundUpload.backgroundSession = s - return s - } - } + // Each is a one-line forward. The coordinator hops onto its own queue and + // returns, so the module queue never waits. - // Session configuration is load-bearing and carried over verbatim from the - // original Obj-C. Config must be set before the session is created (URLSession - // copies it). Background upload tasks require uploadTask(with:fromFile:). - private func makeSession(identifier: String, wifiOnly: Bool) -> URLSession { - let config = URLSessionConfiguration.background(withIdentifier: identifier) - config.isDiscretionary = false - // A per-session, connection-level backstop for the design's library-wide - // transmission cap of 4. The request-level control is the chunked window. - // This limit mostly bounds piles of simple uploads over HTTP/1.1. - config.httpMaximumConnectionsPerHost = 4 - config.waitsForConnectivity = true - config.allowsCellularAccess = !wifiOnly - config.allowsConstrainedNetworkAccess = !wifiOnly - config.allowsExpensiveNetworkAccess = !wifiOnly - return URLSession(configuration: config, delegate: self, delegateQueue: nil) + @objc(configure:) + public func configure(_ options: [String: Any]) { + coordinator.configure(options) } - private func taskMapKey(_ session: URLSession, _ task: URLSessionTask) -> String { - TaskMap.key(session, task) + @objc(enqueue:resolve:reject:) + public func enqueue(_ entry: [String: Any], resolve: @escaping RCTPromiseResolveBlock, + reject: @escaping RCTPromiseRejectBlock) { + coordinator.enqueue(entry, resolve: { resolve($0) }, reject: { reject($0, $1, nil) }) } - // taskDescription is the primary id; the persisted map is the durable fallback. - // A chunked part task's description encodes (uploadId, partIndex). This - // returns the uploadId in both cases. Thus id matching works uniformly. - private func uploadId(_ session: URLSession, _ task: URLSessionTask) -> String { - if let ref = ChunkedCoordinator.partRef(session, task) { return ref.id } - return task.taskDescription ?? TaskMap.meta(forKey: taskMapKey(session, task))?.id ?? "unknown" + @objc(pause:reject:) + public func pause(_ resolve: @escaping RCTPromiseResolveBlock, reject: @escaping RCTPromiseRejectBlock) { + coordinator.pause(resolve: { resolve(nil) }, reject: { reject($0, $1, nil) }) } - private func acceptRules(_ session: URLSession, _ task: URLSessionTask) -> [UploadOutcome.AcceptRule] { - TaskMap.meta(forKey: taskMapKey(session, task))?.accept ?? [] + @objc(resume:reject:) + public func resume(_ resolve: @escaping RCTPromiseResolveBlock, reject: @escaping RCTPromiseRejectBlock) { + coordinator.resume(resolve: { resolve(nil) }, reject: { reject($0, $1, nil) }) } - private var activeSessions: [URLSession] { - [RNBackgroundUpload.backgroundSession, RNBackgroundUpload.wifiOnlySession].compactMap { $0 } + @objc(cancel:resolve:reject:) + public func cancel(_ id: String, resolve: @escaping RCTPromiseResolveBlock, + reject: @escaping RCTPromiseRejectBlock) { + coordinator.cancel(id, resolve: { resolve(nil) }, reject: { reject($0, $1, nil) }) } - // MARK: - Exported methods (called from the TurboModule shell) - - @objc(startUpload:resolve:reject:) - public func startUpload(_ options: [String: Any], - resolve: @escaping RCTPromiseResolveBlock, + @objc(setWifiOnly:resolve:reject:) + public func setWifiOnly(_ enabled: Bool, resolve: @escaping RCTPromiseResolveBlock, reject: @escaping RCTPromiseRejectBlock) { - guard let urlString = options["url"] as? String, let path = options["path"] as? String else { - reject("RN Uploader", "Missing 'url' or 'path'", nil); return - } - guard let requestUrl = URL(string: urlString) else { - reject("RN Uploader", "URL not compliant with RFC 2396", nil); return - } - let type = (options["type"] as? String) ?? "raw" - if type != "raw" { - reject("RN Uploader", "Only type: 'raw' is supported", nil); return - } - - var request = URLRequest(url: requestUrl) - request.httpMethod = (options["method"] as? String) ?? "POST" - if let headers = options["headers"] as? [String: Any] { - for (key, value) in headers { - // Only strings and numbers become headers. The original Obj-C skipped - // anything else, and interpolating instead would put "" (or a - // Swift struct description) on the wire for a null/object value — - // silently corrupting e.g. an Authorization header rather than omitting it. - if let s = value as? String { - request.setValue(s, forHTTPHeaderField: key) - } else if let n = value as? NSNumber { - request.setValue(n.stringValue, forHTTPHeaderField: key) - } - } - } - - let wifiOnly = (options["wifiOnly"] as? Bool) ?? false - let accept = UploadOutcome.parseAcceptRules(options["accept"]) - let uploadId = (options["id"] as? String) ?? UUID().uuidString - let fileURL = URL(string: path) ?? URL(fileURLWithPath: path) - - let session = self.session(wifiOnly: wifiOnly) - let startNew: () throws -> Void = { - let task = try RNBackgroundUpload.uploadTask(session, request, fromFile: fileURL) - task.taskDescription = uploadId - TaskMap.set(TaskMap.Meta(id: uploadId, accept: accept, partIndex: nil), - forKey: self.taskMapKey(session, task)) - task.resume() - } - - // A consumer-supplied id makes startUpload idempotent. This is the same - // behavior as Android's ExistingWorkPolicy.KEEP. If a task with this id is - // already pending or running, we resolve with that id. We do not enqueue a - // second task. We examine both sessions, because a new call can set a - // different wifiOnly value while the first task continues in its first - // session. A generated id cannot collide, so that path does not do the - // (asynchronous) task enumeration. - guard options["id"] != nil else { - do { - try startNew() - resolve(uploadId) - } catch { - reject("RN Uploader", error.localizedDescription, error) - } - return - } - - // Serialize the check-and-create for each id: claim the id synchronously, - // before we dispatch the enumeration. The first caller runs the check and - // creates the task. A concurrent same-id caller parks its promise here. - // When the task lands, we answer the parked calls with the same outcome. - // There is no second task, and there is no polling. - RNBackgroundUpload.lock.lock() - if RNBackgroundUpload.creationsInFlight[uploadId] != nil { - RNBackgroundUpload.creationsInFlight[uploadId]?.append((resolve: resolve, reject: reject)) - RNBackgroundUpload.lock.unlock() - return - } - RNBackgroundUpload.creationsInFlight[uploadId] = [] - RNBackgroundUpload.lock.unlock() - let settle = { (failure: Error?) in - RNBackgroundUpload.lock.lock() - let waiters = RNBackgroundUpload.creationsInFlight.removeValue(forKey: uploadId) ?? [] - RNBackgroundUpload.lock.unlock() - for call in [(resolve: resolve, reject: reject)] + waiters { - if let failure { - call.reject("RN Uploader", failure.localizedDescription, failure) - } else { - call.resolve(uploadId) - } - } - } - - let group = DispatchGroup() - let foundLock = NSLock() - var exists = false - for s in activeSessions { - group.enter() - s.getAllTasks { tasks in - for task in tasks - where self.uploadId(s, task) == uploadId - && (task.state == .running || task.state == .suspended) { - foundLock.lock() - exists = true - foundLock.unlock() - } - group.leave() - } - } - group.notify(queue: .main) { - do { - if !exists { try startNew() } - settle(nil) - } catch { - settle(error) - } - } - } - - @objc(startChunkedUpload:resolve:reject:) - public func startChunkedUpload(_ options: [String: Any], - resolve: @escaping RCTPromiseResolveBlock, - reject: @escaping RCTPromiseRejectBlock) { - chunked.startUpload( - options, - resolve: { id in resolve(id) }, - reject: { message in reject("RN Uploader", message, nil) }) - } - - @objc(removeUpload:resolve:reject:) - public func removeUpload(_ uploadId: String, - resolve: @escaping RCTPromiseResolveBlock, - reject: @escaping RCTPromiseRejectBlock) { - // The chunked release runs first: it cancels the in-flight part tasks - // and deletes the manifest and the bytes. Then we cancel any simple task - // that wears this id. That cancel is kept out of the journal, because an - // explicit release is not an outcome that the consumer awaits. - chunked.remove(uploadId) { - RNBackgroundUpload.lock.lock() - RNBackgroundUpload.removedIds.insert(uploadId) - RNBackgroundUpload.lock.unlock() - let group = DispatchGroup() - let foundLock = NSLock() - var found = false - for session in self.activeSessions { - group.enter() - session.getAllTasks { tasks in - for task in tasks - where self.uploadId(session, task) == uploadId - && ChunkedCoordinator.partRef(session, task) == nil { - foundLock.lock() - found = true - foundLock.unlock() - task.cancel() - } - group.leave() - } - } - group.notify(queue: .main) { - foundLock.lock() - let matched = found - foundLock.unlock() - if !matched { - // Nothing was cancelled. Thus no delegate callback will consume - // the suppression. Drop it. If we keep it, a later upload that - // reuses this id has its real terminal swallowed. - RNBackgroundUpload.lock.lock() - RNBackgroundUpload.removedIds.remove(uploadId) - RNBackgroundUpload.lock.unlock() - } - resolve(nil) - } - } + coordinator.setWifiOnly(enabled, resolve: { resolve(nil) }, reject: { reject($0, $1, nil) }) } - @objc(cancelUpload:resolve:reject:) - public func cancelUpload(_ cancelUploadId: String, - resolve: @escaping RCTPromiseResolveBlock, - reject: @escaping RCTPromiseRejectBlock) { - // A chunked upload cancels through its coordinator: one 'cancelled' - // terminal for the whole upload, journaled before its part tasks are torn - // down. nil means that the id has no manifest. It then falls through to - // the simple-task path. - chunked.cancel(cancelUploadId) { handled in - if let handled { resolve(handled); return } - self.cancelSimpleUpload(cancelUploadId, resolve: resolve) - } + @objc(updateHeaders:resolve:reject:) + public func updateHeaders(_ patch: [String: Any], resolve: @escaping RCTPromiseResolveBlock, + reject: @escaping RCTPromiseRejectBlock) { + coordinator.updateHeaders(patch, resolve: { resolve(nil) }, reject: { reject($0, $1, nil) }) } - private func cancelSimpleUpload(_ cancelUploadId: String, - resolve: @escaping RCTPromiseResolveBlock) { - // Record intent before cancelling so the delegate reports cancelReason 'user'. - RNBackgroundUpload.lock.lock() - RNBackgroundUpload.userCancelledIds.insert(cancelUploadId) - RNBackgroundUpload.lock.unlock() - - let sessions = activeSessions - let group = DispatchGroup() - // Guarded: the two sessions' getAllTasks completions run on independent - // delegate queues, so this is written concurrently. - let foundLock = NSLock() - var found = false - for session in sessions { - group.enter() - session.getAllTasks { tasks in - for task in tasks where self.uploadId(session, task) == cancelUploadId { - foundLock.lock() - found = true - foundLock.unlock() - task.cancel() - } - group.leave() - } - } - group.notify(queue: .main) { - foundLock.lock() - let matched = found - foundLock.unlock() - if !matched { - // Nothing to cancel: drop the intent again so a later upload reusing this - // id isn't misattributed as a user cancel. - RNBackgroundUpload.lock.lock() - RNBackgroundUpload.userCancelledIds.remove(cancelUploadId) - RNBackgroundUpload.lock.unlock() - } - resolve(matched) - } + /// Synchronous, from the in-memory index. + @objc public func getRequests() -> [[String: Any]] { + coordinator.rows() } @objc(getUnacknowledgedEvents:reject:) public func getUnacknowledgedEvents(_ resolve: @escaping RCTPromiseResolveBlock, reject: @escaping RCTPromiseRejectBlock) { - resolve(EventJournal.unacknowledged()) + coordinator.unacknowledgedEvents { resolve($0) } } @objc(ackEvents:resolve:reject:) - public func ackEvents(_ eventIds: [String], - resolve: @escaping RCTPromiseResolveBlock, + public func ackEvents(_ eventIds: [Any], resolve: @escaping RCTPromiseResolveBlock, reject: @escaping RCTPromiseRejectBlock) { - // An acked 'completed' is the ONE moment when a chunked upload's manifest - // and moved bytes may be deleted. Every other terminal keeps them for a - // resume. Find those uploads before the entries are removed. - let completedUploadIds = EventJournal.unacknowledgedEntries() - .filter { $0.type == "completed" && eventIds.contains($0.eventId) } - .map { $0.id } - EventJournal.ack(eventIds) - chunked.releaseCompleted(completedUploadIds) { // no-op for simple uploads - // The spec resolves void. Idempotent: an unknown id is ignored. - resolve(nil) - } + // Resolves void. A non-string id is ignored, like an unknown one. + coordinator.ack(eventIds.compactMap { $0 as? String }) { resolve(nil) } } - @objc(getAllUploads:reject:) - public func getAllUploads(_ resolve: @escaping RCTPromiseResolveBlock, - reject: @escaping RCTPromiseRejectBlock) { - let sessions = activeSessions - let group = DispatchGroup() - let lock = NSLock() - var result: [[String: Any]] = [] - for session in sessions { - group.enter() - session.getAllTasks { tasks in - for task in tasks { - // A chunked upload is one logical row, built from its manifest - // below. Its per-part tasks are transport detail. - if ChunkedCoordinator.partRef(session, task) != nil { continue } - let id = self.uploadId(session, task) - if id == "unknown" { continue } - lock.lock() - // Report the real state. Collapsing everything non-running into - // "pending" told a consumer's boot reconciliation that an upload had - // never started, inviting it to re-enqueue one that was already - // finishing or cancelling. - let state: String - switch task.state { - case .running: state = "running" - case .suspended: state = "pending" - case .canceling: state = "cancelled" - case .completed: state = "completed" - @unknown default: state = "pending" - } - result.append(["id": id, - "state": state, - "bytesSent": task.countOfBytesSent, - "totalBytes": task.countOfBytesExpectedToSend]) - lock.unlock() - } - group.leave() - } - } - group.notify(queue: .main) { - self.chunked.snapshots { chunkedRows in - lock.lock() - let combined = result + chunkedRows - lock.unlock() - resolve(combined) - } - } - } + // MARK: - Background session completion // Called from AppDelegate.application(_:handleEventsForBackgroundURLSession:completionHandler:). // Reachable from a consumer's plain Obj-C via `@import - // react_native_background_upload;` — deliberately NOT on the TurboModule class, - // whose header is Obj-C++ only. + // react_native_background_upload;`, not on the TurboModule class, whose + // header is Obj-C++ only. @objc(setBackgroundSessionCompletionHandler:forIdentifier:) public static func setBackgroundSessionCompletionHandler(_ handler: @escaping () -> Void, forIdentifier identifier: String) { - // Touching `shared` recreates the background sessions when this is a fresh, - // system-relaunched process, which is what lets the queued delegate events - // (and therefore this handler) actually fire. On a relaunch, it also - // claims the handler deferral (see below) BEFORE the handler is stored - // here. Thus the claim provably precedes any drain. + // Touching `shared` recreates the sessions in a system-relaunched + // process, and claims the handler deferral before the handler is stored. _ = shared bgHandlerLock.lock() bgCompletionHandlers[identifier] = handler bgHandlerLock.unlock() } - /// For the chunked coordinator only. It parks every - /// urlSessionDidFinishEvents drain until the matching release. Thus the - /// system cannot suspend the app between a relaunch's replayed part - /// completions and the post-reconcile refill that enqueues the next part - /// tasks. static func deferBackgroundCompletionHandlers() { bgHandlerLock.lock() bgHandlerDeferrals += 1 @@ -546,10 +217,9 @@ public class RNBackgroundUpload: NSObject, URLSessionDataDelegate { if let handler = bgCompletionHandlers.removeValue(forKey: identifier) { handlers.append(handler) } - // A parked id with no stored handler is simply dropped. There is - // nothing to hold. If we keep it, a LATER wake's handler could drain - // before that wake's events were processed. } + // A parked id with no stored handler is dropped. Kept, it could let a + // later wake's handler drain before that wake's events ran. bgSessionsAwaitingDrain.removeAll() } bgHandlerLock.unlock() @@ -560,148 +230,57 @@ public class RNBackgroundUpload: NSObject, URLSessionDataDelegate { public func urlSession(_ session: URLSession, dataTask: URLSessionDataTask, didReceive data: Data) { guard !data.isEmpty else { return } - // Key by sessionId:taskId, not taskIdentifier alone: taskIdentifier is unique - // per session, so two concurrent uploads (one wifiOnly, one not) can share an - // identifier and would otherwise cross-contaminate response bodies. - let key = taskMapKey(session, dataTask) - RNBackgroundUpload.lock.lock() - if let existing = RNBackgroundUpload.responsesData[key] { - existing.append(data) - } else { - RNBackgroundUpload.responsesData[key] = NSMutableData(data: data) - } - RNBackgroundUpload.lock.unlock() + // Keyed by session id and task id: taskIdentifier alone is unique per + // session only. + let key = TaskMap.key(session, dataTask) + bufferLock.lock() + responses[key, default: ResponseBuffer()].append(data) + bufferLock.unlock() } public func urlSession(_ session: URLSession, task: URLSessionTask, didSendBodyData bytesSent: Int64, totalBytesSent: Int64, totalBytesExpectedToSend: Int64) { - // A chunked part's bytes feed the upload's byte-weighted aggregate. A - // per-task percentage would have no meaning to the consumer. - if let ref = ChunkedCoordinator.partRef(session, task) { - chunked.partProgress(id: ref.id, part: ref.part, incarnation: ref.incarnation, - sent: totalBytesSent) - return - } - // 0 rather than -1 when the length is unknown: the documented range is - // 0-100, Android reports 0 for the same case, and a negative value renders - // as a broken progress bar in a consumer that passes it straight through. - var progress: Float = 0 - if totalBytesExpectedToSend > 0 { - progress = 100.0 * Float(totalBytesSent) / Float(totalBytesExpectedToSend) - } - let id = uploadId(session, task) - let now = Date().timeIntervalSince1970 - RNBackgroundUpload.lock.lock() - if progress < 100, - let last = RNBackgroundUpload.lastProgressAt[id], - now - last < RNBackgroundUpload.progressThrottle { - RNBackgroundUpload.lock.unlock() - return + coordinator.taskProgress(key: TaskMap.key(session, task), description: task.taskDescription, + sent: totalBytesSent, expected: totalBytesExpectedToSend) + } + + public func urlSession(_ session: URLSession, task: URLSessionTask, + willBeginDelayedRequest request: URLRequest, + completionHandler: @escaping (URLSession.DelayedRequestDisposition, URLRequest?) -> Void) { + if let next = coordinator.taskWillBegin(key: TaskMap.key(session, task), + description: task.taskDescription) { + completionHandler(.useNewRequest, next) + } else { + completionHandler(.cancel, nil) } - RNBackgroundUpload.lastProgressAt[id] = now - RNBackgroundUpload.lock.unlock() - RNBackgroundUpload.currentDelegate?.emitProgress(["id": id, "progress": progress]) } public func urlSession(_ session: URLSession, task: URLSessionTask, didCompleteWithError error: Error?) { - let id = uploadId(session, task) + let key = TaskMap.key(session, task) let http = task.response as? HTTPURLResponse - let statusCode = http?.statusCode ?? 0 - var headers: [String: String] = [:] if let http { - for (key, value) in http.allHeaderFields { headers["\(key)"] = "\(value)" } - } - - // A chunked part's outcome belongs to the coordinator of its upload: - // accept evaluation against the manifest, the window refill, and one - // journaled terminal, only when the whole upload settles. - if let ref = ChunkedCoordinator.partRef(session, task) { - RNBackgroundUpload.lock.lock() - let bodyData = RNBackgroundUpload.responsesData.removeValue(forKey: taskMapKey(session, task)) - RNBackgroundUpload.lock.unlock() - chunked.handlePartCompletion( - id: ref.id, part: ref.part, incarnation: ref.incarnation, - taskKey: taskMapKey(session, task), - statusCode: http != nil ? statusCode : nil, headers: headers, - body: bodyData.flatMap { String(data: $0 as Data, encoding: .utf8) }, - error: error as NSError?) - return - } - - RNBackgroundUpload.lock.lock() - let bodyData = RNBackgroundUpload.responsesData.removeValue(forKey: taskMapKey(session, task)) - RNBackgroundUpload.lastProgressAt[id] = nil - // Consume the user-cancel intent on EVERY terminal outcome, not only the - // cancelled branch. If cancelUpload lost the race with completion, the id - // would otherwise linger for the life of the process and a later upload - // reusing that id would report a system cancel as a user cancel. - let userCancelled = RNBackgroundUpload.userCancelledIds.remove(id) != nil - let removed = RNBackgroundUpload.removedIds.remove(id) != nil - RNBackgroundUpload.lock.unlock() - - // removeUpload cancelled this task as an explicit release, not as an - // outcome that the consumer awaits. Journal nothing. A non-cancel - // terminal that only raced the removal still reports normally. - if removed, let nsError = error as NSError?, nsError.code == NSURLErrorCancelled { - TaskMap.removeKey(taskMapKey(session, task)) - return - } - - let rawBody = bodyData.flatMap { String(data: $0 as Data, encoding: .utf8) } ?? "" - let (cappedBody, truncated) = EventJournal.capBody(rawBody) - let responseBody = cappedBody ?? "" - - let eventId = UUID().uuidString - let timestamp = Date().timeIntervalSince1970 * 1000 - var event = JournaledEvent(eventId: eventId, id: id, type: "completed", timestamp: timestamp) - if http != nil { - event.responseCode = statusCode - event.responseHeaders = headers - event.responseBody = responseBody - event.responseBodyTruncated = truncated - } - - if error == nil { - // "completed" only for a 2xx or a matching per-request accept rule. - // Any other HTTP response is a terminal http error that carries the - // full response. - let accepted = UploadOutcome.isAccepted( - statusCode, body: rawBody, accept: acceptRules(session, task)) - if accepted { - event.type = "completed" - } else { - event.type = "error" - event.errorKind = "http" - event.error = "HTTP \(statusCode)" - } - } else { - let nsError = error! as NSError - if nsError.code == NSURLErrorCancelled { - event.type = "cancelled" - event.cancelReason = userCancelled ? "user" : "system" - } else { - event.type = "error" - event.errorKind = RNBackgroundUpload.errorKind(for: nsError) - event.error = nsError.localizedDescription - } - } - - TaskMap.removeKey(taskMapKey(session, task)) - // Journals BEFORE it emits. The emit is best-effort, because JS can be - // dead. - RNBackgroundUpload.journalAndEmit(event) + for (name, value) in http.allHeaderFields { headers["\(name)"] = "\(value)" } + } + bufferLock.lock() + let buffer = responses.removeValue(forKey: key) ?? ResponseBuffer() + bufferLock.unlock() + let (body, truncated) = buffer.decoded() + // Synchronous hop: the journal write for a terminal lands before this + // callback returns. + coordinator.taskCompleted(TaskCompletion( + key: key, description: task.taskDescription, + url: task.originalRequest?.url?.absoluteString, statusCode: http?.statusCode, + headers: headers, body: http == nil ? nil : body, bodyTruncated: truncated, + error: error as NSError?)) } public func urlSessionDidFinishEvents(forBackgroundURLSession session: URLSession) { guard let identifier = session.configuration.identifier else { return } RNBackgroundUpload.bgHandlerLock.lock() guard RNBackgroundUpload.bgHandlerDeferrals <= 0 else { - // A relaunch reconcile is in flight. If we hand the system the handler - // now, it can suspend the app before the refill enqueues new part - // tasks. The id is parked. releaseBackgroundCompletionHandlers drains - // it. + // A relaunch reconcile is in flight. Park the id; the release drains it. RNBackgroundUpload.bgSessionsAwaitingDrain.insert(identifier) RNBackgroundUpload.bgHandlerLock.unlock() return @@ -711,40 +290,128 @@ public class RNBackgroundUpload: NSObject, URLSessionDataDelegate { if let handler { DispatchQueue.main.async { handler() } } } - // A background session raises an NSException, not an error, when the file - // cannot be read: for example, when the file was deleted after the caller - // checked it. Uncaught, the exception ends the process. This throws a - // URL-domain error instead, which errorKind(for:) classifies as 'file'. - static func uploadTask(_ session: URLSession, _ request: URLRequest, - fromFile file: URL) throws -> URLSessionUploadTask { - var task: URLSessionUploadTask? + // MARK: - App state + + // The progress throttle is 1 s in the foreground and 10 min in the + // background. The flag is set from notifications, because reading + // applicationState needs the main thread. + private func observeAppState() { + let center = NotificationCenter.default + let throttle = coordinator.throttle + for name in [UIApplication.willEnterForegroundNotification, UIApplication.didBecomeActiveNotification] { + center.addObserver(forName: name, object: nil, queue: nil) { _ in throttle.isForeground = true } + } + center.addObserver(forName: UIApplication.didEnterBackgroundNotification, object: nil, + queue: nil) { _ in throttle.isForeground = false } + DispatchQueue.main.async { + throttle.isForeground = UIApplication.shared.applicationState != .background + } + } +} + +/// Forwards coordinator events to the registered TurboModule, if any. +private final class DelegateSink: EventSink { + func emitState(_ body: [String: Any]) { RNBackgroundUpload.currentDelegate?.emitState(body) } + func emitProgress(_ body: [String: Any]) { RNBackgroundUpload.currentDelegate?.emitProgress(body) } + func emitAttempt(_ body: [String: Any]) { RNBackgroundUpload.currentDelegate?.emitAttempt(body) } + func emitSettled(_ body: [String: Any]) { RNBackgroundUpload.currentDelegate?.emitSettled(body) } + func canDeliver() -> Bool { RNBackgroundUpload.canDeliver } + func listenerReady() { RNBackgroundUpload.markListening() } +} + +/// The two background sessions: one that may use cellular, one Wi-Fi only. +/// allowsCellularAccess and friends are session properties, so a task keeps +/// the session it started on. Both exist from init, so reconcile sees every +/// task. +private final class SessionTransport: Transport { + private static let backgroundSessionId = "ReactNativeBackgroundUpload" + private static let wifiOnlySessionId = "ReactNativeBackgroundUpload_WifiOnly" + + private let lock = NSLock() + private var background: URLSession? + private var wifiOnly: URLSession? + + func createSessions(delegate: URLSessionDelegate) { + lock.lock() + defer { lock.unlock() } + background = Self.makeSession(identifier: Self.backgroundSessionId, wifiOnly: false, delegate: delegate) + wifiOnly = Self.makeSession(identifier: Self.wifiOnlySessionId, wifiOnly: true, delegate: delegate) + } + + func upload(_ request: URLRequest, fromFile file: URL, wifiOnly: Bool, description: String, + beginAt: Date?, beforeResume: (String) -> Void) throws -> UploadTask { + let session = self.session(wifiOnly: wifiOnly) + // A background session uploads from a file only. When it cannot read the + // file it raises an Objective-C exception, not an error. Swift cannot + // catch that, and it ends the app, so the helper catches it. + var created: URLSessionUploadTask? if let exception = RNBGUCatchException({ - task = session.uploadTask(with: request, fromFile: file) + created = session.uploadTask(with: request, fromFile: file) }) { throw NSError(domain: NSURLErrorDomain, code: NSURLErrorCannotOpenFile, userInfo: [ - NSLocalizedDescriptionKey: exception.reason ?? "Cannot read file at \(file.absoluteString)", + NSLocalizedDescriptionKey: exception.reason ?? "Cannot read file at \(file.path)", ]) } - return task! + let task = created! + task.taskDescription = description + if let beginAt { task.earliestBeginDate = beginAt } + let handle = SessionTask(session: session, task: task) + beforeResume(handle.key) + task.resume() + return handle } - // Classify a transport error to match Android's errorKind taxonomy: a missing or - // unreadable source file -> 'file'; other URL-domain errors -> 'network'; anything - // else -> 'unknown'. It is internal because the chunked coordinator also - // classifies with it. - static func errorKind(for error: NSError) -> String { - switch (error.domain, error.code) { - case (NSURLErrorDomain, NSURLErrorFileDoesNotExist), - (NSURLErrorDomain, NSURLErrorCannotOpenFile), - (NSURLErrorDomain, NSURLErrorNoPermissionsToReadFile), - (NSCocoaErrorDomain, NSFileNoSuchFileError), - (NSCocoaErrorDomain, NSFileReadNoSuchFileError), - (NSCocoaErrorDomain, NSFileReadNoPermissionError): - return "file" - case (NSURLErrorDomain, _): - return "network" - default: - return "unknown" + func allTasks(_ completion: @escaping ([UploadTask]) -> Void) { + let sessions = [session(wifiOnly: false), session(wifiOnly: true)] + let group = DispatchGroup() + let collectLock = NSLock() + var collected: [UploadTask] = [] + for session in sessions { + group.enter() + session.getAllTasks { tasks in + collectLock.lock() + collected.append(contentsOf: tasks.map { SessionTask(session: session, task: $0) }) + collectLock.unlock() + group.leave() + } } + group.notify(queue: .global()) { completion(collected) } + } + + private func session(wifiOnly: Bool) -> URLSession { + lock.lock() + defer { lock.unlock() } + // createSessions runs in init, before anything can call this. + return (wifiOnly ? self.wifiOnly : background)! + } + + // Session config is set before the session is created (URLSession copies + // it). Carried over from v9. + private static func makeSession(identifier: String, wifiOnly: Bool, + delegate: URLSessionDelegate) -> URLSession { + let config = URLSessionConfiguration.background(withIdentifier: identifier) + config.isDiscretionary = false + // A per-host backstop. The chunked window of 3 is the real limiter. + config.httpMaximumConnectionsPerHost = 4 + config.waitsForConnectivity = true + config.allowsCellularAccess = !wifiOnly + config.allowsConstrainedNetworkAccess = !wifiOnly + config.allowsExpensiveNetworkAccess = !wifiOnly + return URLSession(configuration: config, delegate: delegate, delegateQueue: nil) } } + +private final class SessionTask: UploadTask { + let key: String + private let task: URLSessionTask + + init(session: URLSession, task: URLSessionTask) { + key = TaskMap.key(session, task) + self.task = task + } + + var taskDescription: String? { task.taskDescription } + var isLive: Bool { task.state == .running || task.state == .suspended } + var beginAt: Date? { task.earliestBeginDate } + func cancel() { task.cancel() } +} diff --git a/ios/RNFileUploader.mm b/ios/RNFileUploader.mm index c0d0ae3b..2a2395c3 100644 --- a/ios/RNFileUploader.mm +++ b/ios/RNFileUploader.mm @@ -66,69 +66,58 @@ + (NSString *)moduleName #pragma mark - Exported methods -// v10 slice 1 ships the JS layer alone. Every queue method rejects with this -// code until slice 3 builds the iOS queue and executor. -static NSString *const kNotImplemented = @"E_NOT_IMPLEMENTED"; +// Each method is a one-line forward to the Swift engine, which hops onto its +// own serial queue and returns. getRequests is the one synchronous method: it +// reads the in-memory index under a lock. -static void RejectNotImplemented(RCTPromiseRejectBlock reject, NSString *method) -{ - reject(kNotImplemented, - [NSString stringWithFormat:@"RNFileUploader.%@: the iOS queue is not built yet", method], - nil); -} - -// configure() carries { lifetimeMs, retry, ...androidNotificationConfig }. iOS -// background uploads have no library-owned notification, and slice 3 persists -// the queue settings. Thus there is nothing to save yet. - (void)configure:(NSDictionary *)options { + [RNBackgroundUpload.shared configure:options]; } - (void)enqueue:(NSDictionary *)entry resolve:(RCTPromiseResolveBlock)resolve reject:(RCTPromiseRejectBlock)reject { - RejectNotImplemented(reject, @"enqueue"); + [RNBackgroundUpload.shared enqueue:entry resolve:resolve reject:reject]; } - (void)pause:(RCTPromiseResolveBlock)resolve reject:(RCTPromiseRejectBlock)reject { - RejectNotImplemented(reject, @"pause"); + [RNBackgroundUpload.shared pause:resolve reject:reject]; } - (void)resume:(RCTPromiseResolveBlock)resolve reject:(RCTPromiseRejectBlock)reject { - RejectNotImplemented(reject, @"resume"); + [RNBackgroundUpload.shared resume:resolve reject:reject]; } - (void)cancel:(NSString *)id resolve:(RCTPromiseResolveBlock)resolve reject:(RCTPromiseRejectBlock)reject { - RejectNotImplemented(reject, @"cancel"); + [RNBackgroundUpload.shared cancel:id resolve:resolve reject:reject]; } - (void)setWifiOnly:(BOOL)enabled resolve:(RCTPromiseResolveBlock)resolve reject:(RCTPromiseRejectBlock)reject { - RejectNotImplemented(reject, @"setWifiOnly"); + [RNBackgroundUpload.shared setWifiOnly:enabled resolve:resolve reject:reject]; } - (void)updateHeaders:(NSDictionary *)patch resolve:(RCTPromiseResolveBlock)resolve reject:(RCTPromiseRejectBlock)reject { - RejectNotImplemented(reject, @"updateHeaders"); + [RNBackgroundUpload.shared updateHeaders:patch resolve:resolve reject:reject]; } -// Synchronous. The live rows of the v10 queue. Slice 3 serializes them from -// the in-memory index; until then the queue is empty. - (NSArray *)getRequests { - return @[]; + return [RNBackgroundUpload.shared getRequests]; } - (void)getUnacknowledgedEvents:(RCTPromiseResolveBlock)resolve @@ -146,7 +135,7 @@ - (void)ackEvents:(NSArray *)ids #pragma mark - RNFileUploaderEventDelegate -// Called synchronously on the URLSession delegate queue. That is safe and +// Called on the engine's serial queue. That is safe and // deliberate: the generated emitter locks its own state and dispatches each // listener through the JS CallInvoker, so it is already thread-safe and already // async onto the JS thread. Deferring to the main queue instead would open a @@ -167,25 +156,24 @@ - (void)safeEmit:(void (^)(RNFileUploader *emitter))block } } -// The v9 Swift engine still reports through the delegate. The v10 spec has no -// per-outcome emitters and a different progress shape ({ id, bytesSent, -// totalBytes }), so until slice 3 rewires the engine to onState/onProgress/ -// onSettled, the live v9 payloads are dropped here. Terminal outcomes are -// journaled first, so nothing durable is lost. -- (void)emitProgress:(NSDictionary *)body +- (void)emitState:(NSDictionary *)body { + [self safeEmit:^(RNFileUploader *m) { [m emitOnState:body]; }]; } -- (void)emitCompleted:(NSDictionary *)body +- (void)emitProgress:(NSDictionary *)body { + [self safeEmit:^(RNFileUploader *m) { [m emitOnProgress:body]; }]; } -- (void)emitError:(NSDictionary *)body +- (void)emitAttempt:(NSDictionary *)body { + [self safeEmit:^(RNFileUploader *m) { [m emitOnAttempt:body]; }]; } -- (void)emitCancelled:(NSDictionary *)body +- (void)emitSettled:(NSDictionary *)body { + [self safeEmit:^(RNFileUploader *m) { [m emitOnSettled:body]; }]; } @end diff --git a/ios/RequestIndex.swift b/ios/RequestIndex.swift new file mode 100644 index 00000000..e7b7a220 --- /dev/null +++ b/ios/RequestIndex.swift @@ -0,0 +1,87 @@ +import Foundation + +/// The in-memory copy of every stored entry. getRequests() reads it +/// synchronously from the JS thread, so it never touches the disk. The +/// coordinator queue writes it after every store write. Guarded by a lock. +final class RequestIndex { + private struct Item { + let entry: QueueEntry + /// varsJSON decoded once, at load or upsert. + let vars: Any + } + + private let lock = NSLock() + private var items: [String: Item] = [:] + + func load(_ entries: [QueueEntry]) { + lock.lock() + defer { lock.unlock() } + items = [:] + for e in entries { items[e.id] = Item(entry: e, vars: JSONText.decode(e.varsJSON)) } + } + + func upsert(_ entry: QueueEntry) { + lock.lock() + defer { lock.unlock() } + // Decode again only when vars changed. + if let old = items[entry.id], old.entry.varsJSON == entry.varsJSON { + items[entry.id] = Item(entry: entry, vars: old.vars) + } else { + items[entry.id] = Item(entry: entry, vars: JSONText.decode(entry.varsJSON)) + } + } + + /// Live progress while an entry runs: memory only, no save, no `state` + /// event. The next commit of the entry saves it. + func setBytes(_ id: String, _ bytesSent: Int64) { + lock.lock() + defer { lock.unlock() } + guard let item = items[id] else { return } + var entry = item.entry + entry.bytesSent = bytesSent + items[id] = Item(entry: entry, vars: item.vars) + } + + func remove(_ id: String) { + lock.lock() + defer { lock.unlock() } + items[id] = nil + } + + func entry(_ id: String) -> QueueEntry? { + lock.lock() + defer { lock.unlock() } + return items[id]?.entry + } + + /// One RequestRow, with the cached vars. + func row(_ id: String) -> [String: Any]? { + lock.lock() + defer { lock.unlock() } + return items[id].map { $0.entry.row(vars: $0.vars) } + } + + /// Every entry, oldest first. + func entries() -> [QueueEntry] { + lock.lock() + defer { lock.unlock() } + return items.values.map(\.entry).sorted(by: Self.order) + } + + /// The RequestRow dictionaries, oldest first, for a stable order. + func rows() -> [[String: Any]] { + lock.lock() + defer { lock.unlock() } + return items.values.sorted { Self.order($0.entry, $1.entry) }.map { $0.entry.row(vars: $0.vars) } + } + + var count: Int { + lock.lock() + defer { lock.unlock() } + return items.count + } + + private static func order(_ a: QueueEntry, _ b: QueueEntry) -> Bool { + a.createdAt != b.createdAt ? a.createdAt < b.createdAt : a.id < b.id + } +} diff --git a/ios/RetryClassifier.swift b/ios/RetryClassifier.swift new file mode 100644 index 00000000..7412fa6c --- /dev/null +++ b/ios/RetryClassifier.swift @@ -0,0 +1,88 @@ +import Foundation + +/// Decides what one attempt's result means: the spec's retry, auth and +/// lifetime table (section 6.1). Pure: no I/O, no session state. +enum RetryClassifier { + enum Class: Equatable { + case accepted + case transient + case auth + case terminalHttp + /// The payload is gone. A retry can never succeed. + case fileMissing + case expired + } + + struct Input { + var statusCode: Int? + var body: String? + var error: NSError? + var accept: [UploadOutcome.AcceptRule] + var policy: RetryPolicy + var fileExists: Bool + var now: Double + var expiresAt: Double + } + + /// A cancellation (NSURLErrorCancelled) never reaches here: the caller + /// handles it from the task's recorded purpose. + /// Past expiresAt only a transient result becomes `expired`; a real + /// response keeps its own class. + static func classify(_ i: Input) -> Class { + let verdict = classifyResult(i) + return verdict == .transient && i.now >= i.expiresAt ? .expired : verdict + } + + private static func classifyResult(_ i: Input) -> Class { + if i.error == nil, let code = i.statusCode, + UploadOutcome.isAccepted(code, body: i.body, accept: i.accept) { + return .accepted + } + if let error = i.error { + // A file that exists but cannot be read now (iOS before first unlock) + // is transient. + if errorKind(for: error) == "file" { return i.fileExists ? .transient : .fileMissing } + return .transient + } + guard let code = i.statusCode else { return .transient } + switch code { + case 401, 403: return .auth + case 408, 429, 500...599: return .transient + case 400...499: return i.policy.exempt.contains(code) ? .transient : .terminalHttp + // 1xx and 3xx the session did not follow. The answer will not change. + default: return .terminalHttp + } + } + + /// Jittered exponential backoff. `attempt` is 1-based: 1 gives about base. + /// min(base * 2^(attempt-1), max), times (1 + jitter * (2r - 1)), >= 0. + static func backoffMs(attempt: Int, policy: RetryPolicy, random: () -> Double) -> Int { + let exponent = Double(min(max(attempt - 1, 0), 40)) + let raw = min(policy.baseMs * pow(2, exponent), policy.maxMs) + let jittered = raw * (1 + policy.jitter * (2 * random() - 1)) + return Int(max(jittered, 0).rounded()) + } + + /// The errorKind taxonomy shared with Android: a missing or unreadable + /// source file is 'file'; other URL-domain errors are 'network'; anything + /// else is 'unknown'. + static func errorKind(for error: NSError) -> String { + switch (error.domain, error.code) { + case (NSURLErrorDomain, NSURLErrorFileDoesNotExist), + (NSURLErrorDomain, NSURLErrorCannotOpenFile), + (NSURLErrorDomain, NSURLErrorNoPermissionsToReadFile), + (NSCocoaErrorDomain, NSFileNoSuchFileError), + (NSCocoaErrorDomain, NSFileReadNoSuchFileError), + (NSCocoaErrorDomain, NSFileReadNoPermissionError): + return "file" + case (NSURLErrorDomain, _): + return "network" + default: + return "unknown" + } + } + + static func isCancellation(_ error: NSError?) -> Bool { + error?.domain == NSURLErrorDomain && error?.code == NSURLErrorCancelled + } +} diff --git a/ios/TaskMap.swift b/ios/TaskMap.swift index 2748b39d..130e950b 100644 --- a/ios/TaskMap.swift +++ b/ios/TaskMap.swift @@ -1,45 +1,50 @@ import Foundation -// Durable ":" -> { id, accept, partIndex } mapping. -// -// Apple documents `taskDescription` only as an uninterpreted app string with no -// guarantee it survives process death, and DTS guidance is to persist task -// metadata externally keyed by the (stable) taskIdentifier. taskDescription -// stays the primary id. This map is the durable fallback. Thus a task -// observed after a relaunch is never orphaned under an unknown id, and the -// accept rules are still known when a task completes after the original -// startUpload options are gone. Chunked part tasks carry `partIndex`. Their -// accept rules live in the manifest, so `accept` is nil for them. -// -// Synchronous serial-queue access; a single JSON file. -enum TaskMap { - struct Meta: Codable { +/// Durable ":" -> Meta mapping. +/// +/// Apple documents `taskDescription` only as an app string with no promise +/// that it survives process death, and DTS guidance is to persist task +/// metadata keyed by the stable taskIdentifier. taskDescription stays the +/// primary owner; this map is the durable fallback. Both are written before +/// the task resumes. +/// +/// It also records why the library cancelled a task (`purpose`), so the +/// NSURLErrorCancelled callback can tell a pause or a supersede (no outcome) +/// from a cancel the system made (a transient retry). +/// +/// One JSON file, cached in memory, written through on every change. +final class TaskMap { + enum Purpose: String, Codable { + case attempt + case pause + case superseded + } + + struct Meta: Codable, Equatable { let id: String - // All are optional. Thus entries that older builds persisted still - // decode. - var accept: [UploadOutcome.AcceptRule]? var partIndex: Int? - // Chunked part tasks only: the manifest incarnation that the task was - // created under. It mirrors the taskDescription encoding (see - // ChunkedEngine). var incarnation: String? + var attempt: Int? + var requestId: String? + var headerGeneration: Int? + var generation: Int? + var purpose: Purpose? - init(id: String, accept: [UploadOutcome.AcceptRule]?, partIndex: Int?, - incarnation: String? = nil) { + init(id: String, partIndex: Int? = nil, incarnation: String? = nil, attempt: Int? = nil, + requestId: String? = nil, headerGeneration: Int? = nil, generation: Int? = nil, + purpose: Purpose? = nil) { self.id = id - self.accept = accept self.partIndex = partIndex self.incarnation = incarnation + self.attempt = attempt + self.requestId = requestId + self.headerGeneration = headerGeneration + self.generation = generation + self.purpose = purpose } private enum CodingKeys: String, CodingKey { - case id, accept, partIndex, incarnation - // Earlier builds persisted `acceptStatus: [Int]` where this build - // persists `accept` rules. The key is read, and never written. Thus a - // task that an older build enqueued keeps its accept rules when it - // completes under this build. This is the same legacy mapping as - // Android's Upload.normalized(). - case acceptStatus + case id, partIndex, incarnation, attempt, requestId, headerGeneration, generation, purpose } init(from decoder: Decoder) throws { @@ -47,59 +52,94 @@ enum TaskMap { id = try c.decode(String.self, forKey: .id) partIndex = try c.decodeIfPresent(Int.self, forKey: .partIndex) incarnation = try c.decodeIfPresent(String.self, forKey: .incarnation) - accept = try c.decodeIfPresent([UploadOutcome.AcceptRule].self, forKey: .accept) - ?? c.decodeIfPresent([Int].self, forKey: .acceptStatus)? - .map { UploadOutcome.AcceptRule(status: $0, bodyIncludes: nil) } - } - - func encode(to encoder: Encoder) throws { - var c = encoder.container(keyedBy: CodingKeys.self) - try c.encode(id, forKey: .id) - try c.encodeIfPresent(accept, forKey: .accept) - try c.encodeIfPresent(partIndex, forKey: .partIndex) - try c.encodeIfPresent(incarnation, forKey: .incarnation) + attempt = try c.decodeIfPresent(Int.self, forKey: .attempt) + requestId = try c.decodeIfPresent(String.self, forKey: .requestId) + headerGeneration = try c.decodeIfPresent(Int.self, forKey: .headerGeneration) + generation = try c.decodeIfPresent(Int.self, forKey: .generation) + // An unknown purpose from a newer build reads as nil. + purpose = (try? c.decodeIfPresent(String.self, forKey: .purpose)).flatMap { Purpose(rawValue: $0) } } } - private static let queue = DispatchQueue(label: "ai.openspace.rnbgupload.taskmap") + static let shared = TaskMap( + fileURL: FileIO.applicationSupport().appendingPathComponent("RNFileUploaderTaskMap.json")) + + let fileURL: URL + private let queue = DispatchQueue(label: "ai.openspace.rnbgupload.taskmap") + private var cache: [String: Meta] + + init(fileURL: URL) { + self.fileURL = fileURL + try? FileManager.default.createDirectory( + at: fileURL.deletingLastPathComponent(), withIntermediateDirectories: true) + cache = Self.read(fileURL) + } static func key(_ session: URLSession, _ task: URLSessionTask) -> String { "\(session.configuration.identifier ?? ""):\(task.taskIdentifier)" } - private static var fileURL: URL { - let base = FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask)[0] - try? FileManager.default.createDirectory(at: base, withIntermediateDirectories: true) - return base.appendingPathComponent("RNFileUploaderTaskMap.json") + func set(_ meta: Meta, forKey key: String) { + queue.sync { + cache[key] = meta + write() + } + } + + func meta(forKey key: String) -> Meta? { + queue.sync { cache[key] } } - static func set(_ meta: Meta, forKey key: String) { + func removeKey(_ key: String) { queue.sync { - var map = read() - map[key] = meta - write(map) + guard cache.removeValue(forKey: key) != nil else { return } + write() } } - static func meta(forKey key: String) -> Meta? { - queue.sync { read()[key] } + /// Records why the library is about to cancel a task. Creates the entry + /// when the task has none (a v9 task). + func setPurpose(_ purpose: Purpose, forKey key: String, id: String) { + queue.sync { + var meta = cache[key] ?? Meta(id: id) + meta.purpose = purpose + cache[key] = meta + write() + } } - static func removeKey(_ key: String) { + func setHeaderGeneration(_ generation: Int, forKey key: String) { queue.sync { - var map = read() - map.removeValue(forKey: key) - write(map) + guard cache[key] != nil else { return } + cache[key]?.headerGeneration = generation + write() } } - private static func read() -> [String: Meta] { - guard let data = try? Data(contentsOf: fileURL) else { return [:] } - return (try? JSONDecoder().decode([String: Meta].self, from: data)) ?? [:] + func keys(where predicate: (Meta) -> Bool) -> [String] { + queue.sync { cache.filter { predicate($0.value) }.map(\.key) } } - private static func write(_ map: [String: Meta]) { - guard let data = try? JSONEncoder().encode(map) else { return } - try? data.write(to: fileURL, options: .atomic) + func removeAll(where predicate: (String, Meta) -> Bool) { + queue.sync { + let before = cache.count + cache = cache.filter { !predicate($0.key, $0.value) } + if cache.count != before { write() } + } + } + + // Caller holds `queue`. + private func write() { + guard let data = try? JSONEncoder().encode(cache) else { return } + do { + try FileIO.writeAtomically(data, to: fileURL) + } catch { + NSLog("[RNFileUploader] task map write failed: \(error.localizedDescription)") + } + } + + private static func read(_ url: URL) -> [String: Meta] { + guard let data = try? Data(contentsOf: url) else { return [:] } + return (try? JSONDecoder().decode([String: Meta].self, from: data)) ?? [:] } } diff --git a/ios/Tests/BodyStagingTests.swift b/ios/Tests/BodyStagingTests.swift new file mode 100644 index 00000000..b5abc7cf --- /dev/null +++ b/ios/Tests/BodyStagingTests.swift @@ -0,0 +1,120 @@ +import XCTest +@testable import RNBGUCore + +final class BodyStagingTests: XCTestCase { + private var root: URL! + private var dir: URL! + + override func setUp() { + root = makeTempDir() + dir = root.appendingPathComponent("entry") + } + + override func tearDown() { + try? FileManager.default.removeItem(at: root) + } + + private func part(_ s: Int64, _ e: Int64) -> QueueEntry.Part { + QueueEntry.Part(url: "https://s3.test", headers: [:], start: s, end: e, accepted: false, rejections: 0) + } + + func testDataBodyIsCanonicalJSON() throws { + let staged = try BodyStaging.stage(.data(json: #"{"a":1}"#), parts: [], into: dir, fallbackBlob: nil) + XCTAssertEqual(staged.contentType, "application/json") + XCTAssertFalse(staged.forceContentType) + XCTAssertTrue(staged.relativePath.hasPrefix("body-")) + XCTAssertEqual(try String(contentsOf: dir.appendingPathComponent(staged.relativePath)), #"{"a":1}"#) + XCTAssertEqual(staged.totalBytes, 7) + XCTAssertFalse(FileIO.exists(dir.appendingPathComponent(staged.relativePath + ".tmp"))) + } + + func testBodilessIsZeroBytes() throws { + let staged = try BodyStaging.stage(.none, parts: [], into: dir, fallbackBlob: nil) + XCTAssertEqual(FileIO.size(dir.appendingPathComponent(staged.relativePath)), 0) + XCTAssertNil(staged.contentType) + } + + func testFormBodyParsesBack() throws { + let photo = root.appendingPathComponent("photo.jpg") + writeFile(photo, "JPEGBYTES") + let staged = try BodyStaging.stage(.form([ + .init(name: "request", contentType: "application/json", string: #"{"id":"p1"}"#, path: nil, fileName: nil), + .init(name: "image", contentType: "image/jpeg", string: nil, path: "file://" + photo.path, fileName: nil), + ]), parts: [], into: dir, fallbackBlob: nil) + XCTAssertTrue(staged.forceContentType) + let ct = try XCTUnwrap(staged.contentType) + XCTAssertTrue(ct.hasPrefix("multipart/form-data; boundary=")) + let boundary = String(ct.dropFirst("multipart/form-data; boundary=".count)) + let text = try String(contentsOf: dir.appendingPathComponent(staged.relativePath)) + let sections = text.components(separatedBy: "--\(boundary)") + XCTAssertEqual(sections.count, 4) // preamble, two fields, closing + XCTAssertTrue(sections[1].contains(#"Content-Disposition: form-data; name="request""#)) + XCTAssertTrue(sections[1].contains("\r\n\r\n{\"id\":\"p1\"}\r\n")) + XCTAssertTrue(sections[2].contains(#"name="image"; filename="photo.jpg""#), "filename defaults to the last path component") + XCTAssertTrue(sections[2].contains("Content-Type: image/jpeg\r\n\r\nJPEGBYTES\r\n")) + XCTAssertEqual(sections[3], "--\r\n") + XCTAssertEqual(staged.totalBytes, Int64(text.utf8.count)) + } + + func testMissingFormFileThrowsBeforeAnyWrite() { + XCTAssertThrowsError(try BodyStaging.stage(.form([ + .init(name: "image", contentType: "image/jpeg", string: nil, path: "/nope/missing.jpg", fileName: nil), + ]), parts: [], into: dir, fallbackBlob: nil)) { error in + XCTAssertEqual(error as? StagingError, .fileMissing("/nope/missing.jpg")) + } + let files = (try? FileManager.default.contentsOfDirectory(atPath: dir.path)) ?? [] + XCTAssertTrue(files.isEmpty) + } + + func testFileIsCopied() throws { + let src = root.appendingPathComponent("a.bin") + writeFile(src, bytes: 64) + let staged = try BodyStaging.stage(.file(path: src.path), parts: [], into: dir, fallbackBlob: nil) + XCTAssertEqual(staged.totalBytes, 64) + XCTAssertTrue(FileIO.exists(src), "a single file body is copied, not moved") + XCTAssertEqual(try Data(contentsOf: src), try Data(contentsOf: dir.appendingPathComponent(staged.relativePath))) + XCTAssertThrowsError(try BodyStaging.stage(.file(path: "/nope"), parts: [], into: dir, fallbackBlob: nil)) { + XCTAssertEqual($0 as? StagingError, .fileMissing("/nope")) + } + } + + func testPartsMoveTheSource() throws { + let src = root.appendingPathComponent("video.mp4") + writeFile(src, bytes: 20) + let staged = try BodyStaging.stage(.parts(file: src.path), parts: [part(0, 10), part(10, 20)], + into: dir, fallbackBlob: nil) + XCTAssertFalse(FileIO.exists(src), "a chunked file is moved") + XCTAssertTrue(staged.relativePath.hasPrefix("blob-")) + XCTAssertEqual(staged.totalBytes, 20) + XCTAssertFalse(staged.adopted) + } + + func testPartsTilingErrorLeavesTheSource() { + let src = root.appendingPathComponent("video.mp4") + writeFile(src, bytes: 20) + XCTAssertThrowsError(try BodyStaging.stage(.parts(file: src.path), parts: [part(0, 10)], into: dir, + fallbackBlob: nil)) { error in + guard case .invalid = error as? StagingError else { return XCTFail("expected invalid, got \(error)") } + } + XCTAssertTrue(FileIO.exists(src), "the check runs before the move") + } + + // A crash between the move and the entry save left the bytes as a blob. + func testPartsAdoptExistingBlobAfterACrash() throws { + writeFile(dir.appendingPathComponent("blob-crashed"), bytes: 20) + let staged = try BodyStaging.stage(.parts(file: root.appendingPathComponent("gone.mp4").path), + parts: [part(0, 20)], into: dir, fallbackBlob: "blob-crashed") + XCTAssertEqual(staged.relativePath, "blob-crashed") + XCTAssertTrue(staged.adopted) + XCTAssertThrowsError(try BodyStaging.stage(.parts(file: "/gone"), parts: [part(0, 20)], into: dir, + fallbackBlob: nil)) { + XCTAssertEqual($0 as? StagingError, .fileMissing("/gone")) + } + } + + func testFileURLAcceptsBothForms() { + XCTAssertEqual(BodyStaging.fileURL("/var/a b.txt").path, "/var/a b.txt") + XCTAssertEqual(BodyStaging.fileURL("file:///var/a%20b.txt").path, "/var/a b.txt") + XCTAssertEqual(BodyStaging.fileURL("file:///var/a b.txt").path, "/var/a b.txt") + } +} diff --git a/ios/Tests/CoordinatorChunkedTests.swift b/ios/Tests/CoordinatorChunkedTests.swift new file mode 100644 index 00000000..2c304141 --- /dev/null +++ b/ios/Tests/CoordinatorChunkedTests.swift @@ -0,0 +1,348 @@ +import XCTest +@testable import RNBGUCore + +/// Chunked entries: window, accept, part failures, recreate, parking. +final class CoordinatorChunkedTests: XCTestCase { + private var h: Harness! + + override func setUp() { + h = Harness() + h.boot() + } + + override func tearDown() { + try? FileManager.default.removeItem(at: h.root) + } + + private func partTask(_ index: Int) -> FakeTask? { + h.transport.live.first { ChunkedEngine.parsePartDescription($0.taskDescription)?.part == index } + } + + func testWindowOfThreeAndCompletion() throws { + let src = h.root.appendingPathComponent("capture.mp4") + writeFile(src, bytes: 50) + _ = try h.enqueue(h.chunkedRaw(id: "cap", size: 50, parts: 5, source: src)).get() + XCTAssertFalse(FileIO.exists(src), "moved into the library") + XCTAssertEqual(h.sink.stateNames, ["queued", "running"]) + XCTAssertEqual(h.transport.live.count, 3, "window of 3") + let first = try XCTUnwrap(partTask(0)) + XCTAssertEqual(first.request.httpMethod, "PUT") + XCTAssertEqual(first.header("Content-Range"), "0-9/50") + XCTAssertEqual(first.header("Content-Type"), "video/mp4", "descriptor headers under part headers") + XCTAssertNotNil(first.header("X-Request-Id")) + XCTAssertEqual(FileIO.size(first.file!), 10) + + h.complete(first) + XCTAssertEqual(h.transport.live.count, 3, "refilled") + XCTAssertNotNil(partTask(3)) + XCTAssertEqual(h.sink.progress.last?["bytesSent"] as? Int64, 10) + XCTAssertEqual(h.entry("cap")?.parts[0].accepted, true) + XCTAssertEqual(h.store.load("cap")?.parts[0].accepted, true, "accept is persisted") + + for i in 1...4 { h.complete(try XCTUnwrap(partTask(i))) } + XCTAssertEqual(h.entry("cap")?.state, .completed) + let settled = try XCTUnwrap(h.sink.settled.last) + XCTAssertEqual(settled["partIndex"] as? Int, 4) + XCTAssertEqual(settled["url"] as? String, "https://s3.test/part5") + XCTAssertEqual(settled["method"] as? String, "PUT") + XCTAssertEqual(settled["attempts"] as? Int, 5) + XCTAssertNil((settled["response"] as? [String: Any])?["status"], "no single response") + XCTAssertEqual(h.sink.progress.last?["bytesSent"] as? Int64, 50) + h.ack([settled["eventId"] as! String]) + XCTAssertFalse(FileIO.exists(h.store.dir("cap"))) + } + + func testPartTheSessionCannotOpenTriesAgainLater() throws { + let src = h.root.appendingPathComponent("capture.mp4") + writeFile(src, bytes: 50) + h.transport.failUpload = { _ in NSError(domain: NSURLErrorDomain, code: NSURLErrorCannotOpenFile) } + _ = try h.enqueue(h.chunkedRaw(id: "cap", size: 50, parts: 5, source: src)).get() + XCTAssertTrue(h.transport.live.isEmpty) + XCTAssertTrue(h.sink.settled.isEmpty, "the blob is whole: not terminal") + h.transport.failUpload = nil + h.advance(4 * 3_600_000) + XCTAssertEqual(h.transport.live.count, 3, "the window fills again") + } + + func testPartTheSessionCannotOpenOverAShortBlobSettlesFile() throws { + let src = h.root.appendingPathComponent("capture.mp4") + writeFile(src, bytes: 50) + h.transport.failUpload = { [unowned self] _ in + let e = self.h.entry("cap")! + try? FileManager.default.removeItem(at: self.h.store.fileURL("cap", e.bodyPath ?? ChunkedManifestV9.blobName)) + return NSError(domain: NSURLErrorDomain, code: NSURLErrorCannotOpenFile) + } + _ = try h.enqueue(h.chunkedRaw(id: "cap", size: 50, parts: 5, source: src)).get() + XCTAssertEqual(h.entry("cap")?.state, .error) + let error = try XCTUnwrap(h.sink.settled.last?["error"] as? [String: Any]) + XCTAssertEqual(error["errorKind"] as? String, "file") + XCTAssertEqual(error["partIndex"] as? Int, 0) + } + + func testPart404WithEmptyExemptIsTerminalAndKeepsBytes() throws { + _ = try h.enqueue(h.chunkedRaw(id: "cap", extra: ["retry": ["terminalHttp": ["exempt": []]]])).get() + let second = try XCTUnwrap(partTask(1)) + let others = h.transport.live.filter { $0 !== second } + h.complete(second, status: 404, body: "NoSuchUpload") + let error = try XCTUnwrap(h.sink.settled.last?["error"] as? [String: Any]) + XCTAssertEqual(error["errorKind"] as? String, "http") + XCTAssertEqual(error["partIndex"] as? Int, 1) + XCTAssertEqual((error["response"] as? [String: Any])?["status"] as? Int, 404) + XCTAssertTrue(others.allSatisfy(\.cancelled), "the other parts stop") + let blob = try XCTUnwrap(h.entry("cap")?.bodyPath) + XCTAssertTrue(FileIO.exists(h.store.fileURL("cap", blob)), "bytes survive a non-completed terminal") + } + + // The part-404 recovery: a same-id enqueue with new parts over the moved bytes. + func testRecreateOverKeptBlobWhenTheSourceIsGone() throws { + _ = try h.enqueue(h.chunkedRaw(id: "cap", extra: ["retry": ["terminalHttp": ["exempt": []]]])).get() + h.complete(try XCTUnwrap(partTask(0))) + h.complete(try XCTUnwrap(partTask(1)), status: 404) + let old = try XCTUnwrap(h.entry("cap")) + let gone = h.root.appendingPathComponent("deleted-by-the-caller.mp4") + _ = try h.enqueue(h.chunkedRaw(id: "cap", source: gone, urlPrefix: "https://s3.test/new")).get() + let e = try XCTUnwrap(h.entry("cap")) + XCTAssertEqual(e.bodyPath, old.bodyPath, "the blob is kept") + XCTAssertNotEqual(e.incarnation, old.incarnation) + XCTAssertEqual(e.generation, old.generation + 1) + XCTAssertTrue(e.parts.allSatisfy { !$0.accepted }) + XCTAssertEqual(e.state, .running) + XCTAssertEqual(h.transport.live.count, 3) + XCTAssertEqual(partTask(0)?.request.url?.absoluteString, "https://s3.test/new1") + } + + func testResumeWithSamePartsKeepsAcceptedAndStartsNothingTwice() throws { + _ = try h.enqueue(h.chunkedRaw(id: "cap", size: 50, parts: 5)).get() + h.complete(try XCTUnwrap(partTask(0))) + let created = h.transport.created.count + _ = try h.enqueue(h.chunkedRaw(id: "cap", size: 50, parts: 5, source: h.root.appendingPathComponent("x"))).get() + XCTAssertEqual(h.transport.created.count, created, "a live resume starts no task") + XCTAssertEqual(h.entry("cap")?.parts[0].accepted, true) + } + + func testDifferentPartsWhileRunningReject() throws { + _ = try h.enqueue(h.chunkedRaw(id: "cap")).get() + guard case .failure(let e) = h.enqueue(h.chunkedRaw(id: "cap", urlPrefix: "https://other.test/")) else { + return XCTFail() + } + XCTAssertEqual(e.code, "E_RUNNING") + } + + func testTilingMismatchRejectsInvalidAndKeepsTheSource() throws { + let src = h.root.appendingPathComponent("short.mp4") + writeFile(src, bytes: 25) + guard case .failure(let e) = h.enqueue(h.chunkedRaw(id: "cap", size: 30, source: src)) else { return XCTFail() } + XCTAssertEqual(e.code, "E_INVALID") + XCTAssertTrue(FileIO.exists(src)) + } + + func testTransientPartRetryHoldsItsSlotWithADelayedTask() throws { + _ = try h.enqueue(h.chunkedRaw(id: "cap", size: 50, parts: 5)).get() + let first = try XCTUnwrap(partTask(0)) + h.complete(first, status: 500) + let retry = try XCTUnwrap(partTask(0)) + XCTAssertTrue(retry !== first) + XCTAssertNotNil(retry.beginAt) + XCTAssertEqual(h.transport.live.count, 3, "no extra part enters the window") + XCTAssertEqual(h.entry("cap")?.parts[0].rejections, 1) + XCTAssertEqual(h.entry("cap")?.state, .running) + } + + func testRowShowsNextAttemptAtWhenEveryPartWaits() throws { + _ = try h.enqueue(h.chunkedRaw(id: "cap", size: 30, parts: 3)).get() + h.complete(try XCTUnwrap(partTask(0)), status: 503) + XCTAssertNil(h.row("cap")?["nextAttemptAt"], "two parts still move") + h.clock += 100 + h.complete(try XCTUnwrap(partTask(1)), status: 503) + let states = h.sink.states.count + h.clock += 100 + h.complete(try XCTUnwrap(partTask(2)), status: 503) + let first = h.clock - 200 + 1_000 + XCTAssertEqual(h.row("cap")?["state"] as? String, "running") + XCTAssertEqual(h.row("cap")?["nextAttemptAt"] as? Double, first, "the earliest begin") + XCTAssertEqual(h.store.load("cap")?.nextAttemptAt, first) + XCTAssertEqual(h.sink.states.count, states + 1, "one state event") + // The first delayed part begins: the entry moves again. + let begun = try XCTUnwrap(partTask(0)) + XCTAssertNotNil(h.coordinator.taskWillBegin(key: begun.key, description: begun.taskDescription)) + XCTAssertNil(h.row("cap")?["nextAttemptAt"]) + } + + func testRelaunchRestoresNextAttemptAtFromTheDelayedPartTasks() throws { + _ = try h.enqueue(h.chunkedRaw(id: "cap", size: 30, parts: 3)).get() + for i in 0..<3 { h.complete(try XCTUnwrap(partTask(i)), status: 503) } + let waiting = h.transport.live + let fresh = Harness(root: h.root) + fresh.clock = h.clock + 200 + let survivors = waiting.map { + FakeTask(key: $0.key, description: $0.taskDescription, request: $0.request, beginAt: $0.beginAt) + } + fresh.relaunch(daemonTasks: survivors) + fresh.boot() + XCTAssertEqual(fresh.row("cap")?["nextAttemptAt"] as? Double, h.clock + 1_000) + // Progress from a part that began while the app was dead clears it. + fresh.coordinator.taskProgress(key: survivors[1].key, description: survivors[1].taskDescription, + sent: 1, expected: 10) + fresh.drain() + XCTAssertNil(fresh.row("cap")?["nextAttemptAt"]) + } + + func testFailedPartSaveCreatesNoTaskAndRefillsAfterABackoff() throws { + _ = try h.enqueue(h.chunkedRaw(id: "cap", size: 50, parts: 5)).get() + let dir = h.store.dir("cap") + setReadOnly(dir, true) + defer { setReadOnly(dir, false) } + let attempts = h.entry("cap")?.attempts + h.complete(try XCTUnwrap(partTask(0)), status: 500) + XCTAssertNil(partTask(0), "no task without the saved attempt") + XCTAssertEqual(h.transport.live.count, 2) + XCTAssertEqual(h.entry("cap")?.attempts, attempts) + setReadOnly(dir, false) + h.advance(1_000) + XCTAssertNotNil(partTask(0)) + XCTAssertEqual(h.store.load("cap")?.attempts, (attempts ?? 0) + 1) + } + + func testPartFileBuildFailureWithAnIntactBlobRefillsLater() throws { + _ = try h.enqueue(h.chunkedRaw(id: "cap", size: 50, parts: 5)).get() + let e = try XCTUnwrap(h.entry("cap")) + // A non-empty directory where part 3's file goes: the rename fails, as + // on a full disk. The blob is whole. + let blocker = h.store.partFileURL("cap", 3, incarnation: e.incarnation, start: 30, end: 40) + writeFile(blocker.appendingPathComponent("x"), "x") + h.complete(try XCTUnwrap(partTask(0))) + XCTAssertTrue(h.sink.settled.isEmpty, "not a file terminal") + XCTAssertEqual(h.entry("cap")?.state, .running) + XCTAssertNil(partTask(3)) + try FileManager.default.removeItem(at: blocker) + h.advance(1_000) + XCTAssertNotNil(partTask(3), "built and sent after the backoff") + } + + func testPart401ParksTheWholeEntryAndHeadersResumeIt() throws { + _ = try h.enqueue(h.chunkedRaw(id: "cap", size: 50, parts: 5)).get() + h.complete(try XCTUnwrap(partTask(0))) + let rejected = try XCTUnwrap(partTask(1)) + let others = h.transport.live.filter { $0 !== rejected } + h.complete(rejected, status: 401) + XCTAssertEqual(h.entry("cap")?.state, .awaitingAuth) + XCTAssertEqual(others.count, 2) + XCTAssertTrue(others.allSatisfy(\.cancelled), "the other in-flight parts stop") + XCTAssertTrue(h.transport.live.isEmpty) + h.updateHeaders(["Authorization": "fresh"]) + XCTAssertEqual(h.entry("cap")?.state, .running) + XCTAssertEqual(h.transport.live.count, 3) + XCTAssertTrue(h.transport.live.allSatisfy { $0.header("Authorization") == "fresh" }) + XCTAssertNil(partTask(0), "accepted parts are not sent again") + } + + func testPart401UnderAnOlderGenerationReissuesThePartAtOnce() throws { + _ = try h.enqueue(h.chunkedRaw(id: "cap", size: 50, parts: 5)).get() + let first = try XCTUnwrap(partTask(1)) + h.updateHeaders(["Authorization": "fresh"]) + h.complete(first, status: 401) + XCTAssertEqual(h.entry("cap")?.state, .running, "not parked") + let again = try XCTUnwrap(partTask(1)) + XCTAssertTrue(again !== first) + XCTAssertNil(again.beginAt, "no backoff") + XCTAssertEqual(again.header("Authorization"), "fresh") + XCTAssertEqual(h.transport.live.count, 3) + } + + func testUpdateHeadersReplacesAHeaderAPartCarries() throws { + var raw = h.chunkedRaw(id: "cap", size: 30, parts: 3) + var d = raw["descriptor"] as! [String: Any] + d["parts"] = (d["parts"] as! [[String: Any]]).map { p in + var p = p + var headers = p["headers"] as! [String: Any] + headers["authorization"] = "part-old" + p["headers"] = headers + return p + } + raw["descriptor"] = d + _ = try h.enqueue(raw).get() + h.complete(try XCTUnwrap(partTask(0)), status: 401) + XCTAssertEqual(h.entry("cap")?.state, .awaitingAuth) + h.updateHeaders(["Authorization": "fresh", "X-New": "1"]) + XCTAssertEqual(h.transport.live.count, 3) + XCTAssertTrue(h.transport.live.allSatisfy { $0.header("Authorization") == "fresh" }, "the part's own header too") + XCTAssertEqual(h.entry("cap")?.parts[0].headers["Authorization"], "fresh") + XCTAssertNil(h.entry("cap")?.parts[0].headers["authorization"], "one spelling") + XCTAssertNil(h.entry("cap")?.parts[0].headers["X-New"], "a name the part lacks stays on the entry") + XCTAssertNotNil(h.entry("cap")?.parts[0].headers["Content-Range"]) + } + + func testRowCarriesByteWeightedLiveProgress() throws { + _ = try h.enqueue(h.chunkedRaw(id: "cap", size: 50, parts: 5)).get() + h.complete(try XCTUnwrap(partTask(0))) + let running = try XCTUnwrap(partTask(1)) + h.coordinator.taskProgress(key: running.key, description: running.taskDescription, sent: 4, expected: 10) + h.drain() + XCTAssertEqual(h.row("cap")?["bytesSent"] as? Int64, 14) + } + + func testPauseKeepsAcceptedPartsAndResumeRefills() throws { + _ = try h.enqueue(h.chunkedRaw(id: "cap", size: 50, parts: 5)).get() + h.complete(try XCTUnwrap(partTask(0))) + h.pause() + XCTAssertTrue(h.transport.live.isEmpty) + XCTAssertEqual(h.entry("cap")?.state, .paused) + h.resume() + XCTAssertEqual(h.entry("cap")?.state, .running) + XCTAssertEqual(h.transport.live.count, 3) + XCTAssertNil(partTask(0)) + } + + func testCancelLiveChunked() throws { + _ = try h.enqueue(h.chunkedRaw(id: "cap", size: 50, parts: 5)).get() + h.complete(try XCTUnwrap(partTask(0))) + let live = h.transport.live + h.cancel("cap") + XCTAssertTrue(live.allSatisfy(\.cancelled)) + XCTAssertEqual(h.sink.settled.last?["kind"] as? String, "cancelled") + // A late accept from a cancelled part still records the bytes but reports nothing. + let settled = h.sink.settled.count + h.complete(live[0]) + XCTAssertEqual(h.sink.settled.count, settled) + h.ack([h.sink.settled.last!["eventId"] as! String]) + XCTAssertFalse(FileIO.exists(h.store.dir("cap"))) + } + + func testCancelLiveChunkedWhoseJournalWriteFailsKeepsThePartsRunning() throws { + _ = try h.enqueue(h.chunkedRaw(id: "cap", size: 50, parts: 5)).get() + h.complete(try XCTUnwrap(partTask(0))) + let live = h.transport.live + let states = h.sink.states.count + setReadOnly(h.journal.root, true) + XCTAssertEqual(h.cancel("cap")?.code, "E_STORAGE") + XCTAssertTrue(live.allSatisfy { !$0.cancelled }) + XCTAssertEqual(h.entry("cap")?.state, .running) + XCTAssertEqual(h.sink.states.count, states) + XCTAssertTrue(h.sink.settled.isEmpty) + // The window still refills: the chunked side did not stop. + h.complete(live[0]) + XCTAssertEqual(h.transport.live.count, 3) + setReadOnly(h.journal.root, false) + XCTAssertNil(h.cancel("cap")) + XCTAssertTrue(h.transport.live.isEmpty) + XCTAssertEqual(h.journal.unacknowledged().count, 1) + } + + func testLateCallbackFromAReplacedPlanIsDropped() throws { + _ = try h.enqueue(h.chunkedRaw(id: "cap", extra: ["retry": ["terminalHttp": ["exempt": []]]])).get() + let stale = try XCTUnwrap(partTask(2)) + h.complete(try XCTUnwrap(partTask(0)), status: 404) + _ = try h.enqueue(h.chunkedRaw(id: "cap", urlPrefix: "https://s3.test/v2-")).get() + h.complete(stale) // from the old incarnation + XCTAssertEqual(h.entry("cap")?.parts[2].accepted, false) + } + + func testShortBlobSettlesFile() throws { + _ = try h.enqueue(h.chunkedRaw(id: "cap", size: 30, parts: 3)).get() + let blob = h.store.fileURL("cap", h.entry("cap")!.bodyPath!) + try Data(count: 12).write(to: blob) + h.complete(try XCTUnwrap(partTask(0))) + let error = try XCTUnwrap(h.sink.settled.last?["error"] as? [String: Any]) + XCTAssertEqual(error["errorKind"] as? String, "file") + } +} diff --git a/ios/Tests/CoordinatorRelaunchTests.swift b/ios/Tests/CoordinatorRelaunchTests.swift new file mode 100644 index 00000000..0fc54e3a --- /dev/null +++ b/ios/Tests/CoordinatorRelaunchTests.swift @@ -0,0 +1,513 @@ +import XCTest +@testable import RNBGUCore + +/// Relaunch reconciliation and the v9 import. +final class CoordinatorRelaunchTests: XCTestCase { + private var h: Harness! + + override func setUp() { + h = Harness() + } + + override func tearDown() { + try? FileManager.default.removeItem(at: h.root) + } + + func testNothingIssuesBeforeReconcileThenReconcileIssues() throws { + _ = try h.enqueue(h.dataRaw(id: "a")).get() + XCTAssertTrue(h.transport.created.isEmpty, "not ready: nothing issues") + XCTAssertEqual(h.row("a")?["state"] as? String, "queued", "getRequests works before reconcile") + h.boot() + XCTAssertEqual(h.transport.live.count, 1) + XCTAssertEqual(h.entry("a")?.state, .running) + } + + func testRelaunchAdoptsTheLiveTaskAndItsCompletionSettles() throws { + h.boot() + _ = try h.enqueue(h.dataRaw(id: "a")).get() + let task = h.transport.live[0] + // New process: the daemon still holds the task. + let survivor = FakeTask(key: task.key, description: task.taskDescription, request: task.request, file: task.file) + let fresh = Harness(root: h.root) + fresh.relaunch(daemonTasks: [survivor]) + fresh.boot() + XCTAssertTrue(fresh.transport.created.isEmpty, "adopted, not re-issued") + XCTAssertEqual(fresh.row("a")?["state"] as? String, "running") + fresh.complete(survivor) + XCTAssertEqual(fresh.entry("a")?.state, .completed) + XCTAssertEqual(fresh.sink.settled.count, 1) + } + + func testRunningWithNoTaskAndNoTaskMapKeyReissuesNow() throws { + h.boot() + _ = try h.enqueue(h.dataRaw(id: "a")).get() + let task = h.transport.live[0] + h.map.removeKey(task.key) // as if the completion was handled, or the task never made it + let fresh = Harness(root: h.root) + fresh.boot() + XCTAssertEqual(fresh.transport.created.count, 1) + XCTAssertEqual(fresh.entry("a")?.attempts, 2) + } + + func testRunningWithAPendingCompletionWaitsForTheReplay() throws { + h.boot() + _ = try h.enqueue(h.dataRaw(id: "a")).get() + let task = h.transport.live[0] + let fresh = Harness(root: h.root) + fresh.boot() + XCTAssertTrue(fresh.transport.created.isEmpty, "the TaskMap key says a completion may be pending") + // The daemon replays the completion that happened while we were dead. + fresh.complete(FakeTask(key: task.key, description: task.taskDescription, request: task.request)) + XCTAssertEqual(fresh.entry("a")?.state, .completed) + fresh.advance(Double(QueueCoordinator.graceMs)) + XCTAssertTrue(fresh.transport.created.isEmpty, "no duplicate request") + } + + func testRunningWithALostTaskReissuesAfterTheGraceAndPrunesItsKey() throws { + h.boot() + _ = try h.enqueue(h.dataRaw(id: "a")).get() + let lost = h.transport.live[0] + let fresh = Harness(root: h.root) + fresh.boot() + XCTAssertTrue(fresh.transport.created.isEmpty) + fresh.advance(Double(QueueCoordinator.graceMs)) + XCTAssertEqual(fresh.transport.created.count, 1) + XCTAssertNil(fresh.map.meta(forKey: lost.key)) + XCTAssertTrue(fresh.map.keys(where: { $0.id == "a" && $0.attempt == 1 }).isEmpty, "\(lost.key) pruned") + // The next launch does not wait the grace again for the same lost task. + let third = Harness(root: h.root) + third.boot() + XCTAssertTrue(third.map.keys(where: { $0.attempt == 1 }).isEmpty) + } + + func testReconcileDropsKeysWhoseIdHasNoEntry() throws { + h.boot() + h.map.set(.init(id: "ghost", attempt: 1, generation: 1, purpose: .attempt), forKey: "any:77") + let live = FakeTask(key: "any:78", description: ChunkedEngine.taskDescription(id: "ghost2", attempt: 1, generation: 1)) + h.map.set(.init(id: "ghost2", attempt: 1, generation: 1, purpose: .attempt), forKey: live.key) + let fresh = Harness(root: h.root) + fresh.relaunch(daemonTasks: [live]) + fresh.boot() + XCTAssertNil(fresh.map.meta(forKey: "any:77")) + XCTAssertEqual(fresh.map.meta(forKey: live.key)?.purpose, .superseded, "a live task keeps its key for its cancel") + } + + // The delayed task never reached the daemon (a crash between the save and + // resume): no TaskMap key. It never ran, so it keeps its ordinal and id. + func testDelayedRetryThatNeverReachedTheDaemonKeepsItsWaitAndOrdinal() throws { + h.boot() + _ = try h.enqueue(h.dataRaw(id: "a")).get() + h.complete(h.transport.live[0], status: 503) + let waiting = h.transport.live[0] + h.map.removeKey(waiting.key) + let fresh = Harness(root: h.root) + fresh.clock = h.clock + 400 + fresh.boot() + let task = try XCTUnwrap(fresh.transport.live.first) + XCTAssertEqual(task.beginAt, Date(timeIntervalSince1970: (h.clock + 1_000) / 1000)) + XCTAssertEqual(fresh.entry("a")?.attempts, 2) + XCTAssertEqual(task.header("X-Request-Id"), waiting.header("X-Request-Id")) + } + + // The key says the delayed task may have run while the app was dead: wait + // for its replay, then mint a new attempt so a late replay is stale. + func testQueuedEntryWithAPendingKeyWaitsTheGraceThenMintsANewAttempt() throws { + h.boot() + _ = try h.enqueue(h.dataRaw(id: "a")).get() + h.complete(h.transport.live[0], status: 503) + let waiting = h.transport.live[0] + let fresh = Harness(root: h.root) + fresh.boot() + XCTAssertTrue(fresh.transport.created.isEmpty, "a replay may be pending") + fresh.advance(Double(QueueCoordinator.graceMs)) + let task = try XCTUnwrap(fresh.transport.live.first) + XCTAssertEqual(fresh.entry("a")?.attempts, 3) + XCTAssertNotEqual(task.header("X-Request-Id"), waiting.header("X-Request-Id")) + XCTAssertTrue(fresh.map.keys(where: { $0.id == "a" && $0.attempt == 2 }).isEmpty, "the lost task's key is pruned") + // A late replay of the lost attempt is dropped. + fresh.complete(FakeTask(key: waiting.key, description: waiting.taskDescription, request: waiting.request)) + XCTAssertEqual(fresh.entry("a")?.state, .running) + } + + // MARK: - Review fixes: lost saves, early completions, pending replays + + /// The journal write landed, the entry.json save did not. The relaunch + /// must not send the request again. + private func settleWithFailedSave(_ finish: (Harness, FakeTask) -> Void) throws -> String { + h.boot() + _ = try h.enqueue(h.dataRaw(id: "a")).get() + let task = h.transport.live[0] + let dir = h.store.dir("a") + setReadOnly(dir, true) + finish(h, task) + setReadOnly(dir, false) + XCTAssertEqual(h.store.load("a")?.state, .running, "the disk still says running") + return try XCTUnwrap(h.journal.unacknowledged().first?.eventId) + } + + func testSettleThenFailedSaveIsRepairedAtRelaunchAndNotSentAgain() throws { + let eventId = try settleWithFailedSave { h, task in h.complete(task) } + let fresh = Harness(root: h.root) + fresh.boot() + XCTAssertTrue(fresh.transport.created.isEmpty, "no second POST") + XCTAssertEqual(fresh.entry("a")?.state, .completed) + XCTAssertEqual(fresh.entry("a")?.settledEventId, eventId) + XCTAssertEqual(fresh.store.load("a")?.state, .completed, "the repair is saved") + fresh.advance(Double(QueueCoordinator.graceMs)) + XCTAssertTrue(fresh.transport.created.isEmpty) + XCTAssertEqual(fresh.unacknowledged().first?["eventId"] as? String, eventId) + fresh.ack([eventId]) + XCTAssertNil(fresh.row("a")) + } + + func testCancelThenFailedSaveDoesNotRunAgainAtRelaunch() throws { + let eventId = try settleWithFailedSave { h, _ in h.cancel("a") } + let fresh = Harness(root: h.root) + fresh.boot() + XCTAssertTrue(fresh.transport.created.isEmpty) + XCTAssertEqual(fresh.entry("a")?.state, .cancelled) + XCTAssertEqual(fresh.entry("a")?.settledEventId, eventId) + } + + /// A cancel on a settled entry that died after the directory was set + /// aside: the next launch finishes it. + func testRelaunchFinishesACancelThatDiedAfterTheSetAside() throws { + h.boot() + _ = try h.enqueue(h.dataRaw(id: "a")).get() + h.complete(h.transport.live[0], status: 400) + _ = try h.store.setAside("a") + let fresh = Harness(root: h.root) + fresh.boot() + XCTAssertNil(fresh.row("a")) + XCTAssertTrue(fresh.journal.unacknowledged().isEmpty, "its outcome goes with it") + XCTAssertTrue(fresh.store.setAsideDirectories().isEmpty) + } + + /// Same, but the outcome files still cannot be deleted: the row comes + /// back, so no outcome is left without its row. + func testRelaunchPutsASetAsideRowBackWhenItsEventsCannotBeDeleted() throws { + h.boot() + _ = try h.enqueue(h.dataRaw(id: "a")).get() + h.complete(h.transport.live[0], status: 400) + _ = try h.store.setAside("a") + setReadOnly(h.journal.root, true) + defer { setReadOnly(h.journal.root, false) } + let fresh = Harness(root: h.root) + fresh.boot() + XCTAssertEqual(fresh.row("a")?["state"] as? String, "error") + XCTAssertEqual(fresh.journal.unacknowledged().count, 1) + XCTAssertTrue(fresh.store.setAsideDirectories().isEmpty) + } + + func testAckBeforeReconcileForgetsARowWhoseSettleSaveFailed() throws { + let eventId = try settleWithFailedSave { h, task in h.complete(task) } + let fresh = Harness(root: h.root) // no boot: the ack runs first + fresh.ack([eventId]) + XCTAssertNil(fresh.row("a")) + XCTAssertFalse(FileIO.exists(fresh.store.dir("a"))) + fresh.boot() + XCTAssertTrue(fresh.transport.created.isEmpty) + } + + func testCompletionBeforeReconcileMintsTheNextAttempt() throws { + h.boot() + _ = try h.enqueue(h.dataRaw(id: "a")).get() + let ran = h.transport.live[0] + let survivor = FakeTask(key: ran.key, description: ran.taskDescription, request: ran.request) + let fresh = Harness(root: h.root) + fresh.transport.deferAllTasks = true + fresh.relaunch(daemonTasks: [survivor]) + fresh.coordinator.reconcileAll {} + fresh.drain() + fresh.complete(survivor, status: 503) // lands before reconcile + fresh.transport.releaseAllTasks() + fresh.drain() + fresh.drain() + let retry = try XCTUnwrap(fresh.transport.live.first) + XCTAssertEqual(fresh.entry("a")?.attempts, 2) + XCTAssertNotEqual(retry.header("X-Request-Id"), ran.header("X-Request-Id"), "a new attempt, a new id") + XCTAssertEqual(ChunkedEngine.parseRequestDescription(retry.taskDescription)?.attempt, 2) + XCTAssertNotNil(retry.beginAt, "it keeps the backoff") + } + + /// Parts 1 and 2 still live in the daemon; part 0 finished while the app + /// was dead, so its TaskMap key is there and its completion may replay. + private func chunkedRelaunchWithPart0Pending() throws -> (Harness, FakeTask) { + h.boot() + _ = try h.enqueue(h.chunkedRaw(id: "cap", size: 50, parts: 5)).get() + let parts = h.transport.live + let part0 = try XCTUnwrap(parts.first { ChunkedEngine.parsePartDescription($0.taskDescription)?.part == 0 }) + let survivors = parts.filter { $0 !== part0 } + .map { FakeTask(key: $0.key, description: $0.taskDescription, request: $0.request) } + let fresh = Harness(root: h.root) + fresh.relaunch(daemonTasks: survivors) + return (fresh, part0) + } + + func testChunkedRelaunchHoldsAPendingPartUntilItsReplay() throws { + let (fresh, part0) = try chunkedRelaunchWithPart0Pending() + var released = false + fresh.coordinator.reconcileAll { released = true } + fresh.drain() + fresh.drain() + XCTAssertTrue(fresh.transport.created.isEmpty, "no second PUT of part 0, and no part 3 past the window") + XCTAssertFalse(released, "the background completion handler waits for the grace") + let file = fresh.store.partFileURL("cap", 0, incarnation: fresh.entry("cap")!.incarnation, start: 0, end: 10) + XCTAssertTrue(FileIO.exists(file), "the part file stays for the replay") + fresh.complete(FakeTask(key: part0.key, description: part0.taskDescription, request: part0.request)) + XCTAssertEqual(fresh.entry("cap")?.parts[0].accepted, true) + XCTAssertEqual(fresh.transport.created.count, 1) + XCTAssertEqual(ChunkedEngine.parsePartDescription(fresh.transport.created[0].taskDescription)?.part, 3) + XCTAssertTrue(released, "the replay came, so the handler releases at once") + fresh.advance(Double(QueueCoordinator.graceMs)) + XCTAssertEqual(fresh.transport.created.count, 1, "the grace timer releases nothing") + } + + func testChunkedPendingPartWithNoReplayIsSentAfterTheGrace() throws { + let (fresh, part0) = try chunkedRelaunchWithPart0Pending() + fresh.boot() + XCTAssertTrue(fresh.transport.created.isEmpty) + fresh.advance(Double(QueueCoordinator.graceMs)) + let resent = try XCTUnwrap(fresh.transport.live.first { + ChunkedEngine.parsePartDescription($0.taskDescription)?.part == 0 }) + XCTAssertNotEqual(resent.key, part0.key) + XCTAssertNil(fresh.map.meta(forKey: part0.key), "the lost task's key is pruned") + // A late replay of the lost task: the part is accepted, and the + // duplicate PUT stops before its part file goes. + fresh.complete(FakeTask(key: part0.key, description: part0.taskDescription, request: part0.request)) + XCTAssertEqual(fresh.entry("cap")?.parts[0].accepted, true) + XCTAssertTrue(resent.cancelled) + } + + func testBackgroundCompletionWaitsForTheSimpleGraceAndNotLonger() throws { + h.boot() + _ = try h.enqueue(h.dataRaw(id: "a")).get() + let fresh = Harness(root: h.root) // the task's key is there, the task is not + var released = false + fresh.coordinator.reconcileAll { released = true } + fresh.drain() + fresh.drain() + XCTAssertFalse(released, "held while the replay may still come") + fresh.advance(Double(QueueCoordinator.graceMs)) + XCTAssertTrue(released) + XCTAssertEqual(fresh.transport.live.count, 1, "released after the re-issue") + + let idle = Harness(root: makeTempDir()) + defer { try? FileManager.default.removeItem(at: idle.root) } + var idleReleased = false + idle.coordinator.reconcileAll { idleReleased = true } + idle.drain() + idle.drain() + XCTAssertTrue(idleReleased, "no grace, no wait") + } + + func testBackgroundCompletionReleasesWhenTheSimpleReplayLands() throws { + h.boot() + _ = try h.enqueue(h.dataRaw(id: "a")).get() + let task = h.transport.live[0] + let fresh = Harness(root: h.root) + var released = false + fresh.coordinator.reconcileAll { released = true } + fresh.drain() + fresh.drain() + XCTAssertFalse(released) + fresh.complete(FakeTask(key: task.key, description: task.taskDescription, request: task.request)) + XCTAssertEqual(fresh.entry("a")?.state, .completed) + XCTAssertTrue(released, "the replay came, so the handler does not wait out the grace") + fresh.advance(Double(QueueCoordinator.graceMs)) + XCTAssertTrue(fresh.transport.created.isEmpty, "the grace timer sends nothing") + } + + func testAReplayThatLeavesTheEntryAloneReissuesAtOnce() throws { + h.boot() + _ = try h.enqueue(h.dataRaw(id: "a")).get() + let task = h.transport.live[0] + let fresh = Harness(root: h.root) + fresh.boot() + XCTAssertTrue(fresh.transport.created.isEmpty) + // The replay was superseded, so it does not move the entry. The attempt + // is lost, and the wait for it is over. + fresh.map.setPurpose(.superseded, forKey: task.key, id: "a") + fresh.complete(FakeTask(key: task.key, description: task.taskDescription, request: task.request), + status: 503) + XCTAssertEqual(fresh.transport.live.count, 1, "re-issued without waiting out the grace") + XCTAssertEqual(fresh.entry("a")?.attempts, 2, "a new ordinal: the lost attempt may have run") + fresh.advance(Double(QueueCoordinator.graceMs)) + XCTAssertEqual(fresh.transport.created.count, 1, "the timer does not send it again") + } + + func testAnEndedGraceTimerDoesNotCloseANewerWait() throws { + h.boot() + _ = try h.enqueue(h.dataRaw(id: "a")).get() + let first = h.transport.live[0] + let fresh = Harness(root: h.root) + fresh.boot() // wait A opens, its timer is due at graceMs + fresh.complete(FakeTask(key: first.key, description: first.taskDescription, request: first.request), + status: 503) // the replay ends wait A; the retry is attempt 2 + let retry = try XCTUnwrap(fresh.transport.live.first) + fresh.advance(Double(QueueCoordinator.graceMs / 2)) + // The retry finished while the app was dead: its key is there, the task + // is not. A second reconcile opens wait B for it. + retry.isLive = false + var released = false + fresh.coordinator.reconcileAll { released = true } + fresh.drain() + fresh.drain() + XCTAssertFalse(released) + fresh.advance(Double(QueueCoordinator.graceMs / 2)) // wait A's timer fires + XCTAssertFalse(released, "wait A's timer does not close wait B") + fresh.advance(Double(QueueCoordinator.graceMs / 2)) + XCTAssertTrue(released) + } + + func testUnownedAndStaleTasksAreCancelled() throws { + let v9 = FakeTask(key: "any:90", description: "bare-v9-id") + let orphan = FakeTask(key: "any:91", description: ChunkedEngine.taskDescription(id: "gone", attempt: 1, generation: 1)) + h.relaunch(daemonTasks: [v9, orphan]) + h.boot() + XCTAssertTrue(v9.cancelled) + XCTAssertTrue(orphan.cancelled) + XCTAssertEqual(h.map.meta(forKey: "any:90")?.purpose, .superseded) + h.complete(v9, error: NSError(domain: NSURLErrorDomain, code: NSURLErrorCancelled)) + XCTAssertTrue(h.sink.attempts.isEmpty) + XCTAssertTrue(h.sink.settled.isEmpty) + } + + func testChunkedRelaunchAdoptsLivePartsAndCancelsDuplicates() throws { + h.boot() + _ = try h.enqueue(h.chunkedRaw(id: "cap", size: 50, parts: 5)).get() + let parts = h.transport.live + let fresh = Harness(root: h.root) + let survivors = parts.map { FakeTask(key: $0.key, description: $0.taskDescription, request: $0.request) } + let duplicate = FakeTask(key: "any:99", description: parts[0].taskDescription) + fresh.relaunch(daemonTasks: survivors + [duplicate]) + fresh.boot() + XCTAssertTrue(duplicate.cancelled, "never two tasks for one part") + XCTAssertTrue(fresh.transport.created.isEmpty, "the window is full with the adopted tasks") + fresh.complete(survivors[0]) + XCTAssertEqual(fresh.transport.created.count, 1, "refill after the adopted part completes") + } + + func testV9ImportMakesLegacyRowsAndKeepsDormantManifests() throws { + // v9 state on disk before the first v10 launch: a journal entry, a + // half-done chunked manifest with no journal entry, and v9 task metadata. + let root = makeTempDir() + let v9Store = QueueStore(root: root.appendingPathComponent("queue")) + let v9Journal = EventJournal(root: root.appendingPathComponent("events")) + V9Journal.write(V9Journal.completed(eventId: "ev1", id: "transfer-1", timestamp: 100), eventId: "ev1", + into: v9Journal.root) + let manifest = ChunkedManifestV9(id: "cap-1", parts: [ + .init(url: "https://s3.test/part1", start: 0, end: 10, accepted: true), + .init(url: "https://s3.test/part2", start: 10, end: 20, accepted: false), + .init(url: "https://s3.test/part3", start: 20, end: 30, accepted: false), + ], expiresAt: h.expiresAt, incarnation: "v9inc") + writeFile(v9Store.fileURL("cap-1", QueueStore.manifestName), + String(data: try JSONEncoder().encode(manifest), encoding: .utf8)!) + writeFile(v9Store.fileURL("cap-1", "blob"), bytes: 30) + TaskMap(fileURL: root.appendingPathComponent("taskmap.json")).set(.init(id: "transfer-1"), forKey: "any:5") + + h = Harness(root: root) // first v10 launch: the import runs in init + XCTAssertNotNil(h.row("transfer-1"), "legacy rows are in the index before any reconcile") + h.boot() + let legacy = try XCTUnwrap(h.row("transfer-1")) + XCTAssertEqual(legacy["key"] as? String, "legacy") + XCTAssertEqual(legacy["state"] as? String, "completed") + XCTAssertTrue(legacy["vars"] is NSNull) + XCTAssertNil(h.row("cap-1"), "a dormant manifest is not a row") + XCTAssertTrue(h.sink.settled.isEmpty, "nothing is delivered") + XCTAssertTrue(h.journal.legacyEvents().isEmpty) + XCTAssertNil(h.map.meta(forKey: "any:5")) + XCTAssertTrue(h.store.isImported()) + + // Diana's re-send with the same parts resumes the v9 bytes. + let resend = h.chunkedRaw(id: "cap-1", size: 30, parts: 3, source: h.root.appendingPathComponent("gone"), + urlPrefix: "https://s3.test/part") + _ = try h.enqueue(resend).get() + let e = try XCTUnwrap(h.entry("cap-1")) + XCTAssertEqual(e.parts.map(\.accepted), [true, false, false]) + XCTAssertEqual(e.incarnation, "v9inc") + XCTAssertEqual(e.bodyPath, "blob") + XCTAssertNil(h.store.loadV9Manifest("cap-1"), "adopted") + XCTAssertEqual(h.transport.live.count, 2, "only the unaccepted parts") + + // Diana cancels the legacy row: gone now. + h.cancel("transfer-1") + XCTAssertNil(h.row("transfer-1")) + } + + /// v9 state with a journaled outcome for a chunked id whose manifest and + /// blob are still on disk. + private func legacyChunked(type: String, accepted: [Bool]) throws -> Harness { + let root = makeTempDir() + let v9Store = QueueStore(root: root.appendingPathComponent("queue")) + let v9Journal = EventJournal(root: root.appendingPathComponent("events")) + let json = type == "error" ? V9Journal.error(eventId: "ev1", id: "cap-1", timestamp: 100) + : V9Journal.completed(eventId: "ev1", id: "cap-1", timestamp: 100) + V9Journal.write(json, eventId: "ev1", into: v9Journal.root) + let parts: [ChunkedManifestV9.Part] = (0..<3).map { i in + ChunkedManifestV9.Part(url: "https://s3.test/part\(i + 1)", start: Int64(i * 10), + end: Int64(i * 10 + 10), accepted: accepted[i]) + } + let manifest = ChunkedManifestV9(id: "cap-1", parts: parts, expiresAt: h.expiresAt, incarnation: "v9inc") + writeFile(v9Store.fileURL("cap-1", QueueStore.manifestName), + String(data: try JSONEncoder().encode(manifest), encoding: .utf8)!) + writeFile(v9Store.fileURL("cap-1", "blob"), bytes: 30) + let fresh = Harness(root: root) + fresh.boot() + return fresh + } + + func testLegacyCompletedRowAdoptsTheV9BytesOnASameIdEnqueue() throws { + let l = try legacyChunked(type: "completed", accepted: [true, true, false]) + defer { try? FileManager.default.removeItem(at: l.root) } + XCTAssertEqual(l.row("cap-1")?["state"] as? String, "completed") + XCTAssertEqual(l.row("cap-1")?["bytesSent"] as? Int64, 0, "a legacy row reports 0/0, as on Android") + XCTAssertEqual(l.row("cap-1")?["totalBytes"] as? Int64, 0) + let gone = l.root.appendingPathComponent("gone") + _ = try l.enqueue(l.chunkedRaw(id: "cap-1", size: 30, parts: 3, source: gone)).get() + let e = try XCTUnwrap(l.entry("cap-1")) + XCTAssertEqual(l.row("cap-1")?["bytesSent"] as? Int64, 20, "the adopted entry counts the v9 accepted parts") + XCTAssertEqual(l.row("cap-1")?["totalBytes"] as? Int64, 30) + XCTAssertFalse(e.legacy) + XCTAssertEqual(e.bodyPath, "blob") + XCTAssertEqual(e.parts.map(\.accepted), [true, true, false], "resumes, not E_FILE_MISSING") + XCTAssertEqual(l.transport.live.count, 1) + } + + func testLegacyErrorRowWithNewPartsKeepsTheV9Blob() throws { + let l = try legacyChunked(type: "error", accepted: [true, false, false]) + defer { try? FileManager.default.removeItem(at: l.root) } + let gone = l.root.appendingPathComponent("gone") + _ = try l.enqueue(l.chunkedRaw(id: "cap-1", size: 30, parts: 3, source: gone, + urlPrefix: "https://s3.test/new")).get() + let e = try XCTUnwrap(l.entry("cap-1")) + XCTAssertEqual(e.bodyPath, "blob") + XCTAssertTrue(e.parts.allSatisfy { !$0.accepted }, "a new plan starts over on the kept bytes") + XCTAssertEqual(l.transport.live.count, 3) + } + + func testV9JournalFilesOfEachKindImportAsLegacyRows() throws { + let root = makeTempDir() + let events = root.appendingPathComponent("events") + V9Journal.write(V9Journal.completed(eventId: "e1", id: "t-done", timestamp: 100), eventId: "e1", into: events) + V9Journal.write(V9Journal.error(eventId: "e2", id: "t-bad", timestamp: 200), eventId: "e2", into: events) + V9Journal.write(V9Journal.cancelled(eventId: "e3", id: "t-off", timestamp: 300), eventId: "e3", into: events) + h = Harness(root: root) + h.boot() + XCTAssertEqual(h.row("t-done")?["state"] as? String, "completed") + XCTAssertEqual(h.row("t-bad")?["state"] as? String, "error") + XCTAssertEqual(h.entry("t-bad")?.lastPartIndex, 2) + XCTAssertEqual(h.row("t-off")?["state"] as? String, "cancelled") + XCTAssertTrue(["t-done", "t-bad", "t-off"].allSatisfy { h.row($0)?["key"] as? String == "legacy" }) + XCTAssertTrue(h.journal.legacyEvents().isEmpty, "the v9 files are gone after the import") + XCTAssertTrue(h.unacknowledged().isEmpty, "nothing is delivered") + } + + func testImportRunsOnce() throws { + h.boot() + V9Journal.write(V9Journal.error(eventId: "late", id: "t", timestamp: 1), eventId: "late", into: h.journal.root) + let fresh = Harness(root: h.root) + fresh.boot() + XCTAssertNil(fresh.row("t"), "the marker stops a second import") + } +} diff --git a/ios/Tests/CoordinatorSimpleTests.swift b/ios/Tests/CoordinatorSimpleTests.swift new file mode 100644 index 00000000..3e8f7b91 --- /dev/null +++ b/ios/Tests/CoordinatorSimpleTests.swift @@ -0,0 +1,780 @@ +import XCTest +@testable import RNBGUCore + +/// State transitions of simple (one-body) entries, over the fake transport. +final class CoordinatorSimpleTests: XCTestCase { + private var h: Harness! + + override func setUp() { + h = Harness() + h.boot() + } + + override func tearDown() { + try? FileManager.default.removeItem(at: h.root) + } + + private func onlyTask(file: StaticString = #filePath, line: UInt = #line) -> FakeTask { + XCTAssertEqual(h.transport.live.count, 1, file: file, line: line) + return h.transport.live.last! + } + + // MARK: - Create and complete + + func testEnqueueWritesAheadThenIssues() throws { + XCTAssertEqual(try h.enqueue(h.dataRaw(id: "a", headers: ["Authorization": "t1"])).get(), "a") + XCTAssertEqual(h.sink.stateNames, ["queued", "running"]) + let task = onlyTask() + XCTAssertEqual(task.request.httpMethod, "POST") + XCTAssertEqual(task.header("Authorization"), "t1") + XCTAssertEqual(task.header("Content-Type"), "application/json") + XCTAssertNotNil(task.header("X-Request-Id")) + XCTAssertEqual(try String(contentsOf: task.file!), #"{"x":1}"#) + let stored = try XCTUnwrap(h.store.load("a")) + XCTAssertEqual(stored.state, .running) + XCTAssertEqual(stored.attempts, 1) + XCTAssertEqual(stored.lastRequestId, task.header("X-Request-Id")) + XCTAssertEqual(h.map.meta(forKey: task.key)?.purpose, .attempt) + } + + func testExistingContentTypeInAnyCaseIsKept() throws { + _ = try h.enqueue(h.dataRaw(id: "a", headers: ["content-type": "application/vnd.x+json"])).get() + XCTAssertEqual(onlyTask().header("Content-Type"), "application/vnd.x+json") + } + + func testCompletedJournalsThenEmitsAndAckForgets() throws { + _ = try h.enqueue(h.dataRaw(id: "a")).get() + h.complete(onlyTask(), status: 201, body: #"{"ok":true}"#, headers: ["X-A": "1"]) + XCTAssertEqual(h.sink.stateNames.last, "completed") + XCTAssertEqual(h.sink.attempts.last?["outcome"] as? String, "completed") + let settled = try XCTUnwrap(h.sink.settled.last) + XCTAssertEqual(settled["kind"] as? String, "completed") + XCTAssertEqual(settled["deliveries"] as? Int, 1) + XCTAssertEqual(settled["url"] as? String, "https://api.test/x") + XCTAssertEqual(settled["method"] as? String, "POST") + XCTAssertEqual(settled["attempts"] as? Int, 1) + let response = try XCTUnwrap(settled["response"] as? [String: Any]) + XCTAssertEqual(response["status"] as? Int, 201) + XCTAssertEqual(response["body"] as? String, #"{"ok":true}"#) + let eventId = try XCTUnwrap(settled["eventId"] as? String) + XCTAssertNotNil(h.journal.load(eventId), "journaled") + XCTAssertEqual(h.row("a")?["state"] as? String, "completed", "row stays until the ack") + + h.ack([eventId, "unknown"]) + XCTAssertNil(h.row("a")) + XCTAssertFalse(FileIO.exists(h.store.dir("a")), "row and bytes go after the ack") + h.ack([eventId]) // idempotent + } + + func testUnacknowledgedCountsDeliveries() throws { + _ = try h.enqueue(h.dataRaw(id: "a")).get() + h.complete(onlyTask()) + XCTAssertEqual(h.unacknowledged().first?["deliveries"] as? Int, 2) + XCTAssertEqual(h.unacknowledged().first?["deliveries"] as? Int, 3) + } + + func testSettleWithNoListenerIsJournaledAtZeroAndTheFirstDrainReturnsOne() throws { + h.sink.listening = false // headless: no JS listener yet + _ = try h.enqueue(h.dataRaw(id: "a")).get() + h.complete(onlyTask()) + XCTAssertTrue(h.sink.settled.isEmpty, "not emitted live") + let eventId = try XCTUnwrap(h.journal.unacknowledged().first?.eventId) + XCTAssertEqual(h.journal.load(eventId)?.deliveries, 0) + // Rule 7 with no listener: nothing emitted, nothing counted. + _ = try h.enqueue(h.dataRaw(id: "a")).get() + XCTAssertTrue(h.sink.settled.isEmpty) + XCTAssertEqual(h.journal.load(eventId)?.deliveries, 0) + XCTAssertEqual(h.unacknowledged().first?["deliveries"] as? Int, 1, "the first delivery") + // The drain marks the listener: the next settle goes out live at 1. + _ = try h.enqueue(h.dataRaw(id: "b")).get() + h.complete(onlyTask()) + XCTAssertEqual(h.sink.settled.last?["id"] as? String, "b") + XCTAssertEqual(h.sink.settled.last?["deliveries"] as? Int, 1) + } + + func testFailedJournalWriteWithNoListenerKeepsZeroForTheDrain() throws { + h.sink.listening = false + _ = try h.enqueue(h.dataRaw(id: "a")).get() + setReadOnly(h.journal.root, true) + defer { setReadOnly(h.journal.root, false) } + h.complete(onlyTask()) + XCTAssertTrue(h.sink.settled.isEmpty) + XCTAssertEqual(h.coordinator.pendingJournal.values.first?.deliveries, 0) + XCTAssertEqual(h.unacknowledged().first?["deliveries"] as? Int, 1) + } + + // MARK: - Same-id rules + + func testRule7CompletedUnackedReemitsWithoutRunning() throws { + _ = try h.enqueue(h.dataRaw(id: "a")).get() + h.complete(onlyTask()) + let first = h.sink.settled.count + _ = try h.enqueue(h.dataRaw(id: "a")).get() + XCTAssertEqual(h.sink.settled.count, first + 1) + XCTAssertEqual(h.sink.settled.last?["deliveries"] as? Int, 2) + XCTAssertEqual(h.sink.settled.last?["eventId"] as? String, h.sink.settled[first - 1]["eventId"] as? String) + XCTAssertTrue(h.transport.live.isEmpty, "no second run") + } + + func testRule3SameBodyWhileRunningReplacesHeadersAndVars() throws { + _ = try h.enqueue(h.dataRaw(id: "a", headers: ["Authorization": "old"])).get() + var raw = h.dataRaw(id: "a", headers: ["Authorization": "new"]) + raw["varsJson"] = #"{"n":2}"# + _ = try h.enqueue(raw).get() + XCTAssertEqual(h.transport.created.count, 1, "the in-flight task keeps its request") + XCTAssertEqual(h.entry("a")?.headers["Authorization"], "new") + XCTAssertEqual((h.row("a")?["vars"] as? [String: Int])?["n"], 2) + XCTAssertEqual(h.entry("a")?.attempts, 1, "a live resume keeps the attempt ordinal") + } + + func testADifferentUrlOrMethodIsADifferentBody() throws { + _ = try h.enqueue(h.dataRaw(id: "a")).get() + guard case .failure(let e) = h.enqueue(h.dataRaw(id: "a", url: "https://api.test/other")) else { + return XCTFail("a new url on a running entry") + } + XCTAssertEqual(e.code, "E_RUNNING") + h.complete(onlyTask(), status: 503) // now waiting, not running + _ = try h.enqueue(h.dataRaw(id: "a", extra: ["method": "PUT"])).get() + let entry = try XCTUnwrap(h.entry("a")) + XCTAssertEqual(entry.generation, 2, "replaced, not resumed") + XCTAssertEqual(entry.method, "PUT") + XCTAssertEqual(onlyTask().request.httpMethod, "PUT") + } + + func testSameBodyOnAWaitingRetryRetriesNowWithTheSameAttempt() throws { + _ = try h.enqueue(h.dataRaw(id: "a", headers: ["Authorization": "old"])).get() + h.complete(onlyTask(), status: 503) + let waiting = onlyTask() + XCTAssertNotNil(waiting.beginAt) + _ = try h.enqueue(h.dataRaw(id: "a", headers: ["Authorization": "new"])).get() + XCTAssertTrue(waiting.cancelled) + let now = onlyTask() + XCTAssertNil(now.beginAt, "no wait") + XCTAssertEqual(now.header("X-Request-Id"), waiting.header("X-Request-Id"), "the waiting attempt never ran") + XCTAssertEqual(now.header("Authorization"), "new") + XCTAssertEqual(h.entry("a")?.attempts, 2) + XCTAssertEqual(h.entry("a")?.state, .running) + XCTAssertNil(h.entry("a")?.nextAttemptAt) + h.deliverCancel(waiting) + XCTAssertEqual(h.entry("a")?.state, .running, "the replaced task's cancel does nothing") + } + + func testRule5DifferentBodyWhileRunningRejects() throws { + _ = try h.enqueue(h.dataRaw(id: "a")).get() + guard case .failure(let e) = h.enqueue(h.dataRaw(id: "a", data: ["x": 2])) else { return XCTFail() } + XCTAssertEqual(e.code, "E_RUNNING") + XCTAssertEqual(h.entry("a")?.bodyFingerprint, h.store.load("a")?.bodyFingerprint, "entry untouched") + } + + func testRule4DifferentBodyWhileWaitingReplacesAndSupersedes() throws { + _ = try h.enqueue(h.dataRaw(id: "a")).get() + h.complete(onlyTask(), status: 503) + let delayed = onlyTask() + XCTAssertNotNil(delayed.beginAt) + let oldBody = try XCTUnwrap(h.entry("a")?.bodyPath) + _ = try h.enqueue(h.dataRaw(id: "a", data: ["x": 2])).get() + XCTAssertTrue(delayed.cancelled) + XCTAssertEqual(h.map.meta(forKey: delayed.key)?.purpose, .superseded) + let e = try XCTUnwrap(h.entry("a")) + XCTAssertEqual(e.generation, 2) + XCTAssertEqual(e.attempts, 1) + XCTAssertFalse(FileIO.exists(h.store.fileURL("a", oldBody)), "the old body is deleted after the save") + let fresh = onlyTask() + XCTAssertEqual(try String(contentsOf: fresh.file!), #"{"x":2}"#) + // The superseded task's cancel callback produces nothing. + let attempts = h.sink.attempts.count + h.deliverCancel(delayed) + XCTAssertEqual(h.sink.attempts.count, attempts) + XCTAssertEqual(h.entry("a")?.state, .running) + } + + func testRule6CancelledUnackedReopensUnderAFreshGeneration() throws { + _ = try h.enqueue(h.dataRaw(id: "a")).get() + h.cancel("a") + let cancelled = try XCTUnwrap(h.sink.settled.last) + XCTAssertEqual(cancelled["cancelReason"] as? String, "user") + _ = try h.enqueue(h.dataRaw(id: "a")).get() + XCTAssertEqual(h.entry("a")?.generation, 2) + XCTAssertEqual(h.entry("a")?.state, .running) + h.ack([cancelled["eventId"] as! String]) + XCTAssertNotNil(h.row("a"), "the old generation's ack does not forget the reopened entry") + } + + func testErrorEntryReopensOnSameBody() throws { + _ = try h.enqueue(h.dataRaw(id: "a")).get() + h.complete(onlyTask(), status: 400, body: "bad") + XCTAssertEqual(h.entry("a")?.state, .error) + let error = try XCTUnwrap(h.sink.settled.last?["error"] as? [String: Any]) + XCTAssertEqual(error["errorKind"] as? String, "http") + XCTAssertEqual((error["response"] as? [String: Any])?["status"] as? Int, 400) + h.ack([h.sink.settled.last!["eventId"] as! String]) + XCTAssertEqual(h.row("a")?["state"] as? String, "error", "an acked error keeps the row") + _ = try h.enqueue(h.dataRaw(id: "a")).get() + XCTAssertEqual(h.entry("a")?.state, .running) + XCTAssertEqual(h.entry("a")?.generation, 2) + } + + func testMissingFileRejectsFileMissingAndParseErrorRejectsInvalid() { + guard case .failure(let missing) = h.enqueue(h.raw(id: "f", descriptor: [ + "url": "https://a.test", "file": "/does/not/exist"])) else { return XCTFail() } + XCTAssertEqual(missing.code, "E_FILE_MISSING") + XCTAssertNil(h.row("f")) + guard case .failure(let bad) = h.enqueue(["id": "b", "key": "k", "descriptor": ["url": "https://a.test"]]) + else { return XCTFail() } + XCTAssertEqual(bad.code, "E_INVALID", "no expiresAt") + } + + // MARK: - Cancel + + func testCancelLiveThenSettled() throws { + _ = try h.enqueue(h.dataRaw(id: "a")).get() + let task = onlyTask() + h.cancel("a") + XCTAssertTrue(task.cancelled) + XCTAssertEqual(h.sink.stateNames.last, "cancelled") + XCTAssertEqual(h.sink.settled.last?["kind"] as? String, "cancelled") + let attempts = h.sink.attempts.count + h.deliverCancel(task) + XCTAssertEqual(h.sink.attempts.count, attempts, "the library's own cancel is not an attempt") + XCTAssertEqual(h.sink.settled.count, 1, "one outcome") + h.ack([h.sink.settled.last!["eventId"] as! String]) + XCTAssertNil(h.row("a")) + } + + func testCancelSettledForgetsNowWithItsEvents() throws { + _ = try h.enqueue(h.dataRaw(id: "a")).get() + h.complete(onlyTask(), status: 400) + h.cancel("a") + XCTAssertNil(h.row("a")) + XCTAssertFalse(FileIO.exists(h.store.dir("a"))) + XCTAssertTrue(h.journal.unacknowledged().isEmpty) + h.cancel("unknown") // no-op + } + + func testCancelWhoseJournalWriteFailsRejectsStorageAndChangesNothing() throws { + _ = try h.enqueue(h.dataRaw(id: "a")).get() + let task = onlyTask() + let states = h.sink.states.count, progress = h.sink.progress.count, timers = h.timers.count + setReadOnly(h.journal.root, true) + defer { setReadOnly(h.journal.root, false) } + let rejected = try XCTUnwrap(h.cancel("a")) + XCTAssertEqual(rejected.code, "E_STORAGE") + XCTAssertTrue(rejected.message.contains("'a'"), rejected.message) + XCTAssertEqual(h.entry("a")?.state, .running) + XCTAssertEqual(h.store.load("a")?.state, .running) + XCTAssertNil(h.entry("a")?.settledEventId) + XCTAssertFalse(task.cancelled, "the work keeps running") + XCTAssertEqual(h.sink.states.count, states) + XCTAssertEqual(h.sink.progress.count, progress) + XCTAssertTrue(h.sink.settled.isEmpty) + XCTAssertTrue(h.coordinator.pendingJournal.isEmpty) + XCTAssertEqual(h.timers.count, timers, "no journal retry is scheduled") + XCTAssertTrue(h.unacknowledged().isEmpty) + } + + func testCancelAgainAfterTheJournalIsWritableSettlesWithOneRecord() throws { + _ = try h.enqueue(h.dataRaw(id: "a")).get() + let task = onlyTask() + setReadOnly(h.journal.root, true) + XCTAssertEqual(h.cancel("a")?.code, "E_STORAGE") + setReadOnly(h.journal.root, false) + XCTAssertNil(h.cancel("a")) + XCTAssertTrue(task.cancelled) + XCTAssertEqual(h.entry("a")?.state, .cancelled) + XCTAssertEqual(h.sink.settled.count, 1) + let records = h.journal.unacknowledged() + XCTAssertEqual(records.count, 1, "one record") + XCTAssertEqual(records.first?.kind, .cancelled) + XCTAssertEqual(records.first?.eventId, h.sink.settled.first?["eventId"] as? String) + XCTAssertTrue(h.coordinator.pendingJournal.isEmpty) + } + + func testCancelSettledWhoseDirectoryCannotMoveRejectsStorageAndKeepsAll() throws { + _ = try h.enqueue(h.dataRaw(id: "a")).get() + h.complete(onlyTask(), status: 400) + setReadOnly(h.store.root, true) + defer { setReadOnly(h.store.root, false) } + XCTAssertEqual(h.cancel("a")?.code, "E_STORAGE") + XCTAssertEqual(h.row("a")?["state"] as? String, "error") + XCTAssertEqual(h.store.load("a")?.state, .error) + XCTAssertEqual(h.journal.unacknowledged().count, 1) + } + + func testCancelSettledWhoseEventsCannotBeDeletedPutsTheRowBack() throws { + _ = try h.enqueue(h.dataRaw(id: "a")).get() + h.complete(onlyTask(), status: 400) + setReadOnly(h.journal.root, true) + XCTAssertEqual(h.cancel("a")?.code, "E_STORAGE") + XCTAssertEqual(h.row("a")?["state"] as? String, "error") + XCTAssertEqual(h.store.load("a")?.state, .error, "the directory is back") + XCTAssertEqual(h.journal.unacknowledged().count, 1) + setReadOnly(h.journal.root, false) + XCTAssertNil(h.cancel("a")) + XCTAssertNil(h.row("a")) + XCTAssertFalse(FileIO.exists(h.store.dir("a"))) + XCTAssertTrue(h.journal.unacknowledged().isEmpty) + XCTAssertTrue(h.store.setAsideDirectories().isEmpty, "nothing left aside") + } + + // MARK: - Retry, backoff, expiry + + func testTransientSchedulesADelayedTask() throws { + _ = try h.enqueue(h.dataRaw(id: "a")).get() + h.complete(onlyTask(), status: 503) + XCTAssertEqual(h.sink.attempts.last?["outcome"] as? String, "error") + XCTAssertEqual(h.sink.attempts.last?["httpCode"] as? Int, 503) + let e = try XCTUnwrap(h.entry("a")) + XCTAssertEqual(e.state, .queued) + XCTAssertEqual(e.nextAttemptAt, h.clock + 1_000) + XCTAssertEqual(h.row("a")?["nextAttemptAt"] as? Double, h.clock + 1_000) + let retry = onlyTask() + XCTAssertEqual(retry.beginAt, Date(timeIntervalSince1970: (h.clock + 1_000) / 1000)) + XCTAssertEqual(e.attempts, 2) + h.complete(retry, status: 503) + XCTAssertEqual(h.entry("a")?.nextAttemptAt, h.clock + 2_000, "backoff doubles") + h.complete(onlyTask(), status: 200) + XCTAssertEqual(h.sink.settled.last?["attempts"] as? Int, 3) + } + + func testDelayedBeginMovesToRunningWithCurrentHeaders() throws { + _ = try h.enqueue(h.dataRaw(id: "a", headers: ["Authorization": "old"])).get() + h.complete(onlyTask(), status: 500) + let delayed = onlyTask() + h.updateHeaders(["authorization": "fresh"]) + let request = try XCTUnwrap(h.coordinator.taskWillBegin(key: delayed.key, description: delayed.taskDescription)) + XCTAssertEqual(request.value(forHTTPHeaderField: "Authorization"), "fresh") + XCTAssertEqual(request.value(forHTTPHeaderField: "X-Request-Id"), delayed.header("X-Request-Id")) + XCTAssertEqual(h.entry("a")?.state, .running) + XCTAssertNil(h.entry("a")?.nextAttemptAt) + XCTAssertEqual(h.map.meta(forKey: delayed.key)?.headerGeneration, 1) + } + + func testDelayedBeginIsCancelledWhenTheEntryMovedOn() throws { + _ = try h.enqueue(h.dataRaw(id: "a")).get() + h.complete(onlyTask(), status: 500) + let delayed = onlyTask() + h.pause() + XCTAssertNil(h.coordinator.taskWillBegin(key: delayed.key, description: delayed.taskDescription)) + } + + func testFirstProgressOfADelayedTaskMovesItToRunning() throws { + _ = try h.enqueue(h.dataRaw(id: "a")).get() + h.complete(onlyTask(), status: 500) + let delayed = onlyTask() + h.coordinator.taskProgress(key: delayed.key, description: delayed.taskDescription, sent: 3, expected: 7) + h.drain() + XCTAssertEqual(h.entry("a")?.state, .running) + XCTAssertEqual(h.sink.progress.last?["bytesSent"] as? Int64, 3) + XCTAssertEqual(h.sink.progress.last?["totalBytes"] as? Int64, 7) + } + + func testSystemCancelIsNoAttemptAndARetry() throws { + _ = try h.enqueue(h.dataRaw(id: "a")).get() + h.deliverCancel(onlyTask()) + XCTAssertTrue(h.sink.attempts.isEmpty, "a cancel is not an attempt") + XCTAssertTrue(h.sink.settled.isEmpty, "never a cancelled outcome") + XCTAssertEqual(h.entry("a")?.state, .queued) + XCTAssertNotNil(onlyTask().beginAt) + } + + func testBackoffPastExpiresAtSettlesExpired() throws { + _ = try h.enqueue(h.dataRaw(id: "a", extra: ["expiresAt": h.clock + 500])).get() + h.complete(onlyTask(), status: 503) + let error = try XCTUnwrap(h.sink.settled.last?["error"] as? [String: Any]) + XCTAssertEqual(error["errorKind"] as? String, "expired") + XCTAssertTrue(FileIO.exists(h.store.dir("a")), "bytes stay") + } + + func testExpiryTimerSettlesAWaitingEntry() throws { + _ = try h.enqueue(h.dataRaw(id: "a", extra: ["expiresAt": h.clock + 60_000])).get() + h.complete(onlyTask(), status: 503) + let waiting = onlyTask() + h.advance(60_200) + XCTAssertEqual(h.entry("a")?.state, .error) + XCTAssertTrue(waiting.cancelled) + XCTAssertEqual((h.sink.settled.last?["error"] as? [String: Any])?["errorKind"] as? String, "expired") + } + + func testExpiryLeavesARunningAttemptToItsOwnResult() throws { + _ = try h.enqueue(h.dataRaw(id: "a", extra: ["expiresAt": h.clock + 60_000])).get() + let task = onlyTask() + h.advance(60_200) + XCTAssertEqual(h.entry("a")?.state, .running, "not settled mid-flight") + XCTAssertFalse(task.cancelled) + h.complete(task, status: 400) + let error = try XCTUnwrap(h.sink.settled.last?["error"] as? [String: Any]) + XCTAssertEqual(error["errorKind"] as? String, "http", "a real response keeps its kind") + + _ = try h.enqueue(h.dataRaw(id: "b", extra: ["expiresAt": h.clock + 60_000])).get() + let second = onlyTask() + h.advance(60_200) + h.complete(second, status: 503) + XCTAssertEqual((h.sink.settled.last?["error"] as? [String: Any])?["errorKind"] as? String, "expired", + "a transient result past expiresAt is expired") + } + + func testAnExpiredEntryThatParksStillExpires() throws { + _ = try h.enqueue(h.dataRaw(id: "a", extra: ["expiresAt": h.clock + 60_000])).get() + let task = onlyTask() + h.advance(60_200) // passes while running + h.complete(task, status: 401) + XCTAssertEqual(h.entry("a")?.state, .awaitingAuth) + h.advance(100) + XCTAssertEqual(h.entry("a")?.state, .error) + XCTAssertEqual((h.sink.settled.last?["error"] as? [String: Any])?["errorKind"] as? String, "expired") + } + + func testPausedEntryCrossingExpiresAtSettlesAtResume() throws { + _ = try h.enqueue(h.dataRaw(id: "a", extra: ["expiresAt": h.clock + 60_000])).get() + _ = try h.enqueue(h.dataRaw(id: "b", extra: ["expiresAt": h.clock + 60_000])).get() + h.complete(h.transport.live.first { $0.taskDescription?.contains("\"b\"") == true }!, status: 401) + h.pause() + h.advance(60_200) + XCTAssertEqual(h.entry("a")?.state, .paused, "a pause does not expire") + XCTAssertEqual(h.entry("b")?.state, .paused) + XCTAssertTrue(h.sink.settled.isEmpty) + h.resume() + XCTAssertEqual(h.entry("a")?.state, .error) + XCTAssertEqual(h.entry("b")?.state, .error, "a parked entry too") + XCTAssertEqual(h.sink.settled.compactMap { ($0["error"] as? [String: Any])?["errorKind"] as? String }, + ["expired", "expired"]) + XCTAssertTrue(h.transport.live.isEmpty, "nothing issues") + } + + func testExpiredAtIssue() throws { + h.pause() + _ = try h.enqueue(h.dataRaw(id: "a", extra: ["expiresAt": h.clock + 10])).get() + h.clock += 20 + h.resume() + XCTAssertEqual(h.entry("a")?.state, .error) + } + + func testFileMissingAtCompletionIsTerminal() throws { + _ = try h.enqueue(h.dataRaw(id: "a")).get() + let task = onlyTask() + try FileManager.default.removeItem(at: task.file!) + h.complete(task, error: NSError(domain: NSURLErrorDomain, code: NSURLErrorFileDoesNotExist)) + XCTAssertEqual((h.sink.settled.last?["error"] as? [String: Any])?["errorKind"] as? String, "file") + } + + func testUnreadableFileIsTransient() throws { + _ = try h.enqueue(h.dataRaw(id: "a")).get() + h.complete(onlyTask(), error: NSError(domain: NSURLErrorDomain, code: NSURLErrorNoPermissionsToReadFile)) + XCTAssertEqual(h.entry("a")?.state, .queued) + XCTAssertEqual(h.sink.attempts.last?["errorKind"] as? String, "file") + } + + // The session raises an exception for a file it cannot open. The + // transport turns it into an error; these check what the queue does next. + + func testSessionCannotOpenAMissingBodySettlesFile() throws { + h.transport.failUpload = { file in + try? FileManager.default.removeItem(at: file) // deleted after the check + return NSError(domain: NSURLErrorDomain, code: NSURLErrorCannotOpenFile) + } + _ = try h.enqueue(h.dataRaw(id: "a")).get() + XCTAssertEqual(h.entry("a")?.state, .error) + XCTAssertEqual((h.sink.settled.last?["error"] as? [String: Any])?["errorKind"] as? String, "file") + XCTAssertTrue(h.transport.live.isEmpty) + } + + func testSessionCannotOpenAReadableBodyTriesAgainLater() throws { + h.transport.failUpload = { _ in NSError(domain: NSURLErrorDomain, code: NSURLErrorCannotOpenFile) } + _ = try h.enqueue(h.dataRaw(id: "a")).get() + XCTAssertEqual(h.entry("a")?.state, .queued) + XCTAssertEqual(h.store.load("a")?.state, .queued, "the disk does not say running") + XCTAssertTrue(h.transport.live.isEmpty) + XCTAssertTrue(h.sink.settled.isEmpty) + h.transport.failUpload = nil + h.advance(4 * 3_600_000) + XCTAssertEqual(h.transport.live.count, 1) + XCTAssertEqual(h.entry("a")?.state, .running) + } + + func testAcceptRuleWithBodyIncludes() throws { + _ = try h.enqueue(h.dataRaw(id: "a", extra: ["accept": [["status": 409, "bodyIncludes": "already completed"]]])).get() + h.complete(onlyTask(), status: 409, body: "upload already completed") + XCTAssertEqual(h.entry("a")?.state, .completed) + } + + func testDefault404IsTransientAndExemptOverride() throws { + _ = try h.enqueue(h.dataRaw(id: "a")).get() + h.complete(onlyTask(), status: 404) + XCTAssertEqual(h.entry("a")?.state, .queued) + h.coordinator.configure(["retry": ["terminalHttp": ["exempt": []]]]) + h.drain() + _ = try h.enqueue(h.dataRaw(id: "b")).get() + h.complete(h.transport.live.last!, status: 404) + XCTAssertEqual(h.entry("b")?.state, .error) + } + + // MARK: - Auth + + func testAuthParksAndUpdateHeadersResumes() throws { + _ = try h.enqueue(h.dataRaw(id: "a", headers: ["Authorization": "expired"])).get() + _ = try h.enqueue(h.dataRaw(id: "b", headers: ["Authorization": "expired"])).get() + let tasks = h.transport.live + let statesBefore = h.sink.states.count + tasks.forEach { h.complete($0, status: 401) } + XCTAssertEqual(h.sink.states.count, statesBefore + 2, "one state event per parked entry") + XCTAssertEqual(h.entry("a")?.state, .awaitingAuth) + XCTAssertTrue(h.transport.live.isEmpty, "no retry while parked") + h.updateHeaders(["authorization": "fresh"]) + XCTAssertEqual(h.transport.live.count, 2) + XCTAssertTrue(h.transport.live.allSatisfy { $0.header("Authorization") == "fresh" }) + XCTAssertEqual(h.entry("a")?.headers, ["authorization": "fresh"], "the old spelling is replaced") + XCTAssertEqual(h.entry("a")?.state, .running) + } + + func testSameBodyOnAParkedEntryKeepsCountingAttempts() throws { + _ = try h.enqueue(h.dataRaw(id: "a", headers: ["Authorization": "expired"])).get() + h.complete(onlyTask(), status: 401) + XCTAssertEqual(h.entry("a")?.attempts, 1) + _ = try h.enqueue(h.dataRaw(id: "a", headers: ["Authorization": "fresh"])).get() + XCTAssertEqual(h.entry("a")?.attempts, 2, "the same generation: no reset") + XCTAssertEqual(ChunkedEngine.parseRequestDescription(onlyTask().taskDescription)?.attempt, 2) + } + + func testAuthUnderAnOlderGenerationReissuesAtOnce() throws { + _ = try h.enqueue(h.dataRaw(id: "a", headers: ["Authorization": "old"])).get() + let first = onlyTask() + h.updateHeaders(["Authorization": "new"]) + h.complete(first, status: 401) + XCTAssertEqual(h.entry("a")?.state, .running, "not parked") + let second = onlyTask() + XCTAssertEqual(second.header("Authorization"), "new") + XCTAssertNil(second.beginAt, "no backoff") + } + + // MARK: - Pause, resume, wifi + + func testPauseProducesNoOutcomeAndResumeReissues() throws { + _ = try h.enqueue(h.dataRaw(id: "a")).get() + let task = onlyTask() + h.pause() + XCTAssertTrue(task.cancelled) + XCTAssertEqual(h.map.meta(forKey: task.key)?.purpose, .pause) + XCTAssertEqual(h.entry("a")?.state, .paused) + h.deliverCancel(task) + XCTAssertTrue(h.sink.settled.isEmpty) + XCTAssertEqual(h.sink.attempts.count, 0) + // A new enqueue while paused is created paused and issues nothing. + _ = try h.enqueue(h.dataRaw(id: "b")).get() + XCTAssertEqual(h.entry("b")?.state, .paused) + XCTAssertTrue(h.transport.live.isEmpty) + h.resume() + XCTAssertEqual(h.transport.live.count, 2) + XCTAssertEqual(h.entry("a")?.attempts, 2) + } + + func testPausedSettingSurvivesRelaunch() throws { + h.pause() + h.relaunch() + h.boot() + _ = try h.enqueue(h.dataRaw(id: "a")).get() + XCTAssertEqual(h.entry("a")?.state, .paused) + } + + func testPauseKeepsAuthParkingAcrossResume() throws { + _ = try h.enqueue(h.dataRaw(id: "a")).get() + h.complete(onlyTask(), status: 403) + h.pause() + XCTAssertEqual(h.entry("a")?.state, .paused) + h.resume() + XCTAssertEqual(h.entry("a")?.state, .awaitingAuth) + XCTAssertTrue(h.transport.live.isEmpty) + } + + func testAcceptedCompletionRacingAPauseSettles() throws { + _ = try h.enqueue(h.dataRaw(id: "a")).get() + let task = onlyTask() + h.pause() + h.complete(task, status: 200) // the response landed before the cancel took effect + XCTAssertEqual(h.entry("a")?.state, .completed) + } + + func testWifiOnlyPicksTheSessionAndMovesAWaitingRetry() throws { + _ = try h.enqueue(h.dataRaw(id: "a")).get() + XCTAssertFalse(onlyTask().wifiOnly) + h.complete(onlyTask(), status: 503) + let waiting = onlyTask() + h.setWifiOnly(true) + XCTAssertTrue(waiting.cancelled) + XCTAssertTrue(onlyTask().wifiOnly) + XCTAssertNotNil(onlyTask().beginAt, "the wait carries over") + _ = try h.enqueue(h.dataRaw(id: "b")).get() + XCTAssertTrue(h.transport.live.last!.wifiOnly) + } + + func testWifiToggleKeepsTheWaitingAttemptOrdinalAndBackoff() throws { + _ = try h.enqueue(h.dataRaw(id: "a")).get() + h.complete(onlyTask(), status: 503) + let waiting = onlyTask() + XCTAssertEqual(h.entry("a")?.attempts, 2) + h.setWifiOnly(true) + h.setWifiOnly(false) + XCTAssertEqual(h.entry("a")?.attempts, 2, "no HTTP attempt ran") + XCTAssertEqual(h.store.load("a")?.attempts, 2) + XCTAssertEqual(onlyTask().header("X-Request-Id"), waiting.header("X-Request-Id")) + h.complete(onlyTask(), status: 503) + XCTAssertEqual(h.entry("a")?.nextAttemptAt, h.clock + 2_000, "the exponent follows real attempts") + } + + func testSupersededTaskThatBeginsLateIsCancelled() throws { + _ = try h.enqueue(h.dataRaw(id: "a")).get() + h.complete(onlyTask(), status: 503) + let waiting = onlyTask() + h.setWifiOnly(true) + XCTAssertNil(h.coordinator.taskWillBegin(key: waiting.key, description: waiting.taskDescription), + "same ordinal, but replaced") + XCTAssertEqual(h.entry("a")?.state, .queued) + } + + // MARK: - Write-ahead failures + + func testFailedAttemptSaveCreatesNoTaskAndIssuesAfterABackoff() throws { + _ = try h.enqueue(h.dataRaw(id: "a")).get() + let dir = h.store.dir("a") + setReadOnly(dir, true) + defer { setReadOnly(dir, false) } + let created = h.transport.created.count + h.complete(onlyTask(), status: 503) + XCTAssertEqual(h.transport.created.count, created, "no task without the saved attempt") + XCTAssertEqual(h.entry("a")?.state, .queued) + XCTAssertEqual(h.entry("a")?.attempts, 1, "the index matches the disk") + XCTAssertEqual(h.store.load("a")?.attempts, 1) + setReadOnly(dir, false) + h.advance(1_000) + let retry = onlyTask() + XCTAssertEqual(h.store.load("a")?.attempts, 2) + XCTAssertEqual(h.store.load("a")?.lastRequestId, retry.header("X-Request-Id")) + } + + // MARK: - Outcomes whose journal file is missing + + func testFailedJournalWriteStillEmitsAndTheAckForgets() throws { + _ = try h.enqueue(h.dataRaw(id: "a")).get() + setReadOnly(h.journal.root, true) + defer { setReadOnly(h.journal.root, false) } + h.complete(onlyTask()) + let settled = try XCTUnwrap(h.sink.settled.last) + let eventId = try XCTUnwrap(settled["eventId"] as? String) + XCTAssertNil(h.journal.load(eventId)) + XCTAssertEqual(settled["deliveries"] as? Int, 1) + XCTAssertEqual(h.unacknowledged().first?["eventId"] as? String, eventId, "kept in memory") + h.ack([eventId]) + XCTAssertNil(h.row("a")) + XCTAssertFalse(FileIO.exists(h.store.dir("a"))) + } + + func testFailedJournalWriteIsRetriedUntilItLands() throws { + _ = try h.enqueue(h.dataRaw(id: "a")).get() + setReadOnly(h.journal.root, true) + h.complete(onlyTask()) + let eventId = try XCTUnwrap(h.sink.settled.last?["eventId"] as? String) + h.advance(Double(QueueCoordinator.journalRetryMs)) // still read-only: waits twice as long + XCTAssertNil(h.journal.load(eventId)) + setReadOnly(h.journal.root, false) + h.advance(Double(QueueCoordinator.journalRetryMs * 2)) + XCTAssertEqual(h.journal.load(eventId)?.deliveries, 1) + XCTAssertTrue(h.coordinator.pendingJournal.isEmpty) + } + + func testRelaunchForgetsSettledRowsWhoseOutcomeFileIsGone() throws { + _ = try h.enqueue(h.dataRaw(id: "done")).get() + h.complete(onlyTask()) + _ = try h.enqueue(h.dataRaw(id: "gone")).get() + h.cancel("gone") + _ = try h.enqueue(h.dataRaw(id: "bad")).get() + h.complete(h.transport.live.last!, status: 400) + // A crash between the ack's delete and the forget, for each row. + h.journal.ack(h.sink.settled.compactMap { $0["eventId"] as? String }) + h.relaunch() + h.boot() + XCTAssertNil(h.row("done")) + XCTAssertFalse(FileIO.exists(h.store.dir("done")), "its bytes go too") + XCTAssertNil(h.row("gone")) + XCTAssertEqual(h.row("bad")?["state"] as? String, "error", "an error row stays until cancel()") + } + + func testAckOfAPrunedEventStillForgetsTheRow() throws { + _ = try h.enqueue(h.dataRaw(id: "a")).get() + h.complete(onlyTask()) + let eventId = try XCTUnwrap(h.sink.settled.last?["eventId"] as? String) + h.journal.ack([eventId]) // the file is gone, as a prune would leave it + h.ack([eventId]) + XCTAssertNil(h.row("a")) + } + + func testInvalidInputRejectsInvalidAndStoresNothing() { + for d: [String: Any] in [ + ["url": "ftp://a.test/x"], + ["url": "https://a.test/x", "headers": ["Authorization": "t\r\nX: 1"]], + ["url": "https://a.test/x", "method": "GET", "dataJson": "{}"], + ] { + guard case .failure(let e) = h.enqueue(h.raw(id: "bad", descriptor: d)) else { return XCTFail("\(d)") } + XCTAssertEqual(e.code, "E_INVALID") + } + XCTAssertNil(h.row("bad")) + XCTAssertFalse(FileIO.exists(h.store.dir("bad"))) + } + + func testUpdateHeadersRejectsALineBreakAndChangesNothing() throws { + _ = try h.enqueue(h.dataRaw(id: "a", headers: ["Authorization": "old"])).get() + var code: String? + h.coordinator.updateHeaders(["Authorization": "new\r\n"], resolve: {}, reject: { c, _ in code = c }) + h.drain() + XCTAssertEqual(code, "E_INVALID") + XCTAssertEqual(h.entry("a")?.headers["Authorization"], "old") + XCTAssertEqual(h.coordinator.settings.headerGeneration, 0) + } + + func testNullValuedKeysSurviveOnTheBodyTheRowAndTheOutcome() throws { + _ = try h.enqueue(h.raw(id: "a", vars: ["status": NSNull(), "n": 1], descriptor: [ + "url": "https://api.test/x", "dataJson": #"{"status":null}"#])).get() + let task = onlyTask() + XCTAssertEqual(try String(contentsOf: task.file!), #"{"status":null}"#) + let vars = try XCTUnwrap(h.row("a")?["vars"] as? [String: Any]) + XCTAssertTrue(vars["status"] is NSNull, "the key is there, as JS null") + h.complete(task) + let settledVars = try XCTUnwrap(h.sink.settled.last?["vars"] as? [String: Any]) + XCTAssertTrue(settledVars["status"] is NSNull) + XCTAssertEqual(settledVars["n"] as? Int, 1) + } + + func testDataNullSendsTheJSONNullBody() throws { + _ = try h.enqueue(h.dataRaw(id: "a", data: NSNull())).get() + let task = onlyTask() + XCTAssertEqual(try String(contentsOf: task.file!), "null") + XCTAssertEqual(task.header("Content-Type"), "application/json") + } + + func testRowsCarryLiveBytesAndAFailureKeepsThem() throws { + _ = try h.enqueue(h.dataRaw(id: "a")).get() + let task = onlyTask() + h.coordinator.taskProgress(key: task.key, description: task.taskDescription, sent: 3, expected: 7) + h.drain() + XCTAssertEqual(h.row("a")?["bytesSent"] as? Int64, 3) + XCTAssertEqual(h.store.load("a")?.bytesSent, 0, "progress is memory only") + h.complete(task, status: 503) + XCTAssertEqual(h.row("a")?["bytesSent"] as? Int64, 0, "a new attempt starts from 0") + let retry = onlyTask() + h.coordinator.taskProgress(key: retry.key, description: retry.taskDescription, sent: 5, expected: 7) + h.drain() + h.complete(retry, status: 400) + XCTAssertEqual(h.sink.settled.last?["bytesSent"] as? Int64, 5, "the failed attempt's live bytes") + } + + // MARK: - Rows // MARK: - Rows + + func testGetRequestsRows() throws { + _ = try h.enqueue(h.dataRaw(id: "a")).get() + let row = try XCTUnwrap(h.row("a")) + XCTAssertEqual(row["key"] as? String, "k") + XCTAssertEqual(row["state"] as? String, "running") + XCTAssertEqual(row["attempts"] as? Int, 1) + XCTAssertNil(row["nextAttemptAt"]) + XCTAssertEqual((row["vars"] as? [String: Int])?["n"], 1) + XCTAssertEqual(row["totalBytes"] as? Int64, 7) + } +} diff --git a/ios/Tests/QueueEntryTests.swift b/ios/Tests/QueueEntryTests.swift new file mode 100644 index 00000000..91520df4 --- /dev/null +++ b/ios/Tests/QueueEntryTests.swift @@ -0,0 +1,302 @@ +import XCTest +@testable import RNBGUCore + +final class EnqueueParserTests: XCTestCase { + /// The bridge form: vars and data as JSON text. A `data` key in + /// `descriptor` becomes `dataJson`. + private func raw(_ descriptor: [String: Any], vars: Any = ["a": 1]) -> [String: Any] { + var d = descriptor + if d["expiresAt"] == nil { d["expiresAt"] = 2_000_000_000_000.0 } + if let data = d.removeValue(forKey: "data") { d["dataJson"] = jsonText(data) } + return ["id": "id-1", "key": "k", "varsJson": jsonText(vars), "descriptor": d] + } + + private func invalid(_ raw: [String: Any], file: StaticString = #filePath, line: UInt = #line) { + XCTAssertThrowsError(try EnqueueParser.parse(raw), file: file, line: line) { + XCTAssertTrue($0 is ParseError, file: file, line: line) + } + } + + func testDataBodyDefaultsToPost() throws { + var r = raw(["url": "https://a.test/x"]) + var d = r["descriptor"] as! [String: Any] + d["dataJson"] = #"{"b":2,"a":1}"# + r["descriptor"] = d + let p = try EnqueueParser.parse(r) + XCTAssertEqual(p.method, "POST") + guard case .data(let json) = p.body else { return XCTFail("expected data") } + XCTAssertEqual(json, #"{"b":2,"a":1}"#, "the text JS built is the body") + XCTAssertEqual(p.varsJSON, #"{"a":1}"#) + } + + func testBodilessDescriptor() throws { + let p = try EnqueueParser.parse(raw(["url": "https://a.test/x", "method": "DELETE"])) + guard case .none = p.body else { return XCTFail("expected none") } + XCTAssertEqual(p.method, "DELETE") + XCTAssertTrue(p.fingerprint.hasPrefix("none")) + } + + func testNullVarsAndNSNullFieldsAreAbsent() throws { + let p = try EnqueueParser.parse(raw(["url": "https://a.test/x", "file": NSNull(), "form": NSNull(), + "dataJson": NSNull()], vars: NSNull())) + XCTAssertEqual(p.varsJSON, "null") + guard case .none = p.body else { return XCTFail("NSNull must read as absent") } + } + + func testNullValuedKeysSurviveAsText() throws { + let p = try EnqueueParser.parse(raw(["url": "https://a.test/x", "data": ["status": NSNull()]], + vars: ["a": NSNull()])) + XCTAssertEqual(p.varsJSON, #"{"a":null}"#) + guard case .data(let json) = p.body else { return XCTFail("expected data") } + XCTAssertEqual(json, #"{"status":null}"#) + } + + func testDataJsonNullIsAJSONNullBody() throws { + let p = try EnqueueParser.parse(raw(["url": "https://a.test/x", "data": NSNull(), "file": NSNull()])) + guard case .data(let json) = p.body else { return XCTFail("dataJson \"null\" is a body") } + XCTAssertEqual(json, "null") + invalid(raw(["url": "https://a.test/x", "data": NSNull(), "file": "/tmp/a"])) + } + + func testObjectVarsOrDataAndBadJSONTextAreInvalid() { + invalid(["id": "i", "key": "k", "vars": ["a": 1], "descriptor": ["url": "https://a.test", "expiresAt": 1]]) + var withData = raw(["url": "https://a.test"]) + var d = withData["descriptor"] as! [String: Any] + d["data"] = ["x": 1] + withData["descriptor"] = d + invalid(withData) + var badVars = raw(["url": "https://a.test"]) + badVars["varsJson"] = "{nope" + invalid(badVars) + d = raw(["url": "https://a.test"])["descriptor"] as! [String: Any] + d["dataJson"] = "{nope" + invalid(["id": "i", "key": "k", "varsJson": "null", "descriptor": d]) + } + + func testGetWithABodyIsInvalid() throws { + invalid(raw(["url": "https://a.test", "method": "GET", "data": ["a": 1]])) + invalid(raw(["url": "https://a.test", "method": "get", "file": "/tmp/a"])) + XCTAssertNoThrow(try EnqueueParser.parse(raw(["url": "https://a.test", "method": "GET"]))) + XCTAssertNoThrow(try EnqueueParser.parse(raw(["url": "https://a.test", "method": "DELETE", "data": ["a": 1]]))) + } + + func testSchemeMustBeHttpOrHttps() throws { + invalid(raw(["url": "ftp://a.test/x"])) + invalid(raw(["url": "file:///tmp/x"])) + invalid(raw(["url": "https:///nohost"])) + invalid(raw(["file": "/tmp/f", "parts": [["url": "javascript://a.test/1", "range": ["start": 0, "end": 1]]]])) + XCTAssertNoThrow(try EnqueueParser.parse(raw(["url": "HTTP://a.test/x"]))) + } + + func testHeaderNamesAndValuesAreValidatedWithoutEchoingTheValue() { + invalid(raw(["url": "https://a.test", "headers": ["Bad Name": "x"]])) + invalid(raw(["url": "https://a.test", "headers": ["": "x"]])) + invalid(raw(["url": "https://a.test", "headers": ["X-A": "secret\r\nX-Injected: 1"]])) + invalid(raw(["url": "https://a.test", "headers": ["X-A": "a\nb"]])) + invalid(raw(["file": "/tmp/f", "parts": [["url": "https://s3.test/1", "headers": ["X": "a\rb"], + "range": ["start": 0, "end": 1]]]])) + invalid(raw(["url": "https://a.test", "form": [["name": "a", "contentType": "t\r\nX: 1", "string": "s"]]])) + XCTAssertThrowsError(try EnqueueParser.headers(["Authorization": "Bearer tok\r\n"])) { + XCTAssertFalse($0.localizedDescription.contains("tok"), "the value never reaches the message") + XCTAssertTrue($0.localizedDescription.contains("Authorization")) + } + } + + func testFormAndFile() throws { + let form = try EnqueueParser.parse(raw(["url": "https://a.test/x", "form": [ + ["name": "request", "contentType": "application/json", "string": "{}"], + ["name": "image", "contentType": "image/jpeg", "path": "/tmp/a.jpg"], + ]])) + guard case .form(let fields) = form.body else { return XCTFail("expected form") } + XCTAssertEqual(fields.count, 2) + XCTAssertEqual(fields[1].path, "/tmp/a.jpg") + + let file = try EnqueueParser.parse(raw(["url": "https://a.test/x", "file": "file:///tmp/a.bin"])) + guard case .file(let path) = file.body else { return XCTFail("expected file") } + XCTAssertEqual(path, "file:///tmp/a.bin") + XCTAssertTrue(file.fingerprint.hasPrefix("file:file:///tmp/a.bin")) + } + + func testMissingUrlWithoutPartsThrows() { + invalid(raw(["data": ["a": 1]])) + } + + func testPartsNeedNoUrlAndValidateRanges() throws { + let ok = try EnqueueParser.parse(raw(["file": "/tmp/f", "parts": [ + ["url": "https://s3.test/1", "range": ["start": 0, "end": 10]], + ]])) + XCTAssertNil(ok.url) + XCTAssertEqual(ok.parts.count, 1) + invalid(raw(["file": "/tmp/f", "parts": [ + ["url": "https://s3.test/1", "range": ["start": 5, "end": 5]], + ]])) + invalid(raw(["parts": [ + ["url": "https://s3.test/1", "range": ["start": 0, "end": 5]], + ]])) + } + + func testTwoBodyKindsThrow() { + invalid(raw(["url": "https://a.test", "data": [:], "file": "/tmp/a"])) + } + + func testHeadersKeepStringsAndNumbersOnly() throws { + let h = try EnqueueParser.headers(["A": "x", "B": 3, "C": NSNull(), "D": ["nested": 1]]) + XCTAssertEqual(h, ["A": "x", "B": "3"]) + } + + func testDataFingerprintIgnoresKeyOrder() throws { + func withText(_ text: String) -> [String: Any] { + var r = raw(["url": "https://a.test"]) + var d = r["descriptor"] as! [String: Any] + d["dataJson"] = text + r["descriptor"] = d + return r + } + let a = try EnqueueParser.parse(withText(#"{"x":1,"y":{"b":1,"a":2}}"#)) + let b = try EnqueueParser.parse(withText(#"{"y":{"a":2,"b":1},"x":1}"#)) + let c = try EnqueueParser.parse(withText(#"{"x":2}"#)) + XCTAssertEqual(a.fingerprint, b.fingerprint) + XCTAssertNotEqual(a.fingerprint, c.fingerprint) + } + + func testFormFingerprintComparesFieldsAsSent() throws { + let f1: [[String: Any]] = [["name": "a", "contentType": "t", "path": "/p1"]] + let f2: [[String: Any]] = [["name": "a", "contentType": "t", "path": "/p2"]] + let a = try EnqueueParser.parse(raw(["url": "https://a.test", "form": f1])) + let b = try EnqueueParser.parse(raw(["url": "https://a.test", "form": f1])) + let c = try EnqueueParser.parse(raw(["url": "https://a.test", "form": f2])) + XCTAssertEqual(a.fingerprint, b.fingerprint) + XCTAssertNotEqual(a.fingerprint, c.fingerprint) + } + + func testPartsFingerprintIgnoresFilePath() throws { + let parts: [[String: Any]] = [["url": "https://s3.test/1", "range": ["start": 0, "end": 4]]] + let a = try EnqueueParser.parse(raw(["file": "/tmp/one", "parts": parts])) + let b = try EnqueueParser.parse(raw(["file": "/tmp/two", "parts": parts])) + let c = try EnqueueParser.parse(raw(["file": "/tmp/one", "parts": [ + ["url": "https://s3.test/other", "range": ["start": 0, "end": 4]]]])) + XCTAssertEqual(a.fingerprint, b.fingerprint) + XCTAssertNotEqual(a.fingerprint, c.fingerprint) + } + + func testRetryOverrideParse() throws { + let p = try EnqueueParser.parse(raw(["url": "https://a.test", "retry": ["terminalHttp": ["exempt": []]]])) + XCTAssertEqual(p.retry, RetryOverride(exempt: [])) + XCTAssertEqual(RetryPolicy.resolve([nil, p.retry]).exempt, []) + XCTAssertEqual(RetryPolicy.resolve([nil, p.retry]).baseMs, 1_000) + } +} + +final class QueueEntryTests: XCTestCase { + private func parsed(_ descriptor: [String: Any], id: String = "e1", vars: Any = ["v": 1]) -> ParsedEnqueue { + var d = descriptor + if d["expiresAt"] == nil { d["expiresAt"] = 5_000.0 } + if let data = d.removeValue(forKey: "data") { d["dataJson"] = jsonText(data) } + return try! EnqueueParser.parse(["id": id, "key": "k", "varsJson": jsonText(vars), "descriptor": d]) + } + + private let staged = StagedBody(kind: .parts, relativePath: "blob-1", contentType: nil, + forceContentType: false, totalBytes: 20, adopted: false) + + private func chunked() -> QueueEntry { + let p = parsed(["file": "/f", "parts": [ + ["url": "https://s3.test/1", "range": ["start": 0, "end": 10]], + ["url": "https://s3.test/2", "range": ["start": 10, "end": 20]], + ]]) + return QueueEntry.created(from: p, staged: staged, headerGeneration: 3, paused: false, now: 100) + } + + func testCreatedDefaults() { + let e = chunked() + XCTAssertEqual(e.state, .queued) + XCTAssertEqual(e.generation, 1) + XCTAssertEqual(e.headerGeneration, 3) + XCTAssertEqual(e.totalBytes, 20) + XCTAssertEqual(e.bodyPath, "blob-1") + let paused = QueueEntry.created(from: parsed(["url": "https://a.test"]), staged: staged, + headerGeneration: 0, paused: true, now: 1) + XCTAssertEqual(paused.state, .paused) + } + + func testResumedKeepsAcceptedAndGeneration() { + var e = chunked().withPartAccepted(0) + e.parts[1].rejections = 4 + e.attempts = 7 + e.generation = 2 + let incoming = parsed(["file": "/f", "headers": ["Authorization": "new"], "expiresAt": 9_000.0, "parts": [ + ["url": "https://s3.test/1", "range": ["start": 0, "end": 10]], + ["url": "https://s3.test/2", "headers": ["X": "y"], "range": ["start": 10, "end": 20]], + ]], vars: ["v": 2]) + let reset = e.resumed(with: incoming, resetBudget: true, now: 200) + XCTAssertTrue(reset.parts[0].accepted) + XCTAssertEqual(reset.parts[1].rejections, 0) + XCTAssertEqual(reset.parts[1].headers, ["X": "y"]) + XCTAssertEqual(reset.attempts, 0) + XCTAssertEqual(reset.generation, 2) + XCTAssertEqual(reset.expiresAt, 9_000) + XCTAssertEqual(reset.headers, ["Authorization": "new"]) + XCTAssertEqual(reset.varsJSON, #"{"v":2}"#) + XCTAssertEqual(reset.createdAt, e.createdAt) + let kept = e.resumed(with: incoming, resetBudget: false, now: 200) + XCTAssertEqual(kept.attempts, 7) + XCTAssertEqual(kept.parts[1].rejections, 4) + } + + func testReplacedBumpsGenerationAndIncarnation() { + var e = chunked().withPartAccepted(0) + e.state = .error + e.settledEventId = "ev" + e.attempts = 5 + let next = e.replaced(with: parsed(["url": "https://a.test/new", "data": ["z": 1]]), + staged: StagedBody(kind: .data, relativePath: "body-2", contentType: "application/json", + forceContentType: false, totalBytes: 7, adopted: false), + now: 300) + XCTAssertEqual(next.generation, e.generation + 1) + XCTAssertNotEqual(next.incarnation, e.incarnation) + XCTAssertEqual(next.state, .queued) + XCTAssertEqual(next.attempts, 0) + XCTAssertNil(next.settledEventId) + XCTAssertEqual(next.bodyKind, .data) + XCTAssertTrue(next.parts.isEmpty) + XCTAssertEqual(next.createdAt, e.createdAt) + } + + func testRowOmitsNilNextAttemptAtAndCarriesVars() { + var e = chunked() + var row = e.row(vars: ["v": 1]) + XCTAssertNil(row["nextAttemptAt"]) + XCTAssertEqual((row["vars"] as? [String: Int])?["v"], 1) + XCTAssertEqual(row["state"] as? String, "queued") + e.nextAttemptAt = 1234 + e.state = .awaitingAuth + row = e.row(vars: NSNull()) + XCTAssertEqual(row["nextAttemptAt"] as? Double, 1234) + XCTAssertEqual(row["state"] as? String, "awaiting-auth") + XCTAssertTrue(row["vars"] is NSNull) + } + + func testTilesExactly() { + func part(_ s: Int64, _ e: Int64) -> QueueEntry.Part { + QueueEntry.Part(url: "u", headers: [:], start: s, end: e, accepted: false, rejections: 0) + } + XCTAssertTrue(QueueEntry.tilesExactly([part(0, 5), part(5, 10)], size: 10)) + XCTAssertTrue(QueueEntry.tilesExactly([part(5, 10), part(0, 5)], size: 10), "order-independent") + XCTAssertFalse(QueueEntry.tilesExactly([part(0, 4), part(5, 10)], size: 10), "gap") + XCTAssertFalse(QueueEntry.tilesExactly([part(0, 6), part(5, 10)], size: 10), "overlap") + XCTAssertFalse(QueueEntry.tilesExactly([part(0, 5), part(5, 11)], size: 10), "past end") + XCTAssertFalse(QueueEntry.tilesExactly([part(0, 5)], size: 10), "short") + XCTAssertFalse(QueueEntry.tilesExactly([], size: 0)) + } + + func testHeaderMergeIsCaseInsensitive() { + let merged = HeaderMerge.merge(["authorization": "old", "A": "1"], ["Authorization": "new"]) + XCTAssertEqual(merged, ["Authorization": "new", "A": "1"]) + XCTAssertEqual(HeaderMerge.value("content-type", in: ["Content-Type": "x"]), "x") + } + + func testEntryRoundTripsThroughJSON() throws { + let e = chunked().withPartAccepted(1) + let back = try JSONDecoder().decode(QueueEntry.self, from: try QueueStore.encode(e)) + XCTAssertEqual(back, e) + } +} diff --git a/ios/Tests/QueueStoreTests.swift b/ios/Tests/QueueStoreTests.swift new file mode 100644 index 00000000..aa0dd579 --- /dev/null +++ b/ios/Tests/QueueStoreTests.swift @@ -0,0 +1,152 @@ +import XCTest +@testable import RNBGUCore + +final class QueueStoreTests: XCTestCase { + private var root: URL! + private var store: QueueStore! + + override func setUp() { + root = makeTempDir() + store = QueueStore(root: root) + } + + override func tearDown() { + try? FileManager.default.removeItem(at: root) + } + + private func entry(_ id: String, state: QueueEntry.State = .queued, body: String? = "body-a") -> QueueEntry { + QueueEntry( + id: id, key: "k", varsJSON: "null", url: "https://a.test", method: "POST", + accept: [], retry: nil, bodyKind: .data, bodyPath: body, bodyContentType: "application/json", + forceContentType: false, bodyFingerprint: "f", parts: [], incarnation: "inc-1", headers: [:], + headerGeneration: 0, state: state, authParked: false, generation: 1, attempts: 0, bytesSent: 0, + totalBytes: 0, expiresAt: 10, nextAttemptAt: nil, settledEventId: nil, lastRequestId: nil, + lastUrl: nil, lastPartIndex: nil, legacy: false, createdAt: 1, updatedAt: 1) + } + + func testSaveLoadRoundTrip() throws { + let e = entry("a/b:c") // a hostile id + try store.save(e) + XCTAssertEqual(store.load("a/b:c"), e) + XCTAssertEqual(store.all(), [e]) + XCTAssertFalse(store.dir("a/b:c").lastPathComponent.contains("/")) + } + + func testAllSkipsCorruptEntryAndTmp() throws { + try store.save(entry("good")) + writeFile(store.fileURL("corrupt", QueueStore.entryName), "{ not json") + writeFile(store.fileURL("tmponly", QueueStore.entryName + ".tmp"), "{}") + XCTAssertEqual(store.all().map(\.id), ["good"]) + XCTAssertNil(store.load("corrupt")) + XCTAssertNil(store.load("tmponly"), "a .tmp with no entry.json reads as absent") + } + + // Crash mid-write: a half-written tmp next to a valid entry. + func testCrashMidWriteKeepsTheLastGoodEntry() throws { + let good = entry("x") + try store.save(good) + let tmp = store.fileURL("x", QueueStore.entryName + ".tmp") + writeFile(tmp, #"{"id":"x","key":"#) // truncated JSON + XCTAssertEqual(store.load("x"), good, "load never reads the tmp") + var next = good + next.state = .running + try store.save(next) + XCTAssertEqual(store.load("x"), next) + XCTAssertFalse(FileIO.exists(tmp), "the next save replaces the tmp") + } + + func testRemoveDeletesTheDirectory() throws { + try store.save(entry("r")) + writeFile(store.fileURL("r", "body-a"), "b") + store.remove("r") + XCTAssertFalse(FileIO.exists(store.dir("r"))) + XCTAssertNil(store.load("r")) + } + + func testSweepDeletesUnreferencedFiles() throws { + let e = entry("s", body: "body-new") + try store.save(e) + for name in ["body-new", "body-old", "blob-old", "entry.json.tmp", "part-0.inc-1.0-5", "part-0.inc-0.0-5"] { + writeFile(store.fileURL("s", name), "x") + } + store.sweep(e) + let left = Set(try FileManager.default.contentsOfDirectory(atPath: store.dir("s").path)) + XCTAssertEqual(left, ["entry.json", "body-new", "part-0.inc-1.0-5"]) + } + + func testAdoptableBlobOnlyWithoutEntry() throws { + writeFile(store.fileURL("c", "blob-123"), "bytes") + XCTAssertEqual(store.adoptableBlob("c"), "blob-123") + try store.save(entry("c")) + XCTAssertNil(store.adoptableBlob("c")) + } + + func testSettingsRoundTripAndDefault() throws { + XCTAssertEqual(store.loadSettings(), QueueSettings()) + var s = QueueSettings() + s.wifiOnly = true + s.headerGeneration = 4 + s.retry = RetryOverride(baseMs: 5) + try store.saveSettings(s) + XCTAssertEqual(QueueStore(root: root).loadSettings(), s) + } + + func testImportMarker() throws { + XCTAssertFalse(store.isImported()) + try store.markImported() + XCTAssertTrue(store.isImported()) + } + + func testWritePartFileTmpRenameAndReuse() throws { + writeFile(store.fileURL("p", "blob"), bytes: 100) + let url = try store.writePartFile(id: "p", blob: "blob", index: 1, start: 10, end: 30, incarnation: "i1") + XCTAssertEqual(FileIO.size(url), 20) + let bytes = try Data(contentsOf: url) + XCTAssertEqual(bytes.first, UInt8(10 % 251)) + // Reuse by identity: the same call returns the same file untouched. + let mtime = try FileManager.default.attributesOfItem(atPath: url.path)[.modificationDate] as? Date + let again = try store.writePartFile(id: "p", blob: "blob", index: 1, start: 10, end: 30, incarnation: "i1") + XCTAssertEqual(again, url) + XCTAssertEqual(try FileManager.default.attributesOfItem(atPath: url.path)[.modificationDate] as? Date, mtime) + // Another incarnation sweeps the stale file of the same index. + let other = try store.writePartFile(id: "p", blob: "blob", index: 1, start: 10, end: 30, incarnation: "i2") + XCTAssertFalse(FileIO.exists(url)) + XCTAssertTrue(FileIO.exists(other)) + store.removePartFile("p", 1) + XCTAssertFalse(FileIO.exists(other)) + } + + func testWritePartFileShortBlobThrows() { + writeFile(store.fileURL("p", "blob"), bytes: 10) + XCTAssertThrowsError(try store.writePartFile(id: "p", blob: "blob", index: 0, start: 0, end: 20, incarnation: "i")) + } + + func testDormantManifestReadAndRemove() throws { + let manifest = ChunkedManifestV9( + id: "v9", parts: [.init(url: "https://s3.test/1", start: 0, end: 5, accepted: true)], + expiresAt: 10, incarnation: "old") + writeFile(store.fileURL("v9", QueueStore.manifestName), + String(data: try JSONEncoder().encode(manifest), encoding: .utf8)!) + XCTAssertEqual(store.loadV9Manifest("v9"), manifest) + XCTAssertEqual(store.allV9Manifests()["v9"], manifest) + try store.save(entry("v9")) + XCTAssertEqual(store.loadV9Manifest("v9"), manifest, "read with or without an entry.json") + store.removeV9Manifest("v9") + XCTAssertNil(store.loadV9Manifest("v9")) + } + + func testV9ManifestFileDecodes() { + // The exact shape a v9 build wrote (stalled, rejections, accepted flags). + let json = """ + {"id":"cap","parts":[{"url":"https://s3.test/1","headers":{"Content-Range":"0-4/10"},"start":0,"end":5,\ + "accepted":true,"rejections":2},{"url":"https://s3.test/2","headers":{},"start":5,"end":10,"accepted":false}],\ + "accept":[{"status":409,"bodyIncludes":"already completed"}],"expiresAt":123,"wifiOnly":false,\ + "createdAt":1,"stalled":true,"incarnation":"inc"} + """ + writeFile(store.fileURL("cap", QueueStore.manifestName), json) + let m = store.loadV9Manifest("cap") + XCTAssertEqual(m?.acceptedBytes, 5) + XCTAssertEqual(m?.totalBytes, 10) + XCTAssertEqual(m?.incarnation, "inc") + } +} diff --git a/ios/Tests/RetryClassifierTests.swift b/ios/Tests/RetryClassifierTests.swift new file mode 100644 index 00000000..b717cc60 --- /dev/null +++ b/ios/Tests/RetryClassifierTests.swift @@ -0,0 +1,83 @@ +import XCTest +@testable import RNBGUCore + +final class RetryClassifierTests: XCTestCase { + private func classify(_ status: Int? = nil, body: String? = nil, error: NSError? = nil, + accept: [UploadOutcome.AcceptRule] = [], exempt: [Int] = [404], + fileExists: Bool = true, now: Double = 0, + expiresAt: Double = 100) -> RetryClassifier.Class { + var policy = RetryPolicy.defaults + policy.exempt = exempt + return RetryClassifier.classify(.init( + statusCode: status, body: body, error: error, accept: accept, policy: policy, + fileExists: fileExists, now: now, expiresAt: expiresAt)) + } + + func testTable() { + XCTAssertEqual(classify(200), .accepted) + XCTAssertEqual(classify(204), .accepted) + XCTAssertEqual(classify(409, body: "upload already completed", + accept: [.init(status: 409, bodyIncludes: "already completed")]), .accepted) + XCTAssertEqual(classify(409, body: "conflict", accept: [.init(status: 409, bodyIncludes: "already completed")]), + .terminalHttp) + XCTAssertEqual(classify(401), .auth) + XCTAssertEqual(classify(403), .auth) + XCTAssertEqual(classify(408), .transient) + XCTAssertEqual(classify(429), .transient) + XCTAssertEqual(classify(500), .transient) + XCTAssertEqual(classify(503), .transient) + XCTAssertEqual(classify(404), .transient, "404 is exempt by default") + XCTAssertEqual(classify(404, exempt: []), .terminalHttp, "the chunked part-404 case") + XCTAssertEqual(classify(400), .terminalHttp) + XCTAssertEqual(classify(422), .terminalHttp) + XCTAssertEqual(classify(304), .terminalHttp) + } + + func testErrors() { + XCTAssertEqual(classify(error: NSError(domain: NSURLErrorDomain, code: NSURLErrorNotConnectedToInternet)), + .transient) + XCTAssertEqual(classify(error: NSError(domain: NSURLErrorDomain, code: NSURLErrorTimedOut)), .transient) + let fileError = NSError(domain: NSURLErrorDomain, code: NSURLErrorFileDoesNotExist) + XCTAssertEqual(classify(error: fileError, fileExists: true), .transient, "unreadable now, maybe not later") + XCTAssertEqual(classify(error: fileError, fileExists: false), .fileMissing) + XCTAssertEqual(classify(error: NSError(domain: "Other", code: 1)), .transient) + } + + func testPastExpiresAtOnlyATransientResultIsExpired() { + XCTAssertEqual(classify(200, now: 100, expiresAt: 100), .accepted) + XCTAssertEqual(classify(503, now: 100, expiresAt: 100), .expired) + XCTAssertEqual(classify(error: NSError(domain: NSURLErrorDomain, code: NSURLErrorTimedOut), + now: 200, expiresAt: 100), .expired) + XCTAssertEqual(classify(error: NSError(domain: NSURLErrorDomain, code: NSURLErrorFileDoesNotExist), + fileExists: true, now: 200, expiresAt: 100), .expired) + XCTAssertEqual(classify(401, now: 101, expiresAt: 100), .auth, "a real response keeps its class") + XCTAssertEqual(classify(400, now: 101, expiresAt: 100), .terminalHttp) + XCTAssertEqual(classify(error: NSError(domain: NSURLErrorDomain, code: NSURLErrorFileDoesNotExist), + fileExists: false, now: 200, expiresAt: 100), .fileMissing) + } + + func testBackoff() { + let p = RetryPolicy.defaults + XCTAssertEqual(RetryClassifier.backoffMs(attempt: 1, policy: p, random: { 0.5 }), 1_000) + XCTAssertEqual(RetryClassifier.backoffMs(attempt: 2, policy: p, random: { 0.5 }), 2_000) + XCTAssertEqual(RetryClassifier.backoffMs(attempt: 3, policy: p, random: { 0.5 }), 4_000) + XCTAssertEqual(RetryClassifier.backoffMs(attempt: 40, policy: p, random: { 0.5 }), 7_200_000, "capped at max") + XCTAssertEqual(RetryClassifier.backoffMs(attempt: 1, policy: p, random: { 0 }), 800, "jitter low bound") + XCTAssertEqual(RetryClassifier.backoffMs(attempt: 1, policy: p, random: { 1 }), 1_200, "jitter high bound") + XCTAssertEqual(RetryClassifier.backoffMs(attempt: 0, policy: p, random: { 0.5 }), 1_000, "attempt < 1 clamps") + var noJitter = p + noJitter.jitter = 0 + XCTAssertEqual(RetryClassifier.backoffMs(attempt: 1000, policy: noJitter, random: { 0.9 }), 7_200_000) + } + + func testErrorKind() { + XCTAssertEqual(RetryClassifier.errorKind(for: NSError(domain: NSCocoaErrorDomain, code: NSFileNoSuchFileError)), "file") + XCTAssertEqual(RetryClassifier.errorKind(for: NSError(domain: NSURLErrorDomain, code: NSURLErrorTimedOut)), "network") + XCTAssertEqual(RetryClassifier.errorKind(for: NSError(domain: "X", code: 1)), "unknown") + } + + func testPolicyResolveLayers() { + let resolved = RetryPolicy.resolve([RetryOverride(baseMs: 10, exempt: [404, 409]), RetryOverride(baseMs: 20)]) + XCTAssertEqual(resolved, RetryPolicy(baseMs: 20, maxMs: 7_200_000, jitter: 0.2, exempt: [404, 409])) + } +} diff --git a/ios/Tests/SupportComponentTests.swift b/ios/Tests/SupportComponentTests.swift new file mode 100644 index 00000000..4780d67d --- /dev/null +++ b/ios/Tests/SupportComponentTests.swift @@ -0,0 +1,319 @@ +import XCTest +@testable import RNBGUCore + +private func sampleEntry(_ id: String, createdAt: Double, vars: String = #"{"n":1}"#) -> QueueEntry { + QueueEntry( + id: id, key: "k", varsJSON: vars, url: "https://a.test", method: "POST", + accept: [], retry: nil, bodyKind: .none, bodyPath: "body-1", bodyContentType: nil, + forceContentType: false, bodyFingerprint: "none", parts: [], incarnation: "i", headers: [:], + headerGeneration: 0, state: .queued, authParked: false, generation: 1, attempts: 0, bytesSent: 0, + totalBytes: 0, expiresAt: 10, nextAttemptAt: nil, settledEventId: nil, lastRequestId: nil, + lastUrl: nil, lastPartIndex: nil, legacy: false, createdAt: createdAt, updatedAt: createdAt) +} + +final class RequestIndexTests: XCTestCase { + func testLoadUpsertRemoveAndOrder() { + let index = RequestIndex() + index.load([sampleEntry("b", createdAt: 2), sampleEntry("a", createdAt: 1)]) + XCTAssertEqual(index.rows().map { $0["id"] as? String }, ["a", "b"]) + var b = sampleEntry("b", createdAt: 2) + b.state = .running + index.upsert(b) + XCTAssertEqual(index.entry("b")?.state, .running) + XCTAssertEqual(index.count, 2) + index.remove("a") + XCTAssertEqual(index.rows().count, 1) + XCTAssertNil(index.entry("a")) + } + + func testVarsDecodedOnceAndRefreshedOnChange() { + let index = RequestIndex() + index.upsert(sampleEntry("a", createdAt: 1)) + let first = index.row("a")?["vars"] as AnyObject + var same = sampleEntry("a", createdAt: 1) + same.attempts = 3 + index.upsert(same) + XCTAssertTrue(first === index.row("a")?["vars"] as AnyObject, "same vars text reuses the decoded object") + index.upsert(sampleEntry("a", createdAt: 1, vars: #"{"n":2}"#)) + XCTAssertEqual((index.row("a")?["vars"] as? [String: Int])?["n"], 2) + } +} + +final class EventJournalTests: XCTestCase { + private var root: URL! + private var journal: EventJournal! + + override func setUp() { + root = makeTempDir() + journal = EventJournal(root: root) + } + + override func tearDown() { + try? FileManager.default.removeItem(at: root) + } + + private func event(_ eventId: String, id: String = "e", at: Double = 1, + kind: JournaledEvent.Kind = .completed) -> JournaledEvent { + JournaledEvent(eventId: eventId, id: id, key: "k", varsJSON: #"{"a":1}"#, at: at, attempts: 2, + requestId: "r", deliveries: 0, bytesSent: 5, totalBytes: 5, url: "https://a.test", + method: "POST", partIndex: nil, generation: 1, kind: kind) + } + + func testAppendAndMarkDelivered() { + XCTAssertTrue(journal.append(event("1"))) + XCTAssertEqual(journal.load("1")?.deliveries, 0) + XCTAssertEqual(journal.markDelivered(["1", "missing"]).map(\.deliveries), [1]) + XCTAssertEqual(journal.markDelivered(["1"]).first?.deliveries, 2) + XCTAssertEqual(journal.load("1")?.deliveries, 2, "persisted") + } + + func testUnacknowledgedSortedAndSkipsV9() throws { + journal.append(event("late", at: 5)) + journal.append(event("early", at: 1)) + V9Journal.write(V9Journal.cancelled(eventId: "old", id: "u", timestamp: 0), eventId: "old", into: root) + XCTAssertEqual(journal.unacknowledged().map(\.eventId), ["early", "late"]) + XCTAssertEqual(journal.legacyEvents(), + [JournaledEventV9(eventId: "old", id: "u", type: "cancelled", timestamp: 0, cancelReason: "user")]) + } + + func testPruneKeepsEventsThatRowsName() { + let small = EventJournal(root: root.appendingPathComponent("small"), maxEntries: 2) + XCTAssertTrue(small.append(event("1"), keeping: ["1"])) + XCTAssertTrue(small.append(event("2"), keeping: ["1"])) + XCTAssertTrue(small.append(event("3"), keeping: ["1"])) + XCTAssertNotNil(small.load("1"), "named by a row") + XCTAssertNil(small.load("2"), "the oldest unnamed file goes") + XCTAssertNotNil(small.load("3"), "the new event is never pruned") + XCTAssertTrue(small.append(event("4"), keeping: ["1", "3"])) + XCTAssertEqual(small.unacknowledged().map(\.eventId), ["1", "3", "4"], "all named: over the cap") + } + + func testAckIsIdempotent() { + journal.append(event("1")) + journal.ack(["1", "unknown"]) + journal.ack(["1"]) + XCTAssertNil(journal.load("1")) + } + + func testRemoveForId() throws { + journal.append(event("1", id: "a")) + journal.append(event("2", id: "b")) + try journal.removeForId("a") + XCTAssertEqual(journal.unacknowledged().map(\.eventId), ["2"]) + try journal.removeForId("a") // nothing left: not an error + } + + func testRemoveForIdThrowsWhenAFileCannotBeDeleted() { + journal.append(event("1", id: "a")) + setReadOnly(journal.root, true) + defer { setReadOnly(journal.root, false) } + XCTAssertThrowsError(try journal.removeForId("a")) + XCTAssertNotNil(journal.load("1")) + } + + func testBodyCapAtOneMegabyte() { + let big = Data(repeating: UInt8(ascii: "a"), count: EventJournal.maxBodyBytes + 10) + let (body, truncated) = EventJournal.decodeBody(big) + XCTAssertTrue(truncated) + XCTAssertEqual(body.utf8.count, EventJournal.maxBodyBytes) + let small = EventJournal.decodeBody(Data("hi".utf8)) + XCTAssertEqual(small.body, "hi") + XCTAssertFalse(small.truncated) + // A cut inside a multi-byte character backs off to a whole one. + let multi = Data("aé".utf8) // 1 + 2 bytes + let cut = EventJournal.decodeBody(multi, cap: 2) + XCTAssertEqual(cut.body, "a") + XCTAssertTrue(cut.truncated) + } + + func testResponseBufferCaps() { + var buffer = ResponseBuffer() + buffer.append(Data(repeating: 1, count: 6), cap: 10) + buffer.append(Data(repeating: 2, count: 6), cap: 10) + XCTAssertEqual(buffer.data.count, 10) + XCTAssertTrue(buffer.truncated) + XCTAssertTrue(buffer.decoded().truncated) + } + + func testBridgedFlattensEachKind() { + var completed = event("c") + completed.response = RawResponseRecord(status: 201, headers: ["h": "v"], body: "{}", bodyTruncated: false) + let c = completed.bridged + XCTAssertEqual(c["kind"] as? String, "completed") + XCTAssertEqual(c["state"] as? String, "completed") + XCTAssertEqual((c["response"] as? [String: Any])?["status"] as? Int, 201) + XCTAssertEqual((c["vars"] as? [String: Int])?["a"], 1) + XCTAssertEqual(c["requestId"] as? String, "r") + XCTAssertNil(c["partIndex"]) + + var chunked = event("cc") + chunked.response = RawResponseRecord(bodyTruncated: false) + let cc = chunked.bridged["response"] as? [String: Any] + XCTAssertNil(cc?["status"], "a chunked completion has no status") + XCTAssertEqual(cc?["bodyTruncated"] as? Bool, false) + + var error = event("e", kind: .error) + error.error = OutcomeErrorRecord(errorKind: "http", message: "HTTP 404", + response: RawResponseRecord(status: 404, bodyTruncated: false), partIndex: 2) + error.partIndex = 2 + let e = error.bridged + XCTAssertEqual((e["error"] as? [String: Any])?["errorKind"] as? String, "http") + XCTAssertEqual((e["error"] as? [String: Any])?["partIndex"] as? Int, 2) + XCTAssertEqual(e["partIndex"] as? Int, 2) + XCTAssertNil(e["response"]) + + var cancelled = event("x", kind: .cancelled) + cancelled.cancelReason = "user" + XCTAssertEqual(cancelled.bridged["cancelReason"] as? String, "user") + + let nullVars = JournaledEvent(eventId: "n", id: "e", key: "k", varsJSON: "null", at: 1, attempts: 0, + deliveries: 1, bytesSent: 0, totalBytes: 0, url: "", method: "POST", + generation: 1, kind: .cancelled) + XCTAssertTrue(nullVars.bridged["vars"] is NSNull) + } +} + +final class TaskMapTests: XCTestCase { + func testRoundTripOfNewFieldsAndPurpose() { + let url = makeTempDir().appendingPathComponent("map.json") + let map = TaskMap(fileURL: url) + map.set(.init(id: "a", partIndex: 1, incarnation: "i", attempt: 3, requestId: "r", headerGeneration: 2, + generation: 4, purpose: .attempt), forKey: "s:1") + map.setPurpose(.pause, forKey: "s:1", id: "a") + map.setHeaderGeneration(5, forKey: "s:1") + let reread = TaskMap(fileURL: url) + XCTAssertEqual(reread.meta(forKey: "s:1"), + .init(id: "a", partIndex: 1, incarnation: "i", attempt: 3, requestId: "r", headerGeneration: 5, + generation: 4, purpose: .pause)) + reread.removeKey("s:1") + XCTAssertNil(TaskMap(fileURL: url).meta(forKey: "s:1")) + } + + func testV9EntryDecodes() throws { + let url = makeTempDir().appendingPathComponent("map.json") + try Data(#"{"s:9":{"id":"old","acceptStatus":[409]},"s:10":{"id":"new","purpose":"future"}}"#.utf8).write(to: url) + let map = TaskMap(fileURL: url) + XCTAssertEqual(map.meta(forKey: "s:9")?.id, "old", "an old key is ignored, not fatal") + XCTAssertNil(map.meta(forKey: "s:9")?.generation) + XCTAssertNil(map.meta(forKey: "s:10")?.purpose, "an unknown purpose reads as nil") + map.removeAll { _, meta in meta.generation == nil } + XCTAssertTrue(map.keys { _ in true }.isEmpty) + } + + func testOwnerResolution() { + let desc = ChunkedEngine.taskDescription(id: "a:b/c", attempt: 2, generation: 3) + XCTAssertEqual(TaskOwner.resolve(description: desc, meta: nil), .request(id: "a:b/c", generation: 3, attempt: 2)) + XCTAssertEqual(TaskOwner.resolve(description: nil, meta: .init(id: "m", generation: 1, purpose: nil)), nil, + "a meta without attempt has no v10 owner") + XCTAssertEqual(TaskOwner.resolve(description: nil, meta: .init(id: "m", attempt: 1, generation: 1)), + .request(id: "m", generation: 1, attempt: 1)) + XCTAssertEqual(TaskOwner.resolve(description: "legacy-bare-id", meta: .init(id: "legacy-bare-id")), nil) + XCTAssertEqual(TaskOwner.resolve(description: nil, meta: .init(id: "p", partIndex: 2, incarnation: "i")), + .part(id: "p", part: 2, incarnation: "i")) + } +} + +final class ChunkedEngineTests: XCTestCase { + func testWindowAndOneTaskPerPart() { + XCTAssertEqual(ChunkedEngine.indexesToEnqueue(pending: [0, 1, 2, 3, 4], inFlight: []), [0, 1, 2]) + XCTAssertEqual(ChunkedEngine.indexesToEnqueue(pending: [0, 1, 2, 3, 4], inFlight: [0, 1]), [2]) + XCTAssertEqual(ChunkedEngine.indexesToEnqueue(pending: [1, 2, 3], inFlight: [0, 1, 2]), []) + XCTAssertEqual(ChunkedEngine.indexesToEnqueue(pending: [3, 4], inFlight: [3]), [4], "never an in-flight index") + } + + func testDescriptionsSurviveHostileIds() { + let id = #"cap:1/"x"\n"# + let part = ChunkedEngine.taskDescription(id: id, part: 7, incarnation: "inc") + XCTAssertEqual(ChunkedEngine.parsePartDescription(part)?.id, id) + XCTAssertEqual(ChunkedEngine.parsePartDescription(part)?.part, 7) + XCTAssertNil(ChunkedEngine.parseRequestDescription(part)) + let req = ChunkedEngine.taskDescription(id: id, attempt: 4, generation: 2) + XCTAssertEqual(ChunkedEngine.parseRequestDescription(req)?.id, id) + XCTAssertEqual(ChunkedEngine.parseRequestDescription(req)?.attempt, 4) + XCTAssertNil(ChunkedEngine.parsePartDescription(req)) + XCTAssertNil(ChunkedEngine.parsePartDescription("bare-v9-id")) + } +} + +final class LegacyImportTests: XCTestCase { + private func manifest(_ id: String) -> ChunkedManifestV9 { + ChunkedManifestV9(id: id, parts: [ + .init(url: "https://s3.test/1", start: 0, end: 5, accepted: true), + .init(url: "https://s3.test/2", start: 5, end: 12, accepted: false), + ], expiresAt: 99, incarnation: "inc") + } + + func testJournalOnly() { + let rows = LegacyImport.plan(events: [ + JournaledEventV9(eventId: "1", id: "t1", type: "error", timestamp: 10), + JournaledEventV9(eventId: "2", id: "t1", type: "completed", timestamp: 20), + JournaledEventV9(eventId: "3", id: "t2", type: "cancelled", timestamp: 5), + ], manifests: [:]) + XCTAssertEqual(rows.map(\.id), ["t2", "t1"]) + XCTAssertEqual(rows.map(\.state), [.cancelled, .completed], "the latest v9 outcome wins") + XCTAssertTrue(rows.allSatisfy { $0.legacy && $0.key == "legacy" && $0.varsJSON == "null" }) + XCTAssertNil(rows[0].bodyPath) + } + + func testJournalAndManifest() { + let rows = LegacyImport.plan(events: [JournaledEventV9(eventId: "1", id: "cap", type: "error", timestamp: 3)], + manifests: ["cap": manifest("cap")]) + XCTAssertEqual(rows.count, 1) + XCTAssertEqual(rows[0].bytesSent, 0, "legacy rows report 0/0, as on Android") + XCTAssertEqual(rows[0].totalBytes, 0) + XCTAssertEqual(rows[0].bodyPath, "blob") + } + + func testManifestOnlyStaysDormant() { + XCTAssertTrue(LegacyImport.plan(events: [], manifests: ["cap": manifest("cap")]).isEmpty) + } +} + +final class ProgressThrottleTests: XCTestCase { + func testIntervals() { + let t = ProgressThrottle() + t.isForeground = true + XCTAssertTrue(t.shouldEmit("a", now: 0)) + XCTAssertFalse(t.shouldEmit("a", now: 999)) + XCTAssertTrue(t.shouldEmit("a", now: 1_000)) + XCTAssertTrue(t.shouldEmit("b", now: 1_001), "per id") + t.isForeground = false + XCTAssertFalse(t.shouldEmit("a", now: 2_500)) + XCTAssertTrue(t.shouldEmit("a", now: 601_000)) + t.reset("a") + XCTAssertTrue(t.shouldEmit("a", now: 601_001)) + } +} + +final class AttemptEventTests: XCTestCase { + private func input(status: Int? = 200, error: NSError? = nil, accepted: Bool = true, + body: String? = "ok") -> AttemptEvent.Input { + AttemptEvent.Input(id: "i", key: "k", requestId: "r", attempt: 1, url: "https://a.test", method: "PUT", + partIndex: 2, statusCode: status, headers: ["h": "v"], body: body, error: error, + accepted: accepted, at: 5) + } + + func testOutcomes() { + let ok = AttemptEvent.build(input()) + XCTAssertEqual(ok["outcome"] as? String, "completed") + XCTAssertEqual(ok["httpCode"] as? Int, 200) + XCTAssertEqual(ok["partIndex"] as? Int, 2) + let http = AttemptEvent.build(input(status: 400, accepted: false)) + XCTAssertEqual(http["outcome"] as? String, "error") + XCTAssertEqual(http["errorKind"] as? String, "http") + let net = AttemptEvent.build(input(status: nil, error: NSError(domain: NSURLErrorDomain, code: NSURLErrorTimedOut), + accepted: false)) + XCTAssertEqual(net["errorKind"] as? String, "network") + XCTAssertNil(net["httpCode"]) + for event in [ok, http, net] { + XCTAssertTrue(["completed", "error"].contains(event["outcome"] as? String), "only completed or error") + XCTAssertNil(event["cancelReason"]) + } + } + + func testBodyCappedAtFourKilobytes() { + let e = AttemptEvent.build(input(body: String(repeating: "x", count: 5_000))) + XCTAssertEqual((e["responseBody"] as? String)?.count, 4_096) + XCTAssertEqual(e["responseBodyTruncated"] as? Bool, true) + } +} diff --git a/ios/Tests/TestSupport.swift b/ios/Tests/TestSupport.swift new file mode 100644 index 00000000..ac890b66 --- /dev/null +++ b/ios/Tests/TestSupport.swift @@ -0,0 +1,301 @@ +import Foundation +import XCTest +@testable import RNBGUCore + +/// A fresh directory per test, removed afterwards. +func makeTempDir(_ name: String = #function) -> URL { + let dir = FileManager.default.temporaryDirectory + .appendingPathComponent("rnbgu-tests", isDirectory: true) + .appendingPathComponent(UUID().uuidString, isDirectory: true) + try! FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) + return dir +} + +/// Makes a directory read-only (writes into it fail) or writable again. +/// Tests restore it before tearDown deletes the tree. +func setReadOnly(_ dir: URL, _ readOnly: Bool) { + try! FileManager.default.setAttributes([.posixPermissions: readOnly ? 0o555 : 0o755], ofItemAtPath: dir.path) +} + +/// JSON text as JS JSON.stringify sends it (keys sorted, for stable tests). +func jsonText(_ value: Any) -> String { + String(data: try! JSONSerialization.data(withJSONObject: value, options: [.fragmentsAllowed, .sortedKeys]), + encoding: .utf8)! +} + +/// Journal files exactly as a v9 build wrote them (JSONEncoder of the v9 +/// JournaledEvent: nil fields omitted), one per kind. Literal, so a change to +/// JournaledEventV9 cannot change the fixture with it. +enum V9Journal { + static func completed(eventId: String, id: String, timestamp: Int) -> String { + #"{"eventId":"\#(eventId)","id":"\#(id)","type":"completed","timestamp":\#(timestamp),"# + + #""responseCode":200,"responseBody":"{\"ok\":true}","responseHeaders":{"Content-Type":"application\/json"}}"# + } + + static func error(eventId: String, id: String, timestamp: Int) -> String { + #"{"eventId":"\#(eventId)","id":"\#(id)","type":"error","timestamp":\#(timestamp),"responseCode":404,"# + + #""responseBody":"NoSuchUpload","error":"HTTP 404","errorKind":"http","partIndex":2}"# + } + + static func cancelled(eventId: String, id: String, timestamp: Int) -> String { + #"{"eventId":"\#(eventId)","id":"\#(id)","type":"cancelled","timestamp":\#(timestamp),"cancelReason":"user"}"# + } + + static func write(_ json: String, eventId: String, into journalRoot: URL) { + writeFile(journalRoot.appendingPathComponent(eventId + ".json"), json) + } +} + +func writeFile(_ url: URL, _ text: String) { + try! FileManager.default.createDirectory(at: url.deletingLastPathComponent(), withIntermediateDirectories: true) + try! Data(text.utf8).write(to: url) +} + +func writeFile(_ url: URL, bytes: Int) { + try! FileManager.default.createDirectory(at: url.deletingLastPathComponent(), withIntermediateDirectories: true) + try! Data((0.. String? { request.value(forHTTPHeaderField: name) } +} + +final class FakeTransport: Transport { + /// Tasks the daemon held before this process started. + var daemonTasks: [FakeTask] = [] + private(set) var created: [FakeTask] = [] + /// Static: task keys stay unique across a relaunch, as session task ids do. + private static var next = 1 + /// When set, allTasks holds its answer until `releaseAllTasks()`, so a + /// completion can land before reconcile. + var deferAllTasks = false + private var pendingAllTasks: (() -> Void)? + + /// When set, upload calls it and throws what it returns: the session could + /// not open the file. It may delete the file first, to race the check. + var failUpload: ((URL) -> Error?)? + + func upload(_ request: URLRequest, fromFile file: URL, wifiOnly: Bool, description: String, + beginAt: Date?, beforeResume: (String) -> Void) throws -> UploadTask { + if let error = failUpload?(file) { throw error } + let task = FakeTask(key: "\(wifiOnly ? "wifi" : "any"):\(Self.next)", description: description, + request: request, file: file, beginAt: beginAt, wifiOnly: wifiOnly) + Self.next += 1 + beforeResume(task.key) + created.append(task) + return task + } + + func allTasks(_ completion: @escaping ([UploadTask]) -> Void) { + let answer = { completion(self.daemonTasks + self.created) } + if deferAllTasks { pendingAllTasks = answer } else { answer() } + } + + func releaseAllTasks() { + let answer = pendingAllTasks + pendingAllTasks = nil + answer?() + } + + var live: [FakeTask] { created.filter(\.isLive) } +} + +final class FakeSink: EventSink { + var states: [[String: Any]] = [] + var progress: [[String: Any]] = [] + var attempts: [[String: Any]] = [] + var settled: [[String: Any]] = [] + + func emitState(_ body: [String: Any]) { states.append(body) } + func emitProgress(_ body: [String: Any]) { progress.append(body) } + func emitAttempt(_ body: [String: Any]) { attempts.append(body) } + func emitSettled(_ body: [String: Any]) { settled.append(body) } + /// A JS listener is attached. A test sets it false for a headless run. + var listening = true + func canDeliver() -> Bool { listening } + func listenerReady() { listening = true } + + var stateNames: [String] { states.compactMap { $0["state"] as? String } } +} + +/// A coordinator over temp stores and fakes. Time and timers are manual. +final class Harness { + let root: URL + let store: QueueStore + let journal: EventJournal + let taskMap: TaskMap + let transport = FakeTransport() + let sink = FakeSink() + var clock: Double = 1_700_000_000_000 + var timers: [(delayMs: Int, block: () -> Void)] = [] + private(set) var coordinator: QueueCoordinator! + let queue = DispatchQueue(label: "test.queue") + + init(root: URL = makeTempDir()) { + self.root = root + store = QueueStore(root: root.appendingPathComponent("queue")) + journal = EventJournal(root: root.appendingPathComponent("events")) + taskMap = TaskMap(fileURL: root.appendingPathComponent("taskmap.json")) + relaunch() + } + + /// A new process over the same disk: the index reloads from the store. + func relaunch(daemonTasks: [FakeTask] = []) { + transport.daemonTasks = daemonTasks + timers = [] + coordinator = QueueCoordinator( + store: store, journal: journal, taskMap: TaskMap(fileURL: taskMap.fileURL), + transport: transport, sink: sink, queue: queue, now: { [unowned self] in self.clock }, + random: { 0.5 }, schedule: { [unowned self] ms, block in self.timers.append((ms, block)) }) + } + + var map: TaskMap { coordinator.taskMap } + + /// Runs the relaunch reconcile to completion. + func boot() { + coordinator.reconcileAll {} + drain() + drain() + } + + func drain() { queue.sync {} } + + /// Fires every timer that is due within `ms` (advancing the clock). + func advance(_ ms: Double) { + clock += ms + let due = timers + timers = [] + for t in due { + if Double(t.delayMs) <= ms { queue.sync { t.block() } } else { timers.append((t.delayMs - Int(ms), t.block)) } + } + } + + @discardableResult + func enqueue(_ raw: [String: Any]) -> Result { + var result: Result? + coordinator.enqueue(raw, resolve: { result = .success($0) }, + reject: { result = .failure(EnqueueError(code: $0, message: $1)) }) + drain() + return result! + } + + /// Returns the rejection, or nil when cancel resolved. + @discardableResult + func cancel(_ id: String) -> EnqueueError? { + var rejected: EnqueueError? + coordinator.cancel(id, resolve: {}, reject: { rejected = EnqueueError(code: $0, message: $1) }) + drain() + return rejected + } + + func ack(_ eventIds: [String]) { + coordinator.ack(eventIds) {} + drain() + } + + func pause() { + coordinator.pause(resolve: {}, reject: { _, _ in XCTFail("pause rejected") }) + drain() + } + + func resume() { + coordinator.resume(resolve: {}, reject: { _, _ in XCTFail("resume rejected") }) + drain() + } + + func updateHeaders(_ patch: [String: Any]) { + coordinator.updateHeaders(patch, resolve: {}, reject: { _, _ in XCTFail("updateHeaders rejected") }) + drain() + } + + func setWifiOnly(_ on: Bool) { + coordinator.setWifiOnly(on, resolve: {}, reject: { _, _ in XCTFail("setWifiOnly rejected") }) + drain() + } + + func unacknowledged() -> [[String: Any]] { + var out: [[String: Any]] = [] + coordinator.unacknowledgedEvents { out = $0 } + drain() + return out + } + + /// Delivers a completion for `task`, as didCompleteWithError would. + func complete(_ task: FakeTask, status: Int? = 200, body: String = "", headers: [String: String] = [:], + error: NSError? = nil) { + task.isLive = false + coordinator.taskCompleted(TaskCompletion( + key: task.key, description: task.taskDescription, url: task.request.url?.absoluteString, + statusCode: error == nil ? status : nil, headers: headers, body: error == nil ? body : nil, + bodyTruncated: false, error: error)) + } + + /// Delivers the NSURLErrorCancelled callback of a task the test cancelled. + func deliverCancel(_ task: FakeTask) { + complete(task, error: NSError(domain: NSURLErrorDomain, code: NSURLErrorCancelled)) + } + + func entry(_ id: String) -> QueueEntry? { coordinator.index.entry(id) } + func row(_ id: String) -> [String: Any]? { coordinator.rows().first { $0["id"] as? String == id } } + + // MARK: - Builders + + var expiresAt: Double { clock + 14 * 24 * 3_600_000 } + + func raw(id: String, key: String = "k", vars: Any = ["n": 1], descriptor: [String: Any]) -> [String: Any] { + var d = descriptor + if d["expiresAt"] == nil { d["expiresAt"] = expiresAt } + return ["id": id, "key": key, "varsJson": jsonText(vars), "descriptor": d] + } + + /// `data` crosses as its JSON text, as the JS layer sends it. + func dataRaw(id: String, data: Any = ["x": 1], url: String = "https://api.test/x", + headers: [String: Any] = [:], extra: [String: Any] = [:]) -> [String: Any] { + var d: [String: Any] = ["url": url, "dataJson": jsonText(data), "headers": headers] + for (k, v) in extra { d[k] = v } + return raw(id: id, descriptor: d) + } + + /// A chunked descriptor over a fresh source file of `size` bytes cut into + /// `parts` equal parts. + func chunkedRaw(id: String, size: Int = 30, parts: Int = 3, source: URL? = nil, + urlPrefix: String = "https://s3.test/part", extra: [String: Any] = [:]) -> [String: Any] { + let file = source ?? root.appendingPathComponent("src-\(UUID().uuidString)") + if source == nil { writeFile(file, bytes: size) } + let step = size / parts + let list: [[String: Any]] = (0..:", the TaskMap key. + var key: String { get } + var taskDescription: String? { get } + /// Running or suspended. A completed or canceling task is not live: its + /// delegate callback settles it. + var isLive: Bool { get } + /// earliestBeginDate: set on a delayed retry. + var beginAt: Date? { get } + func cancel() +} + +protocol Transport: AnyObject { + /// Creates an upload task, sets its description and begin date, calls + /// `beforeResume` with its key (the caller writes the TaskMap there), then + /// resumes it. Throws when the session cannot open `file`: no task exists + /// then, and `beforeResume` was not called. + func upload(_ request: URLRequest, fromFile file: URL, wifiOnly: Bool, description: String, + beginAt: Date?, beforeResume: (String) -> Void) throws -> UploadTask + + /// Every task of both sessions. The completion may run on any queue. + func allTasks(_ completion: @escaping ([UploadTask]) -> Void) +} + +/// Live events for JS. Best-effort: JS may be dead, and every terminal is +/// journaled before emitSettled. +protocol EventSink: AnyObject { + func emitState(_ body: [String: Any]) + func emitProgress(_ body: [String: Any]) + func emitAttempt(_ body: [String: Any]) + func emitSettled(_ body: [String: Any]) + /// true when a JS listener can take a settled outcome: from the first + /// journal drain until the module goes away. When false, an outcome is + /// journaled with deliveries 0 and not emitted; the drain delivers it. + func canDeliver() -> Bool + /// The drain calls this on the coordinator queue before it reads the + /// journal, so a settle is either in the drain or emitted, never both. + func listenerReady() +} + +/// What didCompleteWithError reports, with the response body already capped. +struct TaskCompletion { + let key: String + let description: String? + let url: String? + let statusCode: Int? + let headers: [String: String] + let body: String? + let bodyTruncated: Bool + let error: NSError? +} + +/// Who a task belongs to. From taskDescription first, TaskMap second. +enum TaskOwner: Equatable { + case request(id: String, generation: Int, attempt: Int) + case part(id: String, part: Int, incarnation: String?) + + var id: String { + switch self { + case .request(let id, _, _), .part(let id, _, _): return id + } + } + + static func resolve(description: String?, meta: TaskMap.Meta?) -> TaskOwner? { + if let p = ChunkedEngine.parsePartDescription(description) { + return .part(id: p.id, part: p.part, incarnation: p.incarnation) + } + if let r = ChunkedEngine.parseRequestDescription(description) { + return .request(id: r.id, generation: r.generation, attempt: r.attempt) + } + guard let meta else { return nil } + if let part = meta.partIndex { return .part(id: meta.id, part: part, incarnation: meta.incarnation) } + if let generation = meta.generation, let attempt = meta.attempt { + return .request(id: meta.id, generation: generation, attempt: attempt) + } + // A v9 simple task: a bare id with no generation. No v10 owner. + return nil + } +} + +/// A task's response body while it streams in, capped at the settled body +/// cap. Bytes past the cap are dropped and flagged, so a huge error page +/// cannot grow memory without bound. +struct ResponseBuffer { + private(set) var data = Data() + private(set) var truncated = false + + mutating func append(_ chunk: Data, cap: Int = EventJournal.maxBodyBytes) { + let room = cap - data.count + if chunk.count <= room { + data.append(chunk) + } else { + if room > 0 { data.append(chunk.prefix(room)) } + truncated = true + } + } + + /// The body as text, and whether the cap cut it. + func decoded() -> (body: String, truncated: Bool) { + EventJournal.decodeBody(data, truncated: truncated) + } +} diff --git a/react-native-background-upload.podspec b/react-native-background-upload.podspec index be0e6424..a3d6696a 100644 --- a/react-native-background-upload.podspec +++ b/react-native-background-upload.podspec @@ -15,6 +15,9 @@ Pod::Spec.new do |s| } s.source_files = "ios/**/*.{h,m,mm,swift}" + # ios/Package.swift and ios/Tests are the host-side unit tests (`swift test`). + # They and SwiftPM's build output must not compile into the pod. + s.exclude_files = ["ios/Package.swift", "ios/Tests/**", "ios/.build/**", "ios/.swiftpm/**"] # RNFileUploader.h imports the codegen spec header, which is Obj-C++ only. Keep # it out of the public umbrella so a consumer's plain Obj-C # `@import react_native_background_upload;` still compiles — that import is how