diff --git a/README.md b/README.md index cc1b25ae..01aa3370 100644 --- a/README.md +++ b/README.md @@ -142,8 +142,9 @@ You can also let the client flush batches for you with the `auto_flush_rows` / ` `ws::addr=localhost:9000;auto_flush_rows=10000;auto_flush_interval=1000;`. **Confirm a batch is durably received.** Over QWP each flush returns a frame sequence number (FSN); `awaitAckedFsn` -blocks until the server has acknowledged it. Rows are safe in the store-and-forward log even if the ack has not landed -yet — they replay on reconnect. +blocks until the server has acknowledged it. With `sf_dir`, rows in the store-and-forward log replay after reconnect +or a producer-process restart. For periodic host-power-loss checkpoints, also configure +`sf_durability=periodic;sf_sync_interval_millis=5000;`. ```java try (Sender sender = db.borrowSender()) { @@ -259,7 +260,9 @@ try (QuestDB db = QuestDB.builder() List every cluster node in one `addr` server list; the single string configures both the ingest and query pools across all of them. On the query side, `target` selects the node role to route to (`any`, `primary`, or `replica`) and `failover=on` enables failover across the list. The ingest side reconnects across the same node list on its own — a -store-and-forward sender keeps buffering rows through a failover window and never drops them. +store-and-forward sender keeps buffering rows through a failover window and replays unacknowledged data. The default +`memory` durability mode protects process restarts, not host power loss; use periodic durability for background disk +checkpoints. ```java try (QuestDB db = QuestDB.connect( @@ -439,7 +442,10 @@ Applied by the query pool to select and fail over between the nodes in the `addr | `client_id` | | Opaque client identifier surfaced server-side for observability | The ingest side also accepts store-and-forward and reconnection tuning keys (`auto_flush_*`, `initial_connect_retry`, -`reconnect_*`, `request_durable_ack`, `sf_*`, `max_frame_rejections`, `poison_min_escalation_window_millis`, …). See the +`reconnect_*`, `request_durable_ack`, `sf_*`, `max_frame_rejections`, `poison_min_escalation_window_millis`, …). +`sf_durability=periodic` checkpoints mmap-published data in the background; `sf_sync_interval_millis` defaults to `5000` +in that mode. The interval is a target cadence: JVM scheduling and storage-sync latency add to the actual loss window. +Use `request_durable_ack=on` when end-to-end server durability is also required. See the [QuestDB documentation](https://questdb.com/docs/) for the full reference. ## Requirements diff --git a/core/src/main/c/share/files.c b/core/src/main/c/share/files.c index 75f228fa..b469e17b 100644 --- a/core/src/main/c/share/files.c +++ b/core/src/main/c/share/files.c @@ -65,6 +65,13 @@ JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_openRW0 return (jint) fd; } +JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_openRWExclusive0 + (JNIEnv *e, jclass cl, jlong lpszName) { + int fd; + RESTARTABLE(open((const char *) (uintptr_t) lpszName, O_CREAT | O_EXCL | O_RDWR, 0644), fd); + return (jint) fd; +} + JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_openAppend0 (JNIEnv *e, jclass cl, jlong lpszName) { int fd; @@ -124,6 +131,21 @@ JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_fsync return res; } +JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_fsyncDir0 + (JNIEnv *e, jclass cl, jlong lpszName) { + int fd; + RESTARTABLE(open((const char *) (uintptr_t) lpszName, O_RDONLY), fd); + if (fd < 0) { + return -1; + } + int res; + RESTARTABLE(fsync(fd), res); + int saved_errno = errno; + close(fd); + errno = saved_errno; + return res; +} + JNIEXPORT jboolean JNICALL Java_io_questdb_client_std_Files_truncate (JNIEnv *e, jclass cl, jint fd, jlong size) { int res; @@ -236,6 +258,18 @@ JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_lock return flock((int) fd, LOCK_EX | LOCK_NB); } +JNIEXPORT jint JNICALL Java_io_questdb_client_cutlass_qwp_client_sf_cursor_SlotLock_release0 + (JNIEnv *e, jclass cl, jint fd) { + if (flock((int) fd, LOCK_UN) != 0) { + return -1; + } + /* Unlock success confirms that the slot is reusable. close() is one-shot: + * POSIX leaves descriptor state unspecified on EINTR, so retrying its + * numeric value could close an unrelated descriptor after reuse. */ + (void) close((int) fd); + return 0; +} + JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_mkdir0 (JNIEnv *e, jclass cl, jlong lpszPath, jint mode) { return mkdir((const char *) (uintptr_t) lpszPath, (mode_t) mode); @@ -354,3 +388,15 @@ JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_msync (JNIEnv *e, jclass cl, jlong addr, jlong len, jboolean async) { return msync((void *) (uintptr_t) addr, (size_t) len, async ? MS_ASYNC : MS_SYNC); } + +/* Best-effort page pin. Callers treat a refusal (RLIMIT_MEMLOCK, missing + * privilege) as a soft downgrade, so no errno capture is required here. */ +JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_mlock0 + (JNIEnv *e, jclass cl, jlong addr, jlong len) { + return mlock((void *) (uintptr_t) addr, (size_t) len); +} + +JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_munlock0 + (JNIEnv *e, jclass cl, jlong addr, jlong len) { + return munlock((void *) (uintptr_t) addr, (size_t) len); +} diff --git a/core/src/main/c/share/net.c b/core/src/main/c/share/net.c index 3b0162fc..f3bc1ddd 100644 --- a/core/src/main/c/share/net.c +++ b/core/src/main/c/share/net.c @@ -128,6 +128,12 @@ JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_send return com_questdb_network_Net_EOTHERDISCONNECT; } +JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_shutdown + (JNIEnv *e, jclass cl, jint fd) { + const int result = shutdown((int) fd, SHUT_RDWR); + return result == -1 && errno == ENOTCONN ? 0 : result; +} + JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_recv (JNIEnv *e, jclass cl, jint fd, jlong ptr, jint len) { ssize_t n; diff --git a/core/src/main/c/share/net.h b/core/src/main/c/share/net.h index 27143639..d80617e2 100644 --- a/core/src/main/c/share/net.h +++ b/core/src/main/c/share/net.h @@ -80,6 +80,14 @@ JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_recv JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_send (JNIEnv *, jclass, jint, jlong, jint); +/* + * Class: io_questdb_client_network_Net + * Method: shutdown + * Signature: (I)I + */ +JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_shutdown + (JNIEnv *, jclass, jint); + /* * Class: io_questdb_client_network_Net * Method: getSndBuf diff --git a/core/src/main/c/windows/errno.h b/core/src/main/c/windows/errno.h index 093d52c2..a2971056 100644 --- a/core/src/main/c/windows/errno.h +++ b/core/src/main/c/windows/errno.h @@ -6,4 +6,10 @@ static DWORD dwTlsIndexLastError = 0; void SaveLastError(); +/* Returns the per-thread error code most recently stored by SaveLastError(). + * Both functions are defined in os.c, the only translation unit whose static + * dwTlsIndexLastError copy is initialised by TlsAlloc() in DllMain; every + * other includer's copy of the variable is an unused zero. */ +DWORD GetSavedLastError(); + #endif //ZLIB_ERRNO_H diff --git a/core/src/main/c/windows/files.c b/core/src/main/c/windows/files.c index 119f6315..e2b97af7 100644 --- a/core/src/main/c/windows/files.c +++ b/core/src/main/c/windows/files.c @@ -77,11 +77,16 @@ static jint open_file(const char *utf8Path, } HANDLE h = CreateFileW(wide, desiredAccess, shareMode, NULL, creationDisposition, flagsAndAttributes, NULL); - free(wide); if (h == INVALID_HANDLE_VALUE) { + // Save the CreateFileW failure code BEFORE free(): the thread's + // last-error value may be overwritten by any subsequent API/CRT call + // (HeapFree can SetLastError), and callers such as fsyncDir0 rely on + // the saved value being the CreateFileW error. SaveLastError(); + free(wide); return -1; } + free(wide); return HANDLE_TO_FD(h); } @@ -103,6 +108,15 @@ JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_openRW0 FILE_ATTRIBUTE_NORMAL); } +JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_openRWExclusive0 + (JNIEnv *e, jclass cl, jlong lpszName) { + return open_file((const char *) (uintptr_t) lpszName, + GENERIC_READ | GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + CREATE_NEW, + FILE_ATTRIBUTE_NORMAL); +} + JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_openAppend0 (JNIEnv *e, jclass cl, jlong lpszName) { jint fd = open_file((const char *) (uintptr_t) lpszName, @@ -215,6 +229,52 @@ JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_fsync return 0; } +JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_fsyncDir0 + (JNIEnv *e, jclass cl, jlong lpszName) { + jint fd = open_file((const char *) (uintptr_t) lpszName, + GENERIC_READ | GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS); + if (fd < 0) { + // A directory can be crash-consistent yet not openable for write: + // some ACL/filesystem configurations refuse a GENERIC_WRITE open of a + // directory handle with ERROR_ACCESS_DENIED. Directory-entry durability + // on NTFS is provided by metadata journaling ($LogFile), so degrade to + // best-effort success here rather than hard-failing the SF durability + // path. Consult the error open_file() saved at the CreateFileW failure + // point rather than the live GetLastError(): the intervening free() + // and return path inside open_file() are not guaranteed to preserve + // the thread's last-error value. A genuine failure such as a missing + // directory (ERROR_PATH_NOT_FOUND) still propagates as fatal. + if (GetSavedLastError() == ERROR_ACCESS_DENIED) { + return 0; + } + return -1; + } + if (!FlushFileBuffers(FD_TO_HANDLE(fd))) { + DWORD err = GetLastError(); + CloseHandle(FD_TO_HANDLE(fd)); + // FlushFileBuffers is documented for file/volume handles; NTFS refuses + // it on a directory handle (typically ERROR_ACCESS_DENIED, and + // ERROR_INVALID_FUNCTION on filesystems that do not implement it). + // create/rename/unlink of directory entries are made crash consistent + // by NTFS metadata journaling, so treat those "not supported" + // signatures as best-effort success. This mirrors libgit2/PostgreSQL/ + // SQLite, which do not rely on directory fsync on Windows, and keeps + // the SF manifest-create, slot-lock, and trim/unlink barriers from + // hard-failing. Any other error is a real I/O failure and stays fatal. + if (err == ERROR_ACCESS_DENIED || err == ERROR_INVALID_FUNCTION) { + return 0; + } + SetLastError(err); + SaveLastError(); + return -1; + } + CloseHandle(FD_TO_HANDLE(fd)); + return 0; +} + JNIEXPORT jboolean JNICALL Java_io_questdb_client_std_Files_truncate (JNIEnv *e, jclass cl, jint fd, jlong size) { FILE_END_OF_FILE_INFO eof; @@ -300,18 +360,24 @@ JNIEXPORT jlong JNICALL Java_io_questdb_client_std_Files_length0 OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); - free(wide); if (h == INVALID_HANDLE_VALUE) { + /* Save the CreateFileW failure code BEFORE free(): any subsequent + * API/CRT call (HeapFree can SetLastError) may overwrite the + * thread's last-error value -- same pattern as open_file(). */ SaveLastError(); + free(wide); return -1; } + free(wide); LARGE_INTEGER sz; BOOL ok = GetFileSizeEx(h, &sz); - CloseHandle(h); if (!ok) { + /* Save before CloseHandle() can clobber the last-error value. */ SaveLastError(); + CloseHandle(h); return -1; } + CloseHandle(h); return (jlong) sz.QuadPart; } @@ -331,6 +397,20 @@ JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_lock return 0; } +JNIEXPORT jint JNICALL Java_io_questdb_client_cutlass_qwp_client_sf_cursor_SlotLock_release0 + (JNIEnv *e, jclass cl, jint fd) { + OVERLAPPED ov; + memset(&ov, 0, sizeof(ov)); + if (!UnlockFileEx(FD_TO_HANDLE(fd), 0, MAXDWORD, MAXDWORD, &ov)) { + SaveLastError(); + return -1; + } + /* Unlock success confirms that the slot is reusable. Match POSIX by + * closing once without making handle cleanup part of that signal. */ + (void) CloseHandle(FD_TO_HANDLE(fd)); + return 0; +} + JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_mkdir0 (JNIEnv *e, jclass cl, jlong lpszPath, jint mode) { (void) mode; @@ -340,11 +420,13 @@ JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_mkdir0 return -1; } BOOL ok = CreateDirectoryW(wide, NULL); - free(wide); if (!ok) { + /* Save before free() can clobber the last-error value. */ SaveLastError(); + free(wide); return -1; } + free(wide); return 0; } @@ -373,11 +455,13 @@ JNIEXPORT jboolean JNICALL Java_io_questdb_client_std_Files_remove0 } else { ok = DeleteFileW(wide); } - free(wide); if (!ok) { + /* Save before free() can clobber the last-error value. */ SaveLastError(); + free(wide); return JNI_FALSE; } + free(wide); return JNI_TRUE; } @@ -395,12 +479,15 @@ JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_rename0 return -1; } BOOL ok = MoveFileExW(oldW, newW, MOVEFILE_REPLACE_EXISTING); - free(oldW); - free(newW); if (!ok) { + /* Save before free() can clobber the last-error value. */ SaveLastError(); + free(oldW); + free(newW); return -1; } + free(oldW); + free(newW); return 0; } @@ -438,11 +525,13 @@ JNIEXPORT jlong JNICALL Java_io_questdb_client_std_Files_findFirst0 pattern[pathLen] = '\0'; wchar_t *wide = utf8_to_wide(pattern); - free(pattern); if (!wide) { + /* Save before free() can clobber the last-error value. */ SaveLastError(); + free(pattern); return 0; } + free(pattern); qdb_find_t *find = (qdb_find_t *) malloc(sizeof(qdb_find_t)); if (!find) { @@ -450,12 +539,14 @@ JNIEXPORT jlong JNICALL Java_io_questdb_client_std_Files_findFirst0 return 0; } find->handle = FindFirstFileW(wide, &find->data); - free(wide); if (find->handle == INVALID_HANDLE_VALUE) { + /* Save before free() can clobber the last-error value. */ SaveLastError(); + free(wide); free(find); return 0; } + free(wide); find->hasEntry = 1; win_findname_to_utf8(find); return (jlong) (uintptr_t) find; @@ -615,3 +706,16 @@ JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_msync (void) async; return 0; } + +/* Best-effort page pin. VirtualLock is quota-bound to the process working-set + * minimum; callers treat any refusal as a soft downgrade, so no + * SaveLastError() -- the refusal is not surfaced as an error. */ +JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_mlock0 + (JNIEnv *e, jclass cl, jlong addr, jlong len) { + return VirtualLock((LPVOID) (uintptr_t) addr, (SIZE_T) len) ? 0 : -1; +} + +JNIEXPORT jint JNICALL Java_io_questdb_client_std_Files_munlock0 + (JNIEnv *e, jclass cl, jlong addr, jlong len) { + return VirtualUnlock((LPVOID) (uintptr_t) addr, (SIZE_T) len) ? 0 : -1; +} diff --git a/core/src/main/c/windows/net.c b/core/src/main/c/windows/net.c index fd290629..191295ab 100644 --- a/core/src/main/c/windows/net.c +++ b/core/src/main/c/windows/net.c @@ -230,6 +230,28 @@ JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_configureNonBlocking return res; } +JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_shutdown + (JNIEnv *e, jclass cl, jint fd) { + // Unlike POSIX, a Winsock shutdown() does not unblock a recv() that is + // already in progress on another thread. Cancel any outstanding I/O on the + // handle first so a worker parked in a blocking recv() wakes up. CancelIoEx + // does not close the socket, so the descriptor stays allocated and the fd + // cannot be reused underneath the worker before close() releases it. A + // FALSE return with ERROR_NOT_FOUND (nothing was pending) is expected here + // and deliberately ignored. + CancelIoEx((HANDLE) (SOCKET) fd, NULL); + const int result = shutdown((SOCKET) fd, SD_BOTH); + if (result == SOCKET_ERROR) { + const int error = WSAGetLastError(); + if (error == WSAENOTCONN) { + return 0; + } + WSASetLastError(error); + SaveLastError(); + } + return result; +} + JNIEXPORT jint JNICALL Java_io_questdb_client_network_Net_recv (JNIEnv *e, jclass cl, jint fd, jlong addr, jint len) { const int n = recv((SOCKET) fd, (char *) addr, len, 0); diff --git a/core/src/main/c/windows/os.c b/core/src/main/c/windows/os.c index 73bb4307..e50ce84c 100644 --- a/core/src/main/c/windows/os.c +++ b/core/src/main/c/windows/os.c @@ -237,3 +237,7 @@ JNIEXPORT jlong JNICALL Java_io_questdb_client_std_Os_currentTimeMicros void SaveLastError() { TlsSetValue(dwTlsIndexLastError, (LPVOID) (DWORD_PTR) GetLastError()); }; + +DWORD GetSavedLastError() { + return (DWORD) (DWORD_PTR) TlsGetValue(dwTlsIndexLastError); +} diff --git a/core/src/main/java/io/questdb/client/QuestDB.java b/core/src/main/java/io/questdb/client/QuestDB.java index 1a9aacae..a4c7cb03 100644 --- a/core/src/main/java/io/questdb/client/QuestDB.java +++ b/core/src/main/java/io/questdb/client/QuestDB.java @@ -130,10 +130,17 @@ static QuestDB connect(CharSequence configurationString) { Sender borrowSender(); /** - * Shuts down the pools, closing every underlying {@link Sender} and - * query client. Idempotent. Threads currently blocked in - * {@link #borrowSender()} or {@link Query#submit()} are released with an - * error. + * Shuts down the pools and their published clients. Idempotent. Threads + * currently blocked in {@link #borrowSender()} or {@link Query#submit()} + * are released with an error. + *

+ * In-progress client creation: close() waits up to the builder's + * {@link QuestDBBuilder#acquireTimeoutMillis(long) acquire timeout}, + * hard-capped at 5 seconds, for a creation blocked in DNS, TCP, TLS, or a + * WebSocket handshake. If that budget expires, the creator retains cleanup + * ownership and closes its unpublished client (and releases any SF slot) + * when construction returns. This keeps close() bounded even when + * {@code connect_timeout} is unset without abandoning late resources. *

* Outstanding leases: a borrowed {@link Sender} is never torn down * underneath the thread using it. Instead, close() waits up to the diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index 4ec70dbd..e2891201 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -819,17 +819,22 @@ enum InitialConnectMode { *

  • {@link #MEMORY} — never fsync explicitly. Bytes live in the OS * page cache; survive a JVM crash but not an OS crash. Default * and the lowest-latency setting.
    • - *
  • {@link #FLUSH} — fsync the active segment at every - * {@code Sender.flush()} (and at the implicit close-flush). One - * fsync per user flush, regardless of frame count.
    • - *
  • {@link #APPEND} — fsync after every individual frame append. - * Strongest guarantee, slowest path; pay a disk fsync per row.
    • + *
  • {@link #PERIODIC} — checkpoint published frames in the background + * at {@code sf_sync_interval_millis}. The configured interval is a + * target cadence; scheduler and storage latency add to the actual + * power-loss recovery window.
    • + *
  • {@link #FLUSH} — reserved for a future synchronous + * {@code Sender.flush()} barrier; currently rejected by + * {@code build()}.
    • + *
  • {@link #APPEND} — reserved for a future barrier after every frame + * append; currently rejected by {@code build()}.
    • * */ enum SfDurability { MEMORY, FLUSH, - APPEND + APPEND, + PERIODIC } /** @@ -972,6 +977,7 @@ final class LineSenderBuilder { // syscall cost so smaller segments give finer trim granularity and // make the cap arithmetic friendlier (cap / segment >> 2). private static final long DEFAULT_SEGMENT_BYTES = 4L * 1024 * 1024; + private static final long DEFAULT_SF_SYNC_INTERVAL_MILLIS = 5_000L; // Slot identity within sfDir. Each sender owns // and // takes an advisory exclusive lock there. Default "default" is fine for // single-sender deployments; multi-sender setups must set this explicitly @@ -1119,12 +1125,12 @@ public int getConnectTimeout() { // there is no separate on/off flag (presence of the directory is the switch). // null sfDir → memory-only async ingest (same lock-free architecture, no disk). private String sfDir; - // Durability contract for SF append/flush. Today only MEMORY is - // implemented; FLUSH and APPEND are deferred follow-ups (cursor needs - // to learn fsync first). + // Durability contract for SF append/flush. FLUSH and APPEND remain + // deferred follow-ups; PERIODIC uses the segment manager. private SfDurability sfDurability = SfDurability.MEMORY; private long sfMaxBytes = PARAMETER_NOT_SET_EXPLICITLY; private long sfMaxTotalBytes = PARAMETER_NOT_SET_EXPLICITLY; + private long sfSyncIntervalMillis = PARAMETER_NOT_SET_EXPLICITLY; private boolean shouldDestroyPrivKey; private boolean tlsEnabled; private TlsValidationMode tlsValidationMode; @@ -1443,9 +1449,9 @@ public Sender build() { // Setting sfDir enables store-and-forward (mmap'd, recoverable // across sender restarts); omitting it gives memory-only mode - // (same lock-free architecture, no disk involvement). The - // sf_durability != memory rejection lives in validateParameters - // so it is reached by build() and by no-connect validation alike. + // (same lock-free architecture, no disk involvement). + // Durability-combination validation lives in validateParameters + // so build() and no-connect validation apply the same rules. long actualSfMaxBytes = sfMaxBytes == PARAMETER_NOT_SET_EXPLICITLY ? DEFAULT_SEGMENT_BYTES : sfMaxBytes; @@ -1534,15 +1540,25 @@ public Sender build() { "could not create sf_dir: " + sfDir + " rc=" + rc); } } + if (sfDurability == SfDurability.PERIODIC + && Files.fsyncParentDir(sfDir) != 0) { + throw new LineSenderException( + "could not sync parent directory for sf_dir: " + sfDir); + } slotPath = sfDir + "/" + senderId; } long actualSfAppendDeadlineNanos = sfAppendDeadlineMillis == PARAMETER_NOT_SET_EXPLICITLY ? CursorSendEngine.DEFAULT_APPEND_DEADLINE_NANOS : sfAppendDeadlineMillis * 1_000_000L; + long actualSfSyncIntervalNanos = sfDurability == SfDurability.PERIODIC + ? (sfSyncIntervalMillis == PARAMETER_NOT_SET_EXPLICITLY + ? DEFAULT_SF_SYNC_INTERVAL_MILLIS : sfSyncIntervalMillis) * 1_000_000L + : 0L; CursorSendEngine cursorEngine = new CursorSendEngine( slotPath, actualSfMaxBytes, - actualSfMaxTotalBytes, actualSfAppendDeadlineNanos); + actualSfMaxTotalBytes, actualSfAppendDeadlineNanos, + actualSfSyncIntervalNanos); int actualErrorInboxCapacity = errorInboxCapacity != PARAMETER_NOT_SET_EXPLICITLY ? errorInboxCapacity : io.questdb.client.cutlass.qwp.client.sf.cursor.SenderErrorDispatcher.DEFAULT_CAPACITY; @@ -1623,7 +1639,8 @@ public Sender build() { orphans, maxBackgroundDrainers, actualSfMaxBytes, - actualSfMaxTotalBytes); + actualSfMaxTotalBytes, + actualSfSyncIntervalNanos); } } return connected; @@ -2701,20 +2718,18 @@ public LineSenderBuilder sfAppendDeadlineMillis(long millis) { * directory is the on-switch — there is no separate * enable/disable flag. SF is off iff {@code dir} was never set. *

    - * Every batch is persisted to disk before it leaves the wire and is - * reclaimed as soon as the server acknowledges it. On restart the - * sender replays only batches whose acknowledgement had not been - * received before the previous sender shut down — typically the last - * in-flight batches at close time. Acknowledged batches are not - * replayed: their disk space is freed during normal operation by an - * automatic per-frame trim that force-rotates the active segment - * once every frame in it has been acknowledged. + * The sender publishes each batch into a memory-mapped segment before + * transmission and reclaims acknowledged segments in the background. + * The default {@link SfDurability#MEMORY} mode relies on OS page-cache + * writeback: it survives a producer-process restart but not guaranteed + * host power loss. {@link SfDurability#PERIODIC} adds checked background + * storage barriers at the configured target cadence. *

    - * Note that {@link io.questdb.client.cutlass.qwp.client.QwpWebSocketSender#close()} - * under SF returns once data is on disk, not on server-ack, so a - * sender closed immediately after a flush may still have unacked - * batches in flight; those will be replayed by the next sender - * against the same directory. WebSocket transport only. + * On restart, the sender replays frames after its durable acknowledgement + * watermark. An acknowledgement that reached the server but not that + * watermark can replay, so applications that require row-level + * idempotence should configure server-side deduplication. + * WebSocket transport only. *

    * The sender takes ownership of the underlying SF storage and closes * it when the sender itself is closed. @@ -2734,7 +2749,10 @@ public LineSenderBuilder storeAndForwardDir(String dir) { /** * Selects the durability contract for SF appends and flushes. See - * {@link SfDurability} for the value semantics. + * {@link SfDurability} for the value semantics. The client currently + * supports {@link SfDurability#MEMORY} and + * {@link SfDurability#PERIODIC}; {@code build()} rejects the reserved + * {@code FLUSH} and {@code APPEND} values. *

    * Replaces the prior pair of independent {@code sf_fsync} and * {@code sf_fsync_on_flush} booleans — they were three states @@ -2788,6 +2806,22 @@ public LineSenderBuilder storeAndForwardMaxTotalBytes(long maxTotalBytes) { return this; } + /** + * Sets the target background checkpoint cadence for + * {@link SfDurability#PERIODIC}. Scheduler and storage latency add to + * the configured interval. Defaults to 5000 ms in periodic mode. + */ + public LineSenderBuilder storeAndForwardSyncIntervalMillis(long millis) { + if (protocol != PARAMETER_NOT_SET_EXPLICITLY && protocol != PROTOCOL_WEBSOCKET) { + throw new LineSenderException("store_and_forward is only supported for WebSocket transport"); + } + if (millis <= 0L || millis > Long.MAX_VALUE / 1_000_000L) { + throw new LineSenderException("sf_sync_interval_millis is out of range: ").put(millis); + } + this.sfSyncIntervalMillis = millis; + return this; + } + private static boolean charsEqualsRange(CharSequence a, CharSequence b, int bStart, int bEnd) { int len = bEnd - bStart; if (a.length() != len) { @@ -2810,10 +2844,11 @@ private static int getValue(CharSequence configurationString, int pos, StringSin private static SfDurability parseDurabilityValue(@NotNull StringSink value) { if (Chars.equalsIgnoreCase("memory", value)) return SfDurability.MEMORY; + if (Chars.equalsIgnoreCase("periodic", value)) return SfDurability.PERIODIC; if (Chars.equalsIgnoreCase("flush", value)) return SfDurability.FLUSH; if (Chars.equalsIgnoreCase("append", value)) return SfDurability.APPEND; throw new LineSenderException("invalid sf_durability [value=").put(value) - .put(", allowed-values=[memory, flush, append]]"); + .put(", allowed-values=[memory, periodic, flush, append]]"); } private static int parseIntValue(@NotNull StringSink value, @NotNull String name) { @@ -3370,6 +3405,12 @@ private LineSenderBuilder fromConfig(CharSequence configurationString) { } pos = getValue(configurationString, pos, sink, "sf_durability"); storeAndForwardDurability(parseDurabilityValue(sink)); + } else if (Chars.equals("sf_sync_interval_millis", sink)) { + if (protocol != PROTOCOL_WEBSOCKET) { + throw new LineSenderException("sf_sync_interval_millis is only supported for WebSocket transport"); + } + pos = getValue(configurationString, pos, sink, "sf_sync_interval_millis"); + storeAndForwardSyncIntervalMillis(parseLongValue(sink, "sf_sync_interval_millis")); } else if (Chars.equals("close_flush_timeout_millis", sink)) { if (protocol != PROTOCOL_WEBSOCKET) { throw new LineSenderException("close_flush_timeout_millis is only supported for WebSocket transport"); @@ -3695,6 +3736,9 @@ private LineSenderBuilder fromConfigWebSocket(CharSequence configurationString) if (view.has("sf_max_total_bytes")) { storeAndForwardMaxTotalBytes(wsSize(view, v, "sf_max_total_bytes")); } + if (view.has("sf_sync_interval_millis")) { + storeAndForwardSyncIntervalMillis(wsLong(view, v, "sf_sync_interval_millis")); + } s = view.getStr("sf_dir"); if (s != null) { @@ -3852,6 +3896,7 @@ public java.util.Map wsConfigSnapshotForTest() { m.put("sf_max_total_bytes", sfMaxTotalBytes); m.put("sf_durability", sfDurability == null ? null : sfDurability.name()); m.put("sf_append_deadline_millis", sfAppendDeadlineMillis); + m.put("sf_sync_interval_millis", sfSyncIntervalMillis); m.put("close_flush_timeout_millis", closeFlushTimeoutMillis); m.put("durable_ack_keepalive_interval_millis", durableAckKeepaliveIntervalMillis); m.put("initial_connect_retry", initialConnectMode == null ? null : initialConnectMode.name()); @@ -4057,14 +4102,18 @@ private void validateParameters() { if (autoFlushIntervalMillis == Integer.MAX_VALUE) { throw new LineSenderException("disabling auto-flush is not supported for WebSocket protocol"); } - // The cursor send path does not fsync yet, so any sf_durability - // other than memory is rejected rather than silently downgraded. - // Validating it here (rather than at connect time) lets a - // no-connect config check reject it as a full build() does. - if (sfDurability != SfDurability.MEMORY) { + if ((sfDurability == SfDurability.FLUSH || sfDurability == SfDurability.APPEND)) { throw new LineSenderException( "sf_durability=" + sfDurability.name().toLowerCase() - + " is not yet supported (deferred follow-up; use sf_durability=memory)"); + + " is not yet supported (use sf_durability=memory or periodic)"); + } + if (sfDurability == SfDurability.PERIODIC && sfDir == null) { + throw new LineSenderException("sf_durability=periodic requires sf_dir"); + } + if (sfSyncIntervalMillis != PARAMETER_NOT_SET_EXPLICITLY + && sfDurability != SfDurability.PERIODIC) { + throw new LineSenderException( + "sf_sync_interval_millis requires sf_durability=periodic"); } } else { throw new LineSenderException("unsupported protocol ") diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java index fd16dca7..94018fff 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java @@ -248,6 +248,15 @@ public void close() { } } + /** + * Shuts down socket traffic without releasing the descriptor or freeing + * buffers that a concurrent I/O worker may still access. The owner must + * call {@link #close()} after joining the worker to complete cleanup. + */ + public void closeTraffic() { + socket.closeTraffic(); + } + /** * Connects to a WebSocket server. * diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java index b63b4fa9..f6587631 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java @@ -165,6 +165,10 @@ public class QwpQueryClient implements QuietCloseable { private final Random failoverRandom = new Random(); private long authTimeoutMs = DEFAULT_AUTH_TIMEOUT_MS; private String authorizationHeader; + // Deterministic lifecycle barrier used by facade shutdown tests. Null in + // production; close() clears and invokes it after winning close ownership + // without allowing hook failures to prevent resource teardown. + private volatile Runnable beforeCloseHook; // Upper bound (ms) on each TCP connect attempt. 0 (default) falls back to // the OS connect timeout. private int connectTimeoutMs = 0; @@ -620,6 +624,16 @@ public void close() { // scratch, double-freeing it. return; } + Runnable hook = beforeCloseHook; + beforeCloseHook = null; + if (hook != null) { + try { + hook.run(); + } catch (Throwable ignored) { + // Omit diagnostics for this test-only hook: even rendering its + // failure must not prevent production resource cleanup. + } + } connected = false; lastCloseTimedOut = false; try { @@ -1004,6 +1018,11 @@ public void seedFailoverRandomForTest(long seed) { } } + @TestOnly + public void setBeforeCloseHookForTest(Runnable hook) { + beforeCloseHook = hook; + } + /** * Returns true if the most recent {@link #close()} call abandoned the I/O thread * because it failed to exit within the join timeout. The native buffer pool and diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index d1744065..5a813c83 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -39,6 +39,7 @@ import io.questdb.client.cutlass.line.LineSenderException; import io.questdb.client.cutlass.line.array.DoubleArray; import io.questdb.client.cutlass.line.array.LongArray; +import io.questdb.client.cutlass.qwp.client.sf.cursor.BackgroundDrainer; import io.questdb.client.cutlass.qwp.client.sf.cursor.BackgroundDrainerListener; import io.questdb.client.cutlass.qwp.client.sf.cursor.BackgroundDrainerPool; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; @@ -189,11 +190,19 @@ public class QwpWebSocketSender implements Sender { // Null in production; set reflectively by tests. @TestOnly private volatile java.util.function.Supplier clientFactoryOverride; + // Test-only lifecycle witness. drainOnClose() invokes and clears it after + // confirming a real unacknowledged close target, immediately before waiting. + private volatile Runnable closeDrainWaitingHook; // close() drain timeout in millis. Default applied at construction. // 0 or -1 means "fast close" (skip the drain); otherwise close blocks // up to this many millis for ackedFsn to catch up to publishedFsn. + private volatile boolean closeCleanupComplete; + private boolean closeCleanupStarted; private long closeFlushTimeoutMillis = 5_000L; private volatile boolean closed; + // Test-only lifecycle witness. close() invokes and clears it strictly after + // publishing closed=true and before starting any drain or teardown work. + private volatile Runnable closeStartedHook; private boolean connected; private SenderConnectionDispatcher connectionDispatcher; // Async-delivery sink for SenderConnectionEvent notifications. Default @@ -278,11 +287,23 @@ public class QwpWebSocketSender implements Sender { private boolean ownsCursorEngine; private long pendingBytes; // Set true by close() once the SF slot flock has been released (the normal - // teardown path). Stays false if close() bailed early with the I/O thread - // still running -- then cursorEngine.close() never ran and the flock is - // still held, so the owning pool MUST keep the slot reserved rather than - // hand the still-locked dir to the next borrow ("sf slot already in use"). - private boolean slotLockReleased; + // teardown path). Stays false if an I/O or manager worker did not stop and + // cursorEngine retained the flock, so the owning pool MUST keep the slot + // reserved rather than hand the still-locked dir to the next borrow + // ("sf slot already in use"). May flip to true LATER via the getter's + // re-probe of retainedEngine, once the deferred cleanup (manager-worker + // exit path or delegated I/O-thread close) releases the flock — pools + // re-probe retired slots to recover their capacity. volatile: written on + // the closing thread, read by pool threads. + private volatile boolean slotLockReleased; + // Optional owning-pool notification, relayed after the engine confirms the + // flock release and slotLockReleased is visible to pool re-probes. + private volatile Runnable slotLockReleaseListener; + // Engine whose close() could not complete during sender close() — its + // cleanup is pending on a worker/I/O-thread exit path. isSlotLockReleased() + // re-probes it so a late flock release becomes visible to the owning pool. + // Only ever set inside close(); null for a sender that closed cleanly. + private volatile CursorSendEngine retainedEngine; private int pendingRowCount; private SenderProgressDispatcher progressDispatcher; // Async-delivery sink for ack-watermark advances. Default no-op; a @@ -847,9 +868,12 @@ public boolean awaitAckedFsn(long targetFsn, long timeoutMillis) { if (cursorEngine == null) { return targetFsn < 0L; } - // Surface latched I/O errors before any early-return path, so a - // caller polling with timeoutMillis <= 0 to drive their own loop - // sees the terminal throw instead of an indefinite "not yet". + cursorEngine.checkDurability(); + // Surface latched errors before any early-return path, so a caller + // polling with timeoutMillis <= 0 to drive their own loop sees the + // throw instead of an indefinite "not yet". The durability latch + // above is transient: it throws while latched, and clears once a + // later periodic sync pass fully succeeds so producers can resume. if (cursorSendLoop != null) { cursorSendLoop.checkError(); } @@ -862,6 +886,7 @@ public boolean awaitAckedFsn(long targetFsn, long timeoutMillis) { } long deadlineNanos = System.nanoTime() + timeoutMillis * 1_000_000L; while (cursorEngine.ackedFsn() < targetFsn) { + cursorEngine.checkDurability(); if (cursorSendLoop != null) { cursorSendLoop.checkError(); } @@ -1038,13 +1063,26 @@ public QwpWebSocketSender charColumn(CharSequence columnName, char value) { * replaying frames get a 2.5s grace window plus a 0.5s stop window * — worst case ~3s when a drainer sits in a blocking native * connect (15s background deadline) and must be abandoned to exit - * on its own. + * on its own; + *

  • SF manager stop: normally immediate, bounded by 5s when its worker + * is stuck in a filesystem operation. On timeout the slot remains + * locked rather than being exposed to a stale worker.
    • * */ @Override public void close() { if (!closed) { closed = true; + Runnable hook = closeStartedHook; + closeStartedHook = null; + if (hook != null) { + try { + hook.run(); + } catch (Throwable t) { + // A test witness must never prevent production resource cleanup. + LOG.error("Error in close-started test hook: {}", String.valueOf(t)); + } + } boolean ioThreadStopped = true; // Captures the first error from the flush/drain path AND any // secondary errors from cleanup steps (added via addSuppressed). @@ -1175,106 +1213,26 @@ public void close() { } if (!ioThreadStopped) { - // The I/O thread may still be using the socket and microbatch - // buffers. Freeing them would risk SIGSEGV. - LOG.error("I/O thread is still running, leaking WebSocket client and microbatch buffers"); - // The engine, however, need not leak: delegate its close to - // the I/O thread's exit path, which runs it strictly after - // the thread's last engine access — the mapping and slot - // lock release as soon as the stuck wire call resolves - // (bounded by OS timeouts). slotLockReleased intentionally - // stays false: the lock is released only when the delegated - // close actually runs, so the pool must not reuse the slot - // meanwhile. A false return means the thread exited between - // the failed close() and now — then closing here is safe. - if (ownsCursorEngine && cursorEngine != null && cursorSendLoop != null - && !cursorSendLoop.delegateEngineClose()) { - try { - cursorEngine.close(); - } catch (Throwable t) { - LOG.error("Error closing owned CursorSendEngine: {}", String.valueOf(t)); - terminalError = captureCloseError(terminalError, t); - } - cursorEngine = null; - ownsCursorEngine = false; - slotLockReleased = true; - } - rethrowTerminal(terminalError); - return; - } - - if (buffer0 != null) { - try { - buffer0.close(); - } catch (Throwable t) { - LOG.error("Error closing buffer0: {}", String.valueOf(t)); - terminalError = captureCloseError(terminalError, t); - } - } - if (buffer1 != null) { - try { - buffer1.close(); - } catch (Throwable t) { - LOG.error("Error closing buffer1: {}", String.valueOf(t)); - terminalError = captureCloseError(terminalError, t); - } - } - - if (client != null) { - try { - client.close(); - } catch (Throwable t) { - LOG.error("Error closing WebSocket client: {}", String.valueOf(t)); - terminalError = captureCloseError(terminalError, t); - } - client = null; - } - - if (ownsCursorEngine && cursorEngine != null) { - try { - cursorEngine.close(); - } catch (Throwable t) { - LOG.error("Error closing owned CursorSendEngine: {}", String.valueOf(t)); - terminalError = captureCloseError(terminalError, t); - } - cursorEngine = null; - ownsCursorEngine = false; - } - // Past the ioThreadStopped guard => cursorEngine.close() ran and - // released the SF flock in its finally (or this sender owned no - // engine holding one). Signal the pool it may reuse the slot. - slotLockReleased = true; - - // Shutdown order: dispatcher last, after the I/O loop has stopped - // producing into it. close() drains pending entries with a short - // deadline so any final errors land in the user's handler. - if (errorDispatcher != null) { - try { - errorDispatcher.close(); - } catch (Throwable t) { - LOG.error("Error closing error dispatcher: {}", String.valueOf(t)); - terminalError = captureCloseError(terminalError, t); - } - } - if (progressDispatcher != null) { - try { - progressDispatcher.close(); - } catch (Throwable t) { - LOG.error("Error closing progress dispatcher: {}", String.valueOf(t)); - terminalError = captureCloseError(terminalError, t); + // The worker may still touch every resource below. Hand the + // complete sender-owned tail to its exit path rather than + // permanently leaking everything except the engine. The + // callback is idempotence-gated by closeRemainingResources(). + if (ownsCursorEngine && cursorEngine != null) { + retainedEngine = cursorEngine; } - } - if (connectionDispatcher != null) { - try { - connectionDispatcher.close(); - } catch (Throwable t) { - LOG.error("Error closing connection dispatcher: {}", String.valueOf(t)); - terminalError = captureCloseError(terminalError, t); + Runnable closeCallback = () -> closeRemainingResources(null); + if (cursorSendLoop != null && cursorSendLoop.delegateClose(closeCallback)) { + rethrowTerminal(terminalError); + return; } + // The worker exited between close() failing and delegation. + // Cleanup is safe here and its failures remain suppressed on + // the original close error. + terminalError = closeRemainingResources(terminalError); + } else { + terminalError = closeRemainingResources(terminalError); } - LOG.info("QwpWebSocketSender closed"); - // If close() ended up holding the same instance the user already // caught earlier, suppress the rethrow. The user's catch block // wraps close() (try-with-resources), and Throwable refuses @@ -1286,14 +1244,62 @@ public void close() { } } + @TestOnly + public boolean isCloseCleanupComplete() { + return closeCleanupComplete; + } + /** - * True once {@link #close()} has released the store-and-forward slot - * flock. False means close() leaked the still-running I/O thread (and its - * resources), so the flock is still held; the owning pool must keep the - * slot index reserved instead of reusing the still-locked slot dir. + * True once the store-and-forward slot flock has been released. False + * means an I/O or manager worker did not stop and close() retained the + * lock and worker-reachable resources; the owning pool must keep the slot + * index reserved instead of reusing the still-locked dir. + *

    + * Not a one-shot snapshot: when close() left engine cleanup pending on a + * manager-worker quiescence or I/O-thread exit path, this re-probes the + * retained engine and latches true the moment that cleanup completes — pools re-probe retired + * slots through this getter to recover their capacity. Monotonic: + * false→true only, never back. Cheap (volatile reads on every common + * path) so pools may call it under their capacity lock; only the rare + * orphaned-retry state below does more. + *

    + * The probe is also the recovery surface for a retained engine whose + * flock-release retry fell off the shared driver because the driver + * thread could not start (e.g. OOM at thread creation): close() is + * one-shot, so without the re-arm below that slot's capacity would stay + * lost until process exit. */ public boolean isSlotLockReleased() { - return slotLockReleased; + if (slotLockReleased) { + return true; + } + CursorSendEngine engine = retainedEngine; + if (engine != null) { + if (engine.isCloseCompleted()) { + // Benign latch race: concurrent callers may both observe the + // completed cleanup and both write true. + slotLockReleased = true; + return true; + } + engine.ensureFlockReleaseRetryScheduled(); + } + return false; + } + + /** + * Registers a callback for confirmed SF slot-lock release. Pools use this + * to wake borrowers without waiting for a timeout or housekeeper tick. + */ + public void setSlotLockReleaseListener(Runnable listener) { + slotLockReleaseListener = listener; + if (listener != null && slotLockReleased) { + listener.run(); + } + } + + @TestOnly + public void setSlotLockReleasedForTesting(boolean isReleased) { + slotLockReleased = isReleased; } @Override @@ -1518,15 +1524,19 @@ public void flush() { @Override public long flushAndGetSequence() { checkNotClosed(); + if (cursorEngine != null) { + cursorEngine.checkDurability(); + } ensureNoInProgressRow(); ensureConnected(); long beforeFsn = cursorEngine != null ? cursorEngine.publishedFsn() : -1L; - // Cursor SF: SF.append happens on the user thread inside - // sealAndSwapBuffer, so by the time we reach here every encoded - // batch is durable on its mmap'd segment. No processingCount to - // drain, no awaitPendingAcks. Just surface any I/O thread error. + // Cursor SF: append happens on the user thread inside + // sealAndSwapBuffer, so by the time we reach here every encoded batch + // is published in its mmap'd segment. PERIODIC stable-storage barriers + // run independently in the manager. No processingCount to drain and no + // awaitPendingAcks here; just surface any I/O thread error. flushPendingRows(deferCommit); if (!deferCommit && hasDeferredMessages) { sendCommitMessage(); @@ -1709,6 +1719,36 @@ public long getDroppedConnectionNotifications() { return d == null ? 0L : d.getDroppedNotifications(); } + @TestOnly + public SenderConnectionDispatcher getConnectionDispatcherForTesting() { + return connectionDispatcher; + } + + @TestOnly + public CursorSendEngine getCursorEngineForTesting() { + return cursorEngine; + } + + @TestOnly + public SenderErrorDispatcher getErrorDispatcherForTesting() { + return errorDispatcher; + } + + @TestOnly + public Sender.InitialConnectMode getInitialConnectModeForTesting() { + return initialConnectMode; + } + + @TestOnly + public SenderProgressDispatcher getProgressDispatcherForTesting() { + return progressDispatcher; + } + + @TestOnly + public Runnable getSlotLockReleaseListenerForTesting() { + return slotLockReleaseListener; + } + /** * Number of {@link SenderError} notifications dropped because the * bounded inbox was full. Non-zero means the user-supplied @@ -2135,6 +2175,36 @@ public void setClientFactoryOverride(java.util.function.Supplier live = + ObjList live = pool.snapshot(); for (int i = 0, n = live.size(); i < n; i++) { live.getQuick(i).setListener(listener); @@ -2334,6 +2431,25 @@ public synchronized void startOrphanDrainers( int maxBackgroundDrainers, long segmentSizeBytes, long sfMaxTotalBytes + ) { + startOrphanDrainers( + orphanSlotPaths, + maxBackgroundDrainers, + segmentSizeBytes, + sfMaxTotalBytes, + 0L); + } + + /** + * Starts orphan drainers while preserving the foreground sender's periodic + * store-and-forward checkpoint interval. + */ + public synchronized void startOrphanDrainers( + io.questdb.client.std.ObjList orphanSlotPaths, + int maxBackgroundDrainers, + long segmentSizeBytes, + long sfMaxTotalBytes, + long syncIntervalNanos ) { if (orphanSlotPaths == null || orphanSlotPaths.size() == 0 || maxBackgroundDrainers <= 0) { @@ -2360,17 +2476,18 @@ public synchronized void startOrphanDrainers( // (the factory needs the drainer, the drainer's constructor // needs the factory); the ref write happens-before the drainer // runs because submit() publishes the task afterwards. - final io.questdb.client.cutlass.qwp.client.sf.cursor.BackgroundDrainer[] ref = - new io.questdb.client.cutlass.qwp.client.sf.cursor.BackgroundDrainer[1]; + final BackgroundDrainer[] ref = + new BackgroundDrainer[1]; ReconnectSupplier factory = new ReconnectSupplier( () -> { - io.questdb.client.cutlass.qwp.client.sf.cursor.BackgroundDrainer d = ref[0]; + BackgroundDrainer d = ref[0]; return d != null && d.isStopRequested(); }, "drainer stop requested during connect"); - io.questdb.client.cutlass.qwp.client.sf.cursor.BackgroundDrainer drainer = - new io.questdb.client.cutlass.qwp.client.sf.cursor.BackgroundDrainer( + BackgroundDrainer drainer = + new BackgroundDrainer( slot, segmentSizeBytes, sfMaxTotalBytes, + syncIntervalNanos, factory, reconnectMaxDurationMillis, reconnectInitialBackoffMillis, @@ -2705,22 +2822,37 @@ private WebSocketClient newWebSocketClient() { * reconnect and {@code close()} paths are therefore never queued * behind a drainer's endpoint walk. */ - private WebSocketClient buildAndConnect(ReconnectSupplier ctx) { + private WebSocketClient buildAndConnect(ReconnectSupplier ctx, CursorWebSocketSendLoop.ConnectCancellation cancellation) { if (ctx.isBackground()) { // Lock-free: the walk below touches only internally-synchronized // hostTracker health state and walk-local/cursor-local state on // the background path. - return connectWalk(ctx); + return connectWalk(ctx, cancellation); } connectWalkLock.lock(); try { - return connectWalk(ctx); + return connectWalk(ctx, cancellation); } finally { connectWalkLock.unlock(); } } - private WebSocketClient connectWalk(ReconnectSupplier ctx) { + // Drop the in-flight connect handle once the walk has disposed a client on + // a connect/upgrade FAILURE, so inFlight never dangles at a disposed client + // across the inter-attempt backoff. Without this a concurrent + // close()->cancel() could closeTraffic() a client the walk no longer owns; + // proven a no-op on both production transports today (closed-fd closeTraffic + // no-ops), but a future custom transport that threw on a closed socket would + // spuriously loud-fail close(). Null-guarded: the no-arg reconnect() path + // and the Unsafe.allocateInstance bare-loop tests pass a null handle. Pairs + // with the success-path clear() after upgrade(). + private static void clearInFlight(CursorWebSocketSendLoop.ConnectCancellation cancellation) { + if (cancellation != null) { + cancellation.clear(); + } + } + + private WebSocketClient connectWalk(ReconnectSupplier ctx, CursorWebSocketSendLoop.ConnectCancellation cancellation) { // Background (drainer) factories share this connect walk -- endpoint // list and hostTracker HEALTH state (never the shared round: a // background sweep walks its own RoundCursor and records with @@ -2808,9 +2940,34 @@ private WebSocketClient connectWalk(ReconnectSupplier ctx) { newClient.setQwpClientId(QwpConstants.CLIENT_ID); newClient.setQwpRequestDurableAck(requestDurableAck); newClient.setConnectTimeout(effectiveConnectTimeoutMs(background, connectTimeoutMs)); + if (cancellation != null) { + // Publish the client we are about to block on so a + // concurrent CursorWebSocketSendLoop.close() can break its + // traffic and unwind a black-holed native connect + // (connect_timeout=0 => OS SYN-retry) instead of hanging on + // the untimed shutdown-latch await. The publish-then-check + // handshake pairs with ConnectCancellation.cancel(): if we + // observe cancellation here we skip the blocking connect + // entirely; otherwise cancel() observed this client and + // breaks it. The walk's per-attempt catch disposes the + // client and, since running has flipped false, the + // top-of-loop ctx.isAborted() gate ends the walk. + cancellation.publish(newClient); + if (cancellation.isCancelled()) { + throw new LineSenderException(ctx.abortMessage()); + } + } newClient.connect(ep.host, ep.port); int upgradeTimeoutMs = (int) Math.min(authTimeoutMs, Integer.MAX_VALUE); newClient.upgrade(WRITE_PATH, upgradeTimeoutMs, authorizationHeader); + if (cancellation != null) { + // connect()+upgrade() completed: this client is no longer + // blocking, so drop it from the in-flight handle before it + // becomes the loop's `client` field. close() must then break + // its traffic via the field path exactly once -- leaving it + // in the handle too would double-shut-down the socket. + cancellation.clear(); + } } catch (HttpClientException e) { // Close BEFORE classify: the sibling catch (Error) below does not // guard catch-arm bodies, so an Error thrown inside classify() @@ -2820,6 +2977,7 @@ private WebSocketClient connectWalk(ReconnectSupplier ctx) { // upgradeStatusCode) that are set during upgrade() and survive // close(). newClient.close(); + clearInFlight(cancellation); HttpClientException classified = QwpUpgradeFailures.classify(newClient, ep.host, ep.port, e); if (classified instanceof QwpIngressRoleRejectedException) { QwpIngressRoleRejectedException re = (QwpIngressRoleRejectedException) classified; @@ -2866,6 +3024,7 @@ private WebSocketClient connectWalk(ReconnectSupplier ctx) { continue; } catch (Exception e) { newClient.close(); + clearInFlight(cancellation); hostTracker.recordTransportError(idx, !background); lastError = e; if (!background) { @@ -2888,6 +3047,7 @@ private WebSocketClient connectWalk(ReconnectSupplier ctx) { // the cursor reconnect loop, BackgroundDrainer) rethrows Error // rather than retrying, so this stays a loud one-shot failure. closeQuietlyOnError(newClient); + clearInFlight(cancellation); throw e; } // Guard the post-upgrade tail: from here until newClient is @@ -3084,6 +3244,83 @@ private void checkTableSelected() { } } + private synchronized Throwable closeRemainingResources(Throwable terminalError) { + if (closeCleanupStarted) { + return terminalError; + } + closeCleanupStarted = true; + try { + try { + buffer0.close(); + } catch (Throwable t) { + LOG.error("Error closing buffer0: {}", String.valueOf(t)); + terminalError = captureCloseError(terminalError, t); + } + try { + buffer1.close(); + } catch (Throwable t) { + LOG.error("Error closing buffer1: {}", String.valueOf(t)); + terminalError = captureCloseError(terminalError, t); + } + if (client != null) { + try { + client.close(); + } catch (Throwable t) { + LOG.error("Error closing WebSocket client: {}", String.valueOf(t)); + terminalError = captureCloseError(terminalError, t); + } + client = null; + } + if (ownsCursorEngine && cursorEngine != null) { + CursorSendEngine engine = cursorEngine; + try { + engine.close(); + } catch (Throwable t) { + LOG.error("Error closing owned CursorSendEngine: {}", String.valueOf(t)); + terminalError = captureCloseError(terminalError, t); + } + if (engine.isCloseCompleted()) { + cursorEngine = null; + ownsCursorEngine = false; + slotLockReleased = true; + } else { + slotLockReleased = false; + retainedEngine = engine; + } + } else { + slotLockReleased = true; + } + if (errorDispatcher != null) { + try { + errorDispatcher.close(); + } catch (Throwable t) { + LOG.error("Error closing error dispatcher: {}", String.valueOf(t)); + terminalError = captureCloseError(terminalError, t); + } + } + if (progressDispatcher != null) { + try { + progressDispatcher.close(); + } catch (Throwable t) { + LOG.error("Error closing progress dispatcher: {}", String.valueOf(t)); + terminalError = captureCloseError(terminalError, t); + } + } + if (connectionDispatcher != null) { + try { + connectionDispatcher.close(); + } catch (Throwable t) { + LOG.error("Error closing connection dispatcher: {}", String.valueOf(t)); + terminalError = captureCloseError(terminalError, t); + } + } + LOG.info("QwpWebSocketSender closed"); + return terminalError; + } finally { + closeCleanupComplete = true; + } + } + private int countNonEmptyTables(ObjList keys) { int tableCount = 0; for (int i = 0, n = keys.size(); i < n; i++) { @@ -3185,6 +3422,16 @@ private void drainOnClose(boolean errorOwnedByCustomHandler) { if (cursorEngine.ackedFsn() >= target) { return; } + Runnable hook = closeDrainWaitingHook; + closeDrainWaitingHook = null; + if (hook != null) { + try { + hook.run(); + } catch (Throwable t) { + // A test witness must never prevent production resource cleanup. + LOG.error("Error in close-drain-waiting test hook: {}", String.valueOf(t)); + } + } long deadlineNanos = System.nanoTime() + closeFlushTimeoutMillis * 1_000_000L; while (cursorEngine.ackedFsn() < target) { // Stop on a latched terminal (acks will never reach target); @@ -3545,6 +3792,14 @@ private void flushPendingRowsSplit(ObjList keys, boolean deferComm resetTableBuffersAfterFlush(keys); } + private void onSlotLockReleased() { + slotLockReleased = true; + Runnable listener = slotLockReleaseListener; + if (listener != null) { + listener.run(); + } + } + private void resetTableBuffersAfterFlush(ObjList keys) { for (int i = 0, n = keys.size(); i < n; i++) { CharSequence tableName = keys.getQuick(i); @@ -3850,7 +4105,12 @@ boolean isAborted() { @Override public WebSocketClient reconnect() { - return buildAndConnect(this); + return buildAndConnect(this, null); + } + + @Override + public WebSocketClient reconnect(CursorWebSocketSendLoop.ConnectCancellation cancellation) { + return buildAndConnect(this, cancellation); } } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/AckWatermark.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/AckWatermark.java index 469c91ea..b11ae561 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/AckWatermark.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/AckWatermark.java @@ -24,10 +24,13 @@ package io.questdb.client.cutlass.qwp.client.sf.cursor; +import io.questdb.client.std.Crc32c; import io.questdb.client.std.Files; +import io.questdb.client.std.FilesFacade; import io.questdb.client.std.MemoryTag; import io.questdb.client.std.QuietCloseable; import io.questdb.client.std.Unsafe; +import org.jetbrains.annotations.TestOnly; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,191 +45,267 @@ * "everything <= N is durable"), so a single monotonic watermark suffices; * no per-frame bitmap is needed. *

    - * Layout (16 bytes, little-endian, mmap'd for the engine's lifetime): + * Layout (8192 bytes, little-endian, mmap'd for the engine's lifetime): + * two independently CRC-protected 64-byte records at offsets 0 and 4096. + * Placing each record at the start of a separate 4 KiB slot prevents one + * aligned 512-byte or 4 KiB sector tear from damaging both. Each record contains: * 

    - *   offset 0:  u32 magic = 'AKW1' (set once on first write)
- *   offset 4:  u32 reserved (zero)
- *   offset 8:  i64 fsn
+ *   offset 0:   u32 magic = 'AKW1'
+ *   offset 4:   u32 version = 1
+ *   offset 8:   i64 generation
+ *   offset 16:  i64 fsn
+ *   offset 24:  reserved (zero-filled through offset 59)
+ *   offset 60:  u32 CRC32C of bytes [0, 60)
  *
    + * {@link #write(long)} rewrites the record not selected by the current + * generation and stores its CRC last. Recovery selects the valid record with + * the greatest generation. A torn update therefore falls back to the older + * valid record; if neither record validates, recovery conservatively uses the + * segment-derived seed. *

    - * Zero-alloc, zero-syscall writes: {@link #open(String)} opens the - * file and maps the 16 bytes once. {@link #write(long)} is a single - * 8-byte aligned {@code Unsafe.putLong} into the mapped region. No - * malloc/free, no read/write syscalls, no rename, on the manager's hot - * tick path. An 8-byte aligned store is hardware-atomic on x86_64 and - * arm64, and disk-atomic within one sector (the file is 16 bytes — - * trivially within one sector), so a torn FSN across a crash boundary - * is not a concern. + * Zero-alloc, store-only ACK writes: {@link #open(String)} maps both + * records once. Ordinary ACK-only manager updates mutate the inactive record + * in the mapping and require no malloc/free, read/write syscalls, or rename. *

    - * Why no CRC: the watermark is an optimization. If the on-disk - * value is somehow corrupted (filesystem bug, hardware fault), the - * recovery path's {@code max(lowestBase - 1, watermark)} clamp absorbs - * the inconsistency: a stale-low watermark just means more re-replay, a - * stale-high watermark is impossible because no write produces an FSN - * higher than the segments on disk could account for. CRC adds - * complexity (multi-store with fence + read-side validate) for - * marginal additional safety. + * fsync cadence: ordinary ACK-only manager updates call + * {@link #write(long)} and stay syscall-free. Each non-empty background disk-trim + * quantum calls {@link #sync()} once (one mmap msync and one fd fsync), fsyncs + * the slot directory before unlinking, and fsyncs it again after the batch. A + * fully drained close uses the same covering order so the durable watermark + * guards any acknowledged segment that a host crash restores. *

    - * fsync cadence: intentionally NOT performed. Host crash falls - * back to recovery's {@code lowestBase - 1} seed (same as before this - * feature, no regression), at the cost of a bounded re-replay window - * for whatever durable-acks landed since the last successful page - * cache flush. - *

    - * Lifecycle: single-writer (the {@link SegmentManager} worker - * thread) after construction. Read once at engine startup (any thread, - * before the manager observes the entry). Close releases the mapping - * and fd. Not thread-safe for concurrent writers. + * Lifecycle: single-writer (the {@link SegmentManager} worker thread) + * after construction. Read once at engine startup (any thread, before the + * manager observes the entry). Close releases the mapping and fd. Not + * thread-safe for concurrent writers. */ public final class AckWatermark implements QuietCloseable { /** - * Filename of the watermark within the slot directory. - * Dot-prefixed so directory enumerators that filter by extension - * (segment recovery, OrphanScanner) skip it automatically. + * Filename of the watermark within the slot directory. Dot-prefixed so + * directory enumerators that filter by extension skip it automatically. */ public static final String FILE_NAME = ".ack-watermark"; - public static final int FILE_SIZE = 16; + public static final int FILE_SIZE = 8 * 1024; /** - * Sentinel returned by {@link #read()} when the watermark file is - * present but has not yet been written to (the magic field is zero - * because the OS zero-filled the freshly created file). + * Sentinel returned by {@link #read()} when neither watermark record is + * valid. */ public static final long INVALID = Long.MIN_VALUE; static final int FILE_MAGIC = 0x31574B41; // 'AKW1' little-endian - private static final int FSN_OFFSET = 8; + private static final int CRC_OFFSET = 60; + private static final int FSN_OFFSET = 16; private static final Logger LOG = LoggerFactory.getLogger(AckWatermark.class); private static final int MAGIC_OFFSET = 0; + private static final int RECORD_SIZE = 64; + private static final int RECORD_SLOT_SIZE = 4 * 1024; + private static final int VERSION = 1; private final int fd; + private final FilesFacade filesFacade; private final long mmapAddress; private boolean closed; - // Stamped once per process either at open() (if the file already - // has the magic from a prior session) or on the first write() that - // observes it unset. After the flag flips, write() degenerates to - // a single 8-byte putLong against the mapped FSN slot -- no memory - // load of magic on the hot path. Manager-thread-only after - // construction; no synchronisation needed. - private boolean magicWritten; - - private AckWatermark(int fd, long mmapAddress, boolean magicAlreadyWritten) { + private long fsn; + private long generation; + private boolean isStorageReleased; + + private AckWatermark(FilesFacade filesFacade, int fd, long mmapAddress, + long generation, long fsn) { this.fd = fd; + this.filesFacade = filesFacade; + this.fsn = fsn; + this.generation = generation; this.mmapAddress = mmapAddress; - this.magicWritten = magicAlreadyWritten; } @Override public void close() { if (closed) return; closed = true; - if (mmapAddress != 0L && mmapAddress != Files.FAILED_MMAP_ADDRESS) { - Files.munmap(mmapAddress, FILE_SIZE, MemoryTag.MMAP_DEFAULT); - } - if (fd >= 0) { - Files.close(fd); - } + releaseStorage(); } /** - * Opens (creating if absent) the watermark file in {@code slotDir} - * and maps it for the engine's lifetime. Returns {@code null} on - * any setup failure (open fail, mmap fail, unexpected size) — the - * caller falls back to the no-watermark behaviour, no exception - * escapes. Idempotent at the engine layer: a stale file from a - * prior session is reused as-is; the first {@link #write(long)} - * stamps the magic and the new FSN atomically. + * Opens (creating if absent) the watermark file in {@code slotDir} and + * maps it for the engine's lifetime. Returns {@code null} on any setup + * failure, leaving the caller to choose whether its durability contract + * permits operation without one. + *

    + * Wrong-sized files, including the legacy 16-byte non-CRC format, are + * reset. Trusting a legacy FSN would retain the torn-write ambiguity this + * format removes; resetting it causes conservative replay from the + * segment-derived seed instead. */ public static AckWatermark open(String slotDir) { + return open(FilesFacade.INSTANCE, slotDir); + } + + /** + * Facade-aware variant of {@link #open(String)}. Every filesystem call, + * including the lifetime mapping, goes through {@code filesFacade} so + * tests can observe or fault-inject the watermark mmap. + */ + public static AckWatermark open(FilesFacade filesFacade, String slotDir) { String filePath = slotDir + "/" + FILE_NAME; - // Decide by size: existing-and-correct -> openRW preserves the - // previous session's watermark (defeating which is the whole - // point of NOT calling openCleanRW unconditionally); missing or - // wrong-sized -> openCleanRW + allocate creates a fresh - // FILE_SIZE-byte file (zero magic, read() reports INVALID until - // the first write). - long existing = Files.exists(filePath) ? Files.length(filePath) : -1L; + long existing = filesFacade.exists(filePath) ? filesFacade.length(filePath) : -1L; int fd; if (existing == FILE_SIZE) { - fd = Files.openRW(filePath); + fd = filesFacade.openRW(filePath); } else { - fd = Files.openCleanRW(filePath); - if (fd >= 0 && !Files.allocate(fd, FILE_SIZE)) { - // FilesFacade.allocate contract on a false return: - // close the fd AND unlink the partial file. - Files.close(fd); - Files.remove(filePath); + fd = filesFacade.openCleanRW(filePath); + if (fd >= 0 && !filesFacade.allocate(fd, FILE_SIZE)) { + // FilesFacade.allocate contract on a false return: close the + // fd and unlink the partial file. + filesFacade.close(fd); + filesFacade.remove(filePath); fd = -1; } } if (fd < 0) { - LOG.warn("ack watermark {} could not be opened (rc={}); proceeding without it", - filePath, fd); + LOG.warn("ack watermark {} could not be opened (rc={})", filePath, fd); return null; } - long addr = Files.mmap(fd, FILE_SIZE, 0, Files.MAP_RW, MemoryTag.MMAP_DEFAULT); + long addr = filesFacade.mmap(fd, FILE_SIZE, 0, Files.MAP_RW, MemoryTag.MMAP_DEFAULT); if (addr == Files.FAILED_MMAP_ADDRESS) { - LOG.warn("ack watermark {} could not be mmapped; proceeding without it", filePath); - Files.close(fd); + LOG.warn("ack watermark {} could not be mmapped", filePath); + filesFacade.close(fd); return null; } - // Inspect the existing magic once at open time. If it's already - // set (cross-session reopen, e.g. recovery after a clean - // shutdown), the first write() can skip the magic store - // entirely and degenerate to a single 8-byte FSN put. - int magic = Unsafe.getUnsafe().getInt(addr + MAGIC_OFFSET); - return new AckWatermark(fd, addr, magic == FILE_MAGIC); + Record selected = selectRecord(addr); + return selected == null + ? new AckWatermark(filesFacade, fd, addr, 0L, INVALID) + : new AckWatermark(filesFacade, fd, addr, selected.generation, selected.fsn); + } + + /** + * Releases the native storage while deliberately leaving the logical + * closed flag clear. Test-only: recreates a stale racing writer without + * reflective access to descriptor and mapping internals. + */ + @TestOnly + public boolean releaseStorageButKeepWritableForTesting() { + return releaseStorage(); } /** - * Best-effort removal of a stale watermark file. Used by the - * engine startup path when no segments are recovered — a stale - * watermark file with no segments behind it is meaningless and - * would only confuse the next session's seed. + * Best-effort removal of a stale watermark file. Used by the engine + * startup path when no segments are recovered. */ public static void removeOrphan(String slotDir) { - Files.remove(slotDir + "/" + FILE_NAME); + removeOrphan(FilesFacade.INSTANCE, slotDir); + } + + static boolean removeOrphan(FilesFacade filesFacade, String slotDir) { + return filesFacade.remove(slotDir + "/" + FILE_NAME); } /** - * Single-load read of the current FSN. Returns {@link #INVALID} if - * the file has never been written (magic field is zero, i.e. the - * file was freshly created by {@link #open(String)} and no - * {@link #write(long)} has run yet against this slot). + * Returns the FSN from the greatest-generation valid record, or + * {@link #INVALID} when neither record validates. */ public long read() { if (closed) return INVALID; - int magic = Unsafe.getUnsafe().getInt(mmapAddress + MAGIC_OFFSET); - if (magic != FILE_MAGIC) { - // Either freshly created (all zeros) or some kind of corruption. - // Either way, fall back to the segment-derived seed. - return INVALID; + Record selected = selectRecord(mmapAddress); + if (selected == null) { + fsn = INVALID; + generation = 0L; + } else { + fsn = selected.fsn; + generation = selected.generation; } - return Unsafe.getUnsafe().getLong(mmapAddress + FSN_OFFSET); + return fsn; } /** - * Atomically updates the persisted FSN. Single 8-byte aligned - * store to the mapped region. The first write also stamps the - * magic so the next session's {@link #read()} can distinguish - * "valid watermark" from "freshly created file". + * Flushes the mapped bytes and then the backing fd. The caller must sync + * the slot directory after this succeeds so a newly-created watermark's + * directory entry is durable before segment deletion begins. + */ + public void sync() { + if (closed) { + throw new IllegalStateException("ack watermark is closed"); + } + if (filesFacade.msync(mmapAddress, FILE_SIZE, false) != 0) { + throw new IllegalStateException("could not msync ack watermark"); + } + if (filesFacade.fsync(fd) != 0) { + throw new IllegalStateException("could not fsync ack watermark"); + } + } + + /** + * Updates the inactive record and selects it in memory only after its CRC + * has been stored. The next {@link #sync()} makes the complete update + * durable before any covered segment is deleted. *

    - * Caller responsibility: monotonic ordering. The manager's tick - * loop should only call this when {@code fsn} has advanced past - * the last write. + * Caller responsibility: monotonic ordering. The manager's tick loop only + * calls this when {@code fsn} has advanced past the last write. */ public void write(long fsn) { if (closed) return; - // Steady-state hot path: a single 8-byte aligned putLong, no - // memory load of the magic. Order matters only on the very - // first write of a fresh file: FSN first so the next reader - // that observes the magic (set immediately below) also - // observes a valid FSN -- no fence needed because the same - // thread reads both in program order, and crash recovery - // resumes a fresh process that sees whatever disk-level state - // the kernel flushed. - Unsafe.getUnsafe().putLong(mmapAddress + FSN_OFFSET, fsn); - if (!magicWritten) { - Unsafe.getUnsafe().putInt(mmapAddress + MAGIC_OFFSET, FILE_MAGIC); - magicWritten = true; + long nextGeneration = generation + 1L; + long recordAddress = mmapAddress + (nextGeneration & 1L) * RECORD_SLOT_SIZE; + Unsafe.getUnsafe().setMemory(recordAddress, RECORD_SIZE, (byte) 0); + Unsafe.getUnsafe().putInt(recordAddress + MAGIC_OFFSET, FILE_MAGIC); + Unsafe.getUnsafe().putInt(recordAddress + 4, VERSION); + Unsafe.getUnsafe().putLong(recordAddress + 8, nextGeneration); + Unsafe.getUnsafe().putLong(recordAddress + FSN_OFFSET, fsn); + int crc = Crc32c.update(Crc32c.INIT, recordAddress, CRC_OFFSET); + Unsafe.getUnsafe().putInt(recordAddress + CRC_OFFSET, crc); + generation = nextGeneration; + this.fsn = fsn; + } + + private boolean releaseStorage() { + if (isStorageReleased) { + return false; + } + isStorageReleased = true; + if (mmapAddress != 0L && mmapAddress != Files.FAILED_MMAP_ADDRESS) { + filesFacade.munmap(mmapAddress, FILE_SIZE, MemoryTag.MMAP_DEFAULT); + } + if (fd >= 0) { + filesFacade.close(fd); + } + return true; + } + + private static Record readRecord(long address) { + if (Unsafe.getUnsafe().getInt(address + MAGIC_OFFSET) != FILE_MAGIC + || Unsafe.getUnsafe().getInt(address + 4) != VERSION) { + return null; + } + int expectedCrc = Unsafe.getUnsafe().getInt(address + CRC_OFFSET); + int actualCrc = Crc32c.update(Crc32c.INIT, address, CRC_OFFSET); + if (expectedCrc != actualCrc) { + return null; + } + long generation = Unsafe.getUnsafe().getLong(address + 8); + long fsn = Unsafe.getUnsafe().getLong(address + FSN_OFFSET); + if (generation <= 0 || fsn < -1L) { + return null; + } + return new Record(generation, fsn); + } + + private static Record selectRecord(long address) { + Record first = readRecord(address); + Record second = readRecord(address + RECORD_SLOT_SIZE); + if (first == null) { + return second; + } + if (second == null || first.generation > second.generation) { + return first; + } + return second; + } + + private static final class Record { + private final long fsn; + private final long generation; + + private Record(long generation, long fsn) { + this.fsn = fsn; + this.generation = generation; } } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java index d79080d4..c83f9f93 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java @@ -56,12 +56,15 @@ * *

    * On terminal failure (auth-rejection on reconnect, a cluster-wide durable-ack - * capability gap that exhausts its settle budget, recovery error), the drainer - * drops a {@link OrphanScanner#FAILED_SENTINEL_NAME} sentinel into the slot - * before exiting. Future scans skip the slot until an operator clears the - * sentinel — bounded automatic retry, then human-in-the-loop. A transient - * all-replica failover window is NOT terminal: it is retried indefinitely - * (Invariant B), never quarantined on a wall-clock budget or attempt cap. + * capability gap that exhausts its settle budget, corrupt or incomplete durable + * recovery state), the drainer drops a + * {@link OrphanScanner#FAILED_SENTINEL_NAME} sentinel into the slot before + * exiting. Future scans skip the slot until an operator clears the sentinel — + * bounded automatic retry, then human-in-the-loop. Operational setup failures + * leave no sentinel so a later orphan scan can retry. JVM/programming Errors + * also leave no sentinel and propagate after teardown. A transient all-replica + * failover window is NOT terminal: it is retried indefinitely (Invariant B), + * never quarantined on a wall-clock budget or attempt cap. */ public final class BackgroundDrainer implements Runnable { @@ -106,6 +109,7 @@ public final class BackgroundDrainer implements Runnable { private final long segmentSizeBytes; private final long sfMaxTotalBytes; private final String slotPath; + private final long syncIntervalNanos; /** Latest known {@code engine.ackedFsn()}; published for visibility. */ private volatile long ackedFsn = -1L; private volatile String lastErrorMessage; @@ -170,10 +174,36 @@ public BackgroundDrainer( long durableAckKeepaliveIntervalMillis, int maxHeadFrameRejections, long poisonMinEscalationWindowMillis + ) { + this(slotPath, segmentSizeBytes, sfMaxTotalBytes, 0L, clientFactory, + reconnectMaxDurationMillis, reconnectInitialBackoffMillis, + reconnectMaxBackoffMillis, requestDurableAck, + durableAckKeepaliveIntervalMillis, maxHeadFrameRejections, + poisonMinEscalationWindowMillis); + } + + /** + * Master constructor with the periodic SF checkpoint interval inherited + * from the sender that adopted the orphan slot. + */ + public BackgroundDrainer( + String slotPath, + long segmentSizeBytes, + long sfMaxTotalBytes, + long syncIntervalNanos, + CursorWebSocketSendLoop.ReconnectFactory clientFactory, + long reconnectMaxDurationMillis, + long reconnectInitialBackoffMillis, + long reconnectMaxBackoffMillis, + boolean requestDurableAck, + long durableAckKeepaliveIntervalMillis, + int maxHeadFrameRejections, + long poisonMinEscalationWindowMillis ) { this.slotPath = slotPath; this.segmentSizeBytes = segmentSizeBytes; this.sfMaxTotalBytes = sfMaxTotalBytes; + this.syncIntervalNanos = syncIntervalNanos; this.clientFactory = clientFactory; this.reconnectMaxDurationMillis = reconnectMaxDurationMillis; this.reconnectInitialBackoffMillis = reconnectInitialBackoffMillis; @@ -379,9 +409,9 @@ public WebSocketClient connectWithDurableAckRetry() { // retrying cannot clear it, and spinning here would pin // the slot .lock forever with no .failed sentinel and only // a throttled, possibly-null-message WARN as a trace. - // Rethrow: run()'s outer catch quarantines the slot - // (markFailed + FAILED) and its finally releases the lock - // -- quarantine-and-exit, exactly as genuine terminals do. + // Rethrow: run() records the failure without attempting + // allocation-heavy logging or a .failed write, its finally + // releases the lock, and then the Error remains visible. throw (Error) t; } // INVARIANT B: a transport failure -- the whole cluster is @@ -530,17 +560,60 @@ public void run() { // holds it, the engine constructor throws and we exit silently // (no .failed sentinel — contention is expected, not an error). try { - engine = new CursorSendEngine(slotPath, segmentSizeBytes, - sfMaxTotalBytes, CursorSendEngine.DEFAULT_APPEND_DEADLINE_NANOS); - } catch (IllegalStateException t) { - String msg = t.getMessage(); - if (msg != null && msg.contains("already in use")) { - LOG.info("orphan slot already locked, skipping: {} ({})", - slotPath, msg); - outcome = DrainOutcome.LOCKED_BY_OTHER; - return; + try { + engine = new CursorSendEngine(slotPath, segmentSizeBytes, + sfMaxTotalBytes, CursorSendEngine.DEFAULT_APPEND_DEADLINE_NANOS, + syncIntervalNanos); + } catch (SfSanitizedResidueException first) { + // First sight of proven-dead sealed residue: recovery + // durably zeroed it BEFORE failing closed, so the chain + // on disk is already healed and a .failed sentinel here + // would strand a replayable backlog no scan revisits + // (the sentinel gates isCandidateOrphan and nothing in + // production clears it). Retry once over the healed + // chain; the WARN keeps the incident surfaced. Any + // failure of the retry is genuine and takes the normal + // classification below -- including a repeat of this + // type, which the SfRecoveryException arm then treats + // as the terminal quarantine a non-sticking heal is. + LOG.warn("drainer slot {}: sealed SF residue sanitized during recovery ({}); " + + "retrying engine construction over the healed chain", + slotPath, first.getMessage()); + engine = new CursorSendEngine(slotPath, segmentSizeBytes, + sfMaxTotalBytes, CursorSendEngine.DEFAULT_APPEND_DEADLINE_NANOS, + syncIntervalNanos); } + } catch (SlotLockContentionException t) { + LOG.info("orphan slot already locked, skipping: {} ({})", + slotPath, t.getMessage()); + outcome = DrainOutcome.LOCKED_BY_OTHER; + return; + } catch (SfRecoveryException | MmapSegmentCorruptionException t) { + // The durable chain itself is proven corrupt or incomplete. + // Repeated scans cannot repair it, so preserve the terminal + // quarantine path in the outer catch below. throw t; + } catch (Exception t) { + // Every other pre-publication construction exception is + // retryable: setup I/O, resource pressure, unexpected setup + // faults, and future operational failures do not prove durable + // data corruption. The constructor has closed partial + // resources; SlotLock retains and retries any unconfirmed + // unlock. Leave no .failed sentinel for the next orphan scan. + // Error deliberately escapes to the outer Error path: it is + // observable after teardown but cannot quarantine intact data. + // This path retries on every orphan scan, so the log line is + // the ONLY diagnostic a deterministic bug (e.g. an unexpected + // NPE, whose getMessage() is null) ever produces: attach the + // throwable for the stack trace and carry class+message into + // the telemetry surface, mirroring the outer setup-failure + // catch below. + String msg = t.toString(); + LOG.warn("drainer setup temporarily unavailable for slot {}: {}", + slotPath, msg, t); + lastErrorMessage = msg; + outcome = DrainOutcome.FAILED; + return; } long target = engine.publishedFsn(); if (engine.ackedFsn() >= target) { @@ -590,6 +663,16 @@ public void run() { try { loop.checkError(); } catch (Throwable t) { + // The I/O loop latches a JVM/programming Error inside a + // LineSenderException so checkError() can cross the + // thread boundary. Preserve the original Error contract: + // no wire quarantine, tear down, then propagate it. + if (t instanceof Error) { + throw (Error) t; + } + if (t.getCause() instanceof Error) { + throw (Error) t.getCause(); + } if (loop.capabilityGapTerminal() != null) { // Capability gap mid-drain: recycle the wire, NOT // the slot. connectWithDurableAckRetry() owns the @@ -645,6 +728,18 @@ public void run() { // outer condition, which is false for the same reason. } outcome = DrainOutcome.STOPPED; + } catch (Error t) { + // Resource pressure and JVM/programming failures prove neither + // durable corruption nor a terminal server response. Log the Error + // best-effort, but never let a secondary logging failure (especially + // under OOME) mask the original Error or prevent teardown. + try { + LOG.error("drainer failed with Error for slot {}", slotPath, t); + } catch (Throwable ignored) { + } + lastErrorMessage = t.getMessage(); + outcome = DrainOutcome.FAILED; + throw t; } catch (Throwable t) { String msg = t.getMessage(); if (slotPath != null) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 64ca75d0..80f54486 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -26,10 +26,17 @@ import io.questdb.client.std.Compat; import io.questdb.client.std.Files; +import io.questdb.client.std.FilesFacade; import io.questdb.client.std.ObjList; import io.questdb.client.std.QuietCloseable; +import org.jetbrains.annotations.TestOnly; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.LockSupport; +import java.util.function.LongConsumer; /** * Facade that bundles a {@link SegmentRing} with a {@link SegmentManager} and @@ -63,6 +70,18 @@ public final class CursorSendEngine implements QuietCloseable { public static final long DEFAULT_APPEND_DEADLINE_NANOS = 30_000_000_000L; private static final org.slf4j.Logger LOG = org.slf4j.LoggerFactory.getLogger(CursorSendEngine.class); + private static final ThreadFactory DEFAULT_FLOCK_RELEASE_RETRY_THREAD_FACTORY = + runnable -> new Thread(runnable, "qdb-sf-flock-release-retry"); + private static final long FLOCK_RELEASE_RETRY_BASE_PARK_NANOS = 100_000_000L; // 100 ms + private static final Object FLOCK_RELEASE_RETRY_LOCK = new Object(); + private static final long FLOCK_RELEASE_RETRY_MAX_PARK_NANOS = 5_000_000_000L; // 5 s + private static final ArrayDeque FLOCK_RELEASE_RETRY_QUEUE = new ArrayDeque<>(); + private static volatile Runnable afterFlockReleaseRetryFailureHook; + private static volatile Runnable beforeDeferredCloseCreationHook; + private static volatile LongConsumer flockReleaseRetryParkOverride; + private static Thread flockReleaseRetryThread; + private static volatile ThreadFactory flockReleaseRetryThreadFactory = + DEFAULT_FLOCK_RELEASE_RETRY_THREAD_FACTORY; private final long appendDeadlineNanos; // Number of times appendBlocking observed BACKPRESSURE_NO_SPARE on its first // ring.appendOrFsn attempt. One increment per blocking-call that had to wait @@ -70,6 +89,11 @@ public final class CursorSendEngine implements QuietCloseable { // writer; volatile because the user may sample it from any thread. private final java.util.concurrent.atomic.AtomicLong backpressureStallCount = new java.util.concurrent.atomic.AtomicLong(); + // Constructed before an owned manager acquires its native path scratch, so + // callback allocation failure cannot orphan manager resources. A timed-out + // close can then hand it to either manager path without allocating. + private final Runnable deferredClose; + private final FilesFacade filesFacade; private final SegmentManager manager; // We own the manager iff the user constructed us with no manager — in that // case close() also stops the manager. When the manager is shared across @@ -78,6 +102,7 @@ public final class CursorSendEngine implements QuietCloseable { private final SegmentRing ring; private final long segmentSizeBytes; private final String sfDir; + private final long syncIntervalNanos; // Held for the engine's lifetime in disk mode. {@code null} in memory // mode (no slot, no lock). Released by {@link #close()}; the kernel // also drops it on hard process exit. @@ -104,16 +129,67 @@ public final class CursorSendEngine implements QuietCloseable { // range with a cumulative self-acknowledge once everything below is // server-acked (CursorWebSocketSendLoop.tryRetireOrphanTail). private long recoveredOrphanTipFsn = -1L; - // Engine-owned mmap'd watermark file. {@code null} in memory mode and - // in disk mode if open() failed (we proceed without it; recovery just - // falls back to lowestBase - 1). Lifetime tied to the engine: opened - // in the constructor, closed by {@link #close()}. The segment manager - // writes through this on every tick where ackedFsn has advanced. + // Engine-owned mmap'd watermark file. {@code null} only in memory mode; + // disk mode fails construction unless the watermark is usable because + // segment-derived recovery cannot distinguish acknowledged residue from + // replayable frames. Lifetime tied to the engine: opened in the + // constructor, closed by {@link #close()}. The segment manager writes + // through this on every tick where ackedFsn has advanced. private final AckWatermark watermark; // close() is publicly callable from any thread (Sender.close from a user // thread, JVM shutdown hooks, test cleanup). volatile + synchronized // close() makes the check-and-set atomic and gives readers a fence. private volatile boolean closed; + // True once close() has run its full cleanup sequence INCLUDING a + // CONFIRMED slot-flock release — finishClose() publishes this strictly + // after SlotLock.release() reports success, never before. Pool threads + // treat the flip as "the slot dir is reusable" and free the slot index + // the moment they observe it (QwpWebSocketSender.isSlotLockReleased -> + // SenderPool.reprobeRetiredSlots), so publishing before the release + // would let a replacement engine's SlotLock.acquire collide with the + // still-open fd. Stays false when a close attempt could not confirm + // manager-worker quiescence (or the flock release itself failed) and had + // to leak the ring/watermark/slot lock — in that case a later close() + // call retries the cleanup (the worker may have exited by then). + // volatile: latched by finishClose(), but read lock-free by + // isCloseCompleted() from pool threads re-probing a retired slot (see the + // getter for why it must not synchronize). + private volatile boolean closeCompleted; + // Invoked after closeCompleted publishes a confirmed flock release. Used + // by an owning sender pool to wake capacity-starved borrowers immediately. + private volatile Runnable slotLockReleaseListener; + // Test-only hook run by finishClose() between the terminal cleanup and + // the flock release. Lets a test park the releasing thread inside the + // cleanup/release window and assert that closeCompleted stays false — + // i.e. that completion is never observable while the flock is still + // held. volatile: finishClose may run on the manager worker's exit + // thread while the hook is installed from a test thread. + private volatile Runnable beforeFlockReleaseHook; + // Ensures this engine has at most one entry in the shared flock-release + // retry driver. The error path only: ordinary closes never enqueue work. + private final AtomicBoolean flockReleaseRetryStarted = new AtomicBoolean(); + // Published before deferredClose is registered. The manager lock provides + // the callback handoff fence; volatile also covers a direct test/retry read. + private volatile boolean fullyDrainedForDeferredClose; + // Exactly-once claim on the terminal cleanup (finishClose). Contended by + // close() and a worker-exit handoff (completeDeferredClose); whoever wins + // the CAS runs the cleanup, the loser never touches ring/watermark/flock. + // Deliberately NOT the engine monitor: a retried close() holds the + // monitor while joining the manager worker, and the worker cannot die + // until its exit-path cleanup finishes — monitor-based exclusion would + // stall that close() for its full join budget (test-visible livelock). + // With the CAS the worker's cleanup never blocks, so the join returns as + // soon as the pass ends. + private final AtomicBoolean terminalCleanupClaimed = new AtomicBoolean(); + // True only after a quiescent close reached the final watermark barrier + // and that barrier failed. It permits the shared retry driver to claim + // terminal cleanup; false while cleanup is merely waiting for manager + // quiescence, where an independent retry would race the live worker. + private volatile boolean terminalCleanupRetryReady; + // Published only after ring/watermark/unlink cleanup is finished. A close + // that loses terminalCleanupClaimed may retry the flock only after this + // becomes true, otherwise it could expose the slot while cleanup is live. + private volatile boolean terminalResourcesCleaned; // Producer-thread-only: timestamp of the last "we're backpressured" log // line, used to throttle. Plain long is fine. private long lastBackpressureLogNs; @@ -125,7 +201,7 @@ public final class CursorSendEngine implements QuietCloseable { */ public CursorSendEngine(String sfDir, long segmentSizeBytes) { this(sfDir, segmentSizeBytes, SegmentManager.UNLIMITED_TOTAL_BYTES, - DEFAULT_APPEND_DEADLINE_NANOS); + DEFAULT_APPEND_DEADLINE_NANOS, 0L); } /** @@ -137,9 +213,18 @@ public CursorSendEngine(String sfDir, long segmentSizeBytes) { */ public CursorSendEngine(String sfDir, long segmentSizeBytes, long maxTotalBytes, long appendDeadlineNanos) { - this(sfDir, segmentSizeBytes, - new SegmentManager(segmentSizeBytes, SegmentManager.DEFAULT_POLL_NANOS, maxTotalBytes), - true, appendDeadlineNanos); + this(sfDir, segmentSizeBytes, maxTotalBytes, appendDeadlineNanos, 0L); + } + + /** + * Creates an engine with an optional periodic data-checkpoint interval. + * A positive interval requires disk-backed store-and-forward mode. + */ + public CursorSendEngine(String sfDir, long segmentSizeBytes, + long maxTotalBytes, long appendDeadlineNanos, + long syncIntervalNanos) { + this(sfDir, segmentSizeBytes, null, true, maxTotalBytes, + appendDeadlineNanos, syncIntervalNanos); } /** @@ -148,17 +233,53 @@ public CursorSendEngine(String sfDir, long segmentSizeBytes, * ownership of the manager. Uses the default append deadline. */ public CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager manager) { - this(sfDir, segmentSizeBytes, manager, false, DEFAULT_APPEND_DEADLINE_NANOS); + this(sfDir, segmentSizeBytes, manager, false, DEFAULT_APPEND_DEADLINE_NANOS, 0L); + } + + public CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager manager, + long syncIntervalNanos) { + this(sfDir, segmentSizeBytes, manager, false, + DEFAULT_APPEND_DEADLINE_NANOS, syncIntervalNanos); } private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager manager, boolean ownsManager, long appendDeadlineNanos) { + this(sfDir, segmentSizeBytes, manager, ownsManager, appendDeadlineNanos, 0L); + } + + private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager manager, + boolean ownsManager, long appendDeadlineNanos, + long syncIntervalNanos) { + this(sfDir, segmentSizeBytes, manager, ownsManager, + SegmentManager.UNLIMITED_TOTAL_BYTES, appendDeadlineNanos, syncIntervalNanos); + } + + private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager manager, + boolean ownsManager, long maxTotalBytes, long appendDeadlineNanos, + long syncIntervalNanos) { + // Allocate the bound callback before constructing an owned manager. + // Field initializers have completed, but no engine-owned native/disk + // resource exists yet. If callback allocation throws, construction + // stops without a manager whose native path scratch could be orphaned. + this.deferredClose = createDeferredClose(); + // sfDir == null → memory-only mode (non-SF async ingest). Same // cursor architecture, no disk involvement; segments // live in malloc'd native memory. // sfDir != null → store-and-forward mode. Segments are mmap'd files // under sfDir, recoverable across sender restarts. boolean memoryMode = sfDir == null; + // Validate before an owned manager allocates its native path scratch. + if (syncIntervalNanos < 0L) { + throw new IllegalArgumentException("syncIntervalNanos must not be negative"); + } + if (memoryMode && syncIntervalNanos > 0L) { + throw new IllegalArgumentException("periodic sync requires disk store-and-forward mode"); + } + if (ownsManager && manager == null) { + manager = new SegmentManager( + segmentSizeBytes, SegmentManager.DEFAULT_POLL_NANOS, maxTotalBytes); + } SlotLock acquiredLock = null; if (!memoryMode) { try { @@ -170,13 +291,11 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man // recovery and create overlapping FSN ranges. SlotLock.acquire // also creates the slot dir if it doesn't exist yet — no // separate mkdir step needed here. - acquiredLock = SlotLock.acquire(sfDir); + acquiredLock = SlotLock.acquire(sfDir, syncIntervalNanos > 0L); } catch (Throwable t) { - // The delegating constructors evaluate `new SegmentManager(...)` - // BEFORE this body runs, so on a pre-try throw (e.g. slot lock - // collision) an owned manager is already alive and would leak - // its native path-scratch sink -- 256 bytes per failed - // construction attempt. Close it before propagating. + // Callback creation and owned-manager construction have already + // completed. A slot-lock failure must close the owned manager's + // native path scratch before propagating. if (ownsManager) { try { manager.close(); @@ -190,8 +309,10 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man this.sfDir = sfDir; this.segmentSizeBytes = segmentSizeBytes; this.manager = manager; + this.filesFacade = manager.filesFacade(); this.ownsManager = ownsManager; this.appendDeadlineNanos = appendDeadlineNanos; + this.syncIntervalNanos = syncIntervalNanos; // Track the ring locally until every step succeeds — only commit it // to this.ring at the very end. If anything between ring allocation @@ -204,9 +325,11 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man // session before deciding to start fresh. Without this the engine // would create a new sf-initial.sfa at baseSeq=0, overlapping FSNs // already on disk and corrupting ACK translation, trim, and replay. - SegmentRing recovered = memoryMode ? null - : SegmentRing.openExisting(sfDir, segmentSizeBytes); - this.wasRecoveredFromDisk = recovered != null; + SegmentRing.Recovery recovery = memoryMode ? null + : SegmentRing.recover(filesFacade, sfDir, segmentSizeBytes); + SegmentRing recovered = recovery == null ? null : recovery.ring(); + this.wasRecoveredFromDisk = recovery != null + && recovery.status() == SegmentRing.RecoveryStatus.RECOVERED; if (recovered != null) { ringInProgress = recovered; // Seed ackedFsn to one below the lowest segment's baseSeq. @@ -253,14 +376,18 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man // - trim ran before persist: segments are gone (so // lowestBase is higher than watermark), watermark // is stale; max picks lowestBase - 1. - // open() returns null on any setup failure so a missing - // mmap doesn't take down the engine -- we just fall - // back to the bare lowestBase - 1 seed. - watermarkInProgress = AckWatermark.open(sfDir); + // Segment-derived state cannot tell whether frames in the + // lowest surviving segment were acknowledged. Starting + // without the watermark would therefore expose acknowledged + // residue for replay, so disk recovery must fail closed when + // the file cannot be opened or mapped. + watermarkInProgress = AckWatermark.open(filesFacade, sfDir); + if (watermarkInProgress == null) { + throw new SfOperationalException( + "could not open required ack watermark for SF slot " + sfDir); + } long baseSeed = lowestBase - 1; - long watermarkFsn = watermarkInProgress != null - ? watermarkInProgress.read() - : AckWatermark.INVALID; + long watermarkFsn = watermarkInProgress.read(); // Reject watermarks past publishedFsn: a correctly // operating prior session cannot have produced one, so // a value above the on-disk frame ceiling is corruption @@ -307,32 +434,75 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man // so the new session's first read() correctly reports // INVALID (magic=0 on a freshly zero-filled file). if (!memoryMode) { - AckWatermark.removeOrphan(sfDir); - watermarkInProgress = AckWatermark.open(sfDir); + AckWatermark.removeOrphan(filesFacade, sfDir); + watermarkInProgress = AckWatermark.open(filesFacade, sfDir); + if (watermarkInProgress == null) { + throw new SfOperationalException( + "could not open required ack watermark for SF slot " + sfDir); + } } MmapSegment initial; String initialPath = null; + SfManifest initialManifest = null; if (memoryMode) { initial = MmapSegment.createInMemory(0L, segmentSizeBytes); } else { + // Created WITHOUT the manifest-required flag: the flag is + // a durable promise that sf-manifest.bin exists, so it + // must only be stamped after the manifest itself is on + // disk. Stamping it at create time would leave a crash + // window (initial durable, manifest not yet created) + // whose recovery hard-fails with "segment requires + // missing manifest" even though nothing was lost. initialPath = sfDir + "/sf-initial.sfa"; - initial = MmapSegment.create(initialPath, 0L, segmentSizeBytes); + initial = MmapSegment.create(filesFacade, initialPath, 0L, segmentSizeBytes); } try { - ringInProgress = new SegmentRing(initial, segmentSizeBytes); + if (!memoryMode) { + // Ordering is load-bearing for crash recovery: + // 1. initial segment durable (header + dirent), + // 2. manifest created (its own create fsyncs), + // 3. flag stamped on the segment. + // A crash after (1) recovers via the legacy + // (manifest-less) path; after (2) via the manifest + // path with an unflagged-but-valid empty active; + // after (3) it is the steady state. Every window + // self-heals without operator action. + initial.syncHeader(); + if (filesFacade.fsyncDir(sfDir) != 0) { + throw new MmapSegmentException( + "could not sync fresh SF segment directory " + sfDir); + } + initialManifest = SfManifest.create(filesFacade, sfDir, 0L, 0L); + initial.markManifestRequired(); + } + ringInProgress = new SegmentRing(initial, segmentSizeBytes, initialManifest); + initialManifest = null; } catch (Throwable t) { initial.close(); + if (initialManifest != null) { + initialManifest.close(); + SfManifest.removeFile(filesFacade, sfDir); + } if (initialPath != null) { - Files.remove(initialPath); + filesFacade.remove(initialPath); } throw t; } } + if (syncIntervalNanos > 0L) { + ringInProgress.enablePeriodicSync(); + if (recovered != null) { + // Establish a durable baseline before exposing data that a + // previous MEMORY-mode process may have left in page cache. + ringInProgress.syncAllLiveSegments(); + } + } if (ownsManager) { manager.start(); } - manager.register(ringInProgress, sfDir, watermarkInProgress); + manager.register(ringInProgress, sfDir, watermarkInProgress, syncIntervalNanos); // All construction succeeded — commit the ring and // watermark references. this.ring = ringInProgress; @@ -399,6 +569,10 @@ public MmapSegment activeSegment() { return ring.getActive(); } + MmapSegment pinActiveSegment() { + return ring.pinActiveSegment(); + } + /** * Append the payload, blocking up to {@link #appendDeadlineNanos} when * the cursor ring is at its memory/disk cap and waiting for ACK-driven @@ -431,11 +605,13 @@ public long appendBlocking(long payloadAddr, int payloadLen) { if (now >= deadlineNs) { throw new io.questdb.client.cutlass.line.LineSenderException( "cursor ring backpressured for ").put(appendDeadlineNanos / 1_000_000L) - .put(" ms — wire path is not draining (server slow / disconnected, or sf_max_total_bytes too small)"); + .put(" ms - wire path is not draining (server slow / disconnected, " + + "periodic disk sync is slow, or sf_max_total_bytes is too small)"); } if (now - lastBackpressureLogNs >= BACKPRESSURE_LOG_THROTTLE_NANOS) { lastBackpressureLogNs = now; - LOG.warn("cursor producer backpressured ({} stalls so far); waiting for I/O drain — will throw after {} ms", + LOG.warn("cursor producer backpressured ({} stalls so far); waiting for I/O or periodic disk sync; " + + "will throw after {} ms", backpressureStallCount.get(), appendDeadlineNanos / 1_000_000L); } LockSupport.parkNanos(50_000L); // 50 µs @@ -478,51 +654,239 @@ public long appendOrFsn(long payloadAddr, int payloadLen, long spinDeadlineNanos return SegmentRing.BACKPRESSURE_NO_SPARE; } + public void checkDurability() { + ring.checkDurability(); + } + @Override public synchronized void close() { - if (closed) return; + if (closed && closeCompleted) return; closed = true; // Capture drain state BEFORE closing the ring — once the ring is // closed, its accessors aren't safe to read. The active segment is - // never trimmed by drainTrimmable (only sealed segments are), so + // never trim-eligible (only sealed segments are), so // when everything published has been acked we have to unlink the // residual .sfa files here. Without this, the next sender (or a // drainer adopting this slot) would replay already-acked data // against potentially-fresh server state — duplicate writes when // the server has no dedup state for those messageSequences. // Memory mode has no files to unlink. - // The whole close sequence runs under try/finally so the slot lock - // is ALWAYS released, even if manager/ring close or unlink throws — - // otherwise a kernel-held flock outlives the engine and the next - // sender for the same slot collides on a lock the dead engine - // never released. + // + // Cleanup is gated on worker quiescence: releasing the ring, + // watermark, segment files or the slot lock while the manager worker + // is still mid service pass would let a replacement engine acquire + // the slot and have its files unlinked by the stale worker — + // store-and-forward data loss after restart. When the bounded worker + // join times out on an owned manager, cleanup OWNERSHIP TRANSFERS to + // the worker's exit path (deferUntilWorkerExit): the worker is + // provably the last thread able to touch the slot directory, so it + // releases everything the moment its in-flight pass finishes instead + // of leaking the slot until process exit. + // + // "Fully drained" includes BOTH the obvious case (every published + // FSN has been acked) AND the never-published case (publishedFsn + // < 0). The latter matters because a drainer adopting an empty + // orphan slot — segments filtered as empty by recovery, engine + // recreates a fresh sf-initial.sfa — would otherwise leave that + // fresh empty file behind, the next scanner finds it, adopts the + // slot again, and the cycle repeats forever (M6). + // Own try/catch so sabotaged/broken ring state cannot skip the + // quiescence barrier below or the slot-lock release in finishClose. + boolean drained = false; try { - // "Fully drained" includes BOTH the obvious case (every published - // FSN has been acked) AND the never-published case (publishedFsn - // < 0). The latter matters because a drainer adopting an empty - // orphan slot — segments filtered as empty by recovery, engine - // recreates a fresh sf-initial.sfa — would otherwise leave that - // fresh empty file behind, the next scanner finds it, adopts the - // slot again, and the cycle repeats forever (M6). - boolean fullyDrained = sfDir != null + drained = sfDir != null && (ring.publishedFsn() < 0 || ring.ackedFsn() >= ring.publishedFsn()); - // Each cleanup step in its own try/catch so a single failure - // doesn't strand later cleanups — mirrors the constructor's - // catch block. Without this, a throw from manager.deregister - // or manager.close() would leave the ring mmap'd and any - // residual .sfa files on disk, where the next sender can - // adopt them and replay already-acked data. + } catch (Throwable ignored) { + } + final boolean fullyDrained = drained; + fullyDrainedForDeferredClose = fullyDrained; + // Each cleanup step in its own try/catch so a single failure + // doesn't strand later cleanups — mirrors the constructor's + // catch block. Without this, a throw from manager.deregister + // or manager.close() would leave the ring mmap'd and any + // residual .sfa files on disk, where the next sender can + // adopt them and replay already-acked data. + try { + manager.deregister(ring); + } catch (Throwable ignored) { + } + // Quiescence barrier. deregister alone only removes the entry + // from the registry — the worker may still be mid service pass + // for this ring (creating a spare file, trimming, unlinking). + boolean workerQuiescent = false; + if (ownsManager) { + // Stopping and reaping a private manager is a stronger barrier + // than waiting for this ring alone. Do it directly so a stuck + // worker consumes at most one workerJoinTimeoutMillis budget, + // rather than one here and a second one in manager.close(). try { - manager.deregister(ring); + manager.close(); } catch (Throwable ignored) { } - if (ownsManager) { - try { - manager.close(); - } catch (Throwable ignored) { + try { + workerQuiescent = manager.isWorkerReaped(); + } catch (Throwable ignored) { + } + } else { + // A shared manager must keep serving its other rings, so wait + // only for the deregistered ring's current pass to finish. + try { + workerQuiescent = manager.awaitRingQuiescence(ring); + } catch (Throwable ignored) { + } + } + if (!workerQuiescent && ownsManager) { + // Ownership handoff: manager.close() already stopped the worker + // loop (running=false), so the worker exits as soon as its + // in-flight pass finishes — it is merely slow, or wedged in a + // syscall. Either way it is the last thread able to touch the + // slot, so hand it the terminal cleanup instead of leaking the + // slot until process exit. completeDeferredClose and a retried + // close() race through the terminalCleanupClaimed CAS, so the + // cleanup runs exactly once and the worker never blocks on the + // engine monitor a retried close() holds while joining it. + boolean handedOff = false; + boolean registrationFailed = false; + try { + handedOff = manager.deferOwnedEngineCloseUntilWorkerExit(deferredClose); + } catch (Throwable ignored) { + // Unexpected monitor/VM failure carries no worker-liveness + // information. Ordinary handoff cannot allocate: both the + // callback and the manager's single slot were preallocated. + registrationFailed = true; + } + if (handedOff) { + LOG.error("SF manager worker did not quiesce during engine close; ring, watermark " + + "and slot-lock release are handed to the worker's exit path and run the " + + "moment its in-flight service pass finishes. The slot stays locked (and " + + "isCloseCompleted() false) until then, so no replacement engine can race " + + "the stale worker on slot {}", sfDir == null ? "" : sfDir); + return; + } + if (registrationFailed) { + // The handoff never registered and the worker was never + // observed — it must be presumed live and mid service pass. + // Retain every worker-reachable resource (ring, watermark, + // segment files, slot flock) and leave terminalCleanupClaimed + // unclaimed and closeCompleted false, exactly like the + // shared-manager leak branch: manager.close() above already + // stopped the worker loop, so a retried close() converges via + // isWorkerReaped() once the in-flight pass ends. The kernel + // releases the slot flock on process exit regardless. + LOG.error("SF worker-exit handoff registration failed during engine close; " + + "leaking the ring, watermark and slot lock so a possibly-live " + + "worker cannot corrupt a future engine on slot {}. close() may be " + + "invoked again to retry cleanup once the worker has exited.", + sfDir == null ? "" : sfDir); + return; + } + // Handoff rejected: the worker loop exited between the failed + // bounded join and the registration attempt (both sides share the + // manager's lock, so this observation is exact). A worker past + // its loop can never touch slot paths again — inline cleanup is + // as safe as a reaped worker. + workerQuiescent = true; + } + if (!workerQuiescent) { + // A shared manager keeps serving sibling rings, so it cannot use + // the whole-worker exit handoff above. Transfer cleanup to this + // ring's current pass instead. Registration and pass completion + // share the manager lock: true means the worker owns cleanup; + // false means the pass already finished and inline cleanup is safe. + boolean handedOff = false; + boolean registrationFailed = false; + try { + handedOff = manager.deferUntilRingQuiescent(ring, deferredClose); + } catch (Throwable ignored) { + registrationFailed = true; + } + if (handedOff) { + LOG.error("SF manager worker did not quiesce during engine close; ring, watermark " + + "and slot-lock release are handed to the current ring pass and run the " + + "moment that pass finishes. The slot stays locked (and " + + "isCloseCompleted() false) until then, so no replacement engine can " + + "race the stale worker on slot {}", + sfDir == null ? "" : sfDir); + return; + } + if (registrationFailed) { + LOG.error("SF ring-pass cleanup handoff registration failed during engine close; " + + "leaking the ring, watermark and slot lock so a possibly-live worker " + + "cannot corrupt a future engine on slot {}. The kernel releases the " + + "flock on process exit.", sfDir == null ? "" : sfDir); + return; + } + workerQuiescent = true; + } + if (!terminalCleanupClaimed.compareAndSet(false, true)) { + // A worker-exit handoff or earlier close owns the one-time + // ring/watermark cleanup. Once that work is published complete, + // this close may still retry an unconfirmed flock release. + retryFlockReleaseIfReady(); + return; + } + finishClose(fullyDrained); + } + + /** + * Terminal cleanup: closes the ring and watermark, unlinks drained + * segment files, releases the slot flock, and — only once the release + * is confirmed — latches {@link #closeCompleted}. Publish order + * is load-bearing: pools free the slot index the instant they observe + * {@code closeCompleted} (via {@code isSlotLockReleased()}), so it must + * never be visible while the flock is still held, or a replacement + * engine races the release and fails acquisition on a live slot. + * The caller must hold the engine monitor and + * must have established that the manager worker can no longer touch the + * slot directory (reaped, provably exited, or running this on its own + * exit path) AND have won the {@link #terminalCleanupClaimed} CAS — the + * claim token, not the engine monitor, is the exclusion between a + * worker-exit handoff and a retried close(), so ring/watermark/flock are + * never double-released and the worker's cleanup can never deadlock + * against a close() that holds the monitor while joining the worker. + */ + private void finishClose(boolean fullyDrained) { + if (!fullyDrained && syncIntervalNanos > 0L) { + try { + ring.syncAllLiveSegments(); + } catch (RuntimeException | Error e) { + terminalCleanupRetryReady = true; + terminalCleanupClaimed.set(false); + startFlockReleaseRetry(); + throw e; + } + } + // On a fully-drained close, persist the final acked FSN through the + // still-mapped watermark BEFORE closing the ring/watermark and BEFORE + // unlinking any segment file. The manager persists the watermark only + // on its own tick, so it may lag the final ack. If any part of this + // covering barrier fails, retain the ring, watermark and slot flock: + // publishing the slot would let a successor recover acknowledged + // residue from a stale durable watermark. The shared retry driver owns + // convergence after the one-shot public close reports the failure. + if (fullyDrained && watermark != null) { + try { + long finalAckedFsn = ring.ackedFsn(); + if (finalAckedFsn >= 0) { + watermark.write(finalAckedFsn); + watermark.sync(); + if (filesFacade.fsyncDir(sfDir) != 0) { + throw new IllegalStateException( + "could not fsync SF slot directory before segment cleanup"); + } } + } catch (RuntimeException | Error e) { + terminalCleanupRetryReady = true; + terminalCleanupClaimed.set(false); + startFlockReleaseRetry(); + throw e; } + } + terminalCleanupRetryReady = false; + + try { + RuntimeException durabilityFailure = null; try { ring.close(); } catch (Throwable ignored) { @@ -541,24 +905,363 @@ public synchronized void close() { } catch (Throwable ignored) { } } - if (fullyDrained) { + if (fullyDrained && watermark != null) { + boolean segmentsRemoved = false; try { - unlinkAllSegmentFiles(sfDir); + segmentsRemoved = unlinkAllSegmentFiles(filesFacade, sfDir); } catch (Throwable ignored) { } + // Remove the watermark ONLY once every segment unlink is + // durable. Until the directory fsync succeeds, stable storage + // may still contain acknowledged segments and therefore still + // needs the durable watermark. + if (segmentsRemoved) { + if (filesFacade.fsyncDir(sfDir) != 0) { + durabilityFailure = new IllegalStateException( + "could not fsync SF slot directory after segment cleanup"); + } else { + AckWatermark.removeOrphan(filesFacade, sfDir); + } + } else { + LOG.warn("close-time segment cleanup incomplete on slot {}; retaining the ack " + + "watermark so residual acknowledged segments stay covered -- the next " + + "engine on this slot recovers them as fully acked and retries the " + + "unlink on its own close", sfDir); + } + } + if (durabilityFailure != null) { + throw durabilityFailure; + } + } finally { + // The final watermark covering barrier has succeeded, so terminal + // resources can be released even if later best-effort cleanup or + // its post-unlink directory sync failed. The durable watermark + // still covers any segment a host crash restores. + // + // ORDER MATTERS: explicitly release the flock, verify it, and + // only then publish closeCompleted. Pools read isCloseCompleted() + // as "the slot dir is reusable" and free the slot index the moment + // it flips. SlotLock closes the fd once after confirmed unlock, + // but that close result cannot safely control publication because + // POSIX may consume the fd even when close reports failure. + Runnable hook = beforeFlockReleaseHook; + if (hook != null) { try { - AckWatermark.removeOrphan(sfDir); + hook.run(); } catch (Throwable ignored) { + // test-only; must never block the release } } - } finally { - if (slotLock != null) { + terminalResourcesCleaned = true; + retryFlockReleaseIfReady(); + } + } + + @TestOnly + public SegmentManager getManagerForTesting() { + return manager; + } + + @TestOnly + public SegmentRing getRingForTesting() { + return ring; + } + + @TestOnly + public SlotLock getSlotLockForTesting() { + return slotLock; + } + + @TestOnly + public long getSyncIntervalNanosForTesting() { + return syncIntervalNanos; + } + + /** + * Installs a hook invoked after each failed shared-driver flock release. + * Test-only: makes persistent retry progress observable without sleeps. + */ + @TestOnly + public static void setAfterFlockReleaseRetryFailureHook(Runnable hook) { + afterFlockReleaseRetryFailureHook = hook; + } + + /** + * Installs a constructor fault hook immediately before the bound deferred + * close callback is created. Test-only: proves callback allocation failure + * occurs before an owned manager acquires native resources. + */ + @TestOnly + public static void setBeforeDeferredCloseCreationHook(Runnable hook) { + beforeDeferredCloseCreationHook = hook; + } + + /** + * Replaces the shared retry driver's inter-round park with a callback + * receiving the park duration the driver would have used. Test-only: + * makes the retry cadence inspectable and rounds coordinatable without + * wall-clock waits. + */ + @TestOnly + public static void setFlockReleaseRetryParkOverride(LongConsumer override) { + flockReleaseRetryParkOverride = override; + } + + /** + * Overrides creation of the single shared flock-release retry thread. + * Test-only: makes thread creation/start failure and persistent retry + * scalability deterministic without relying on scheduler timing. + */ + @TestOnly + public static void setFlockReleaseRetryThreadFactory(ThreadFactory factory) { + synchronized (FLOCK_RELEASE_RETRY_LOCK) { + if (flockReleaseRetryThread != null || !FLOCK_RELEASE_RETRY_QUEUE.isEmpty()) { + throw new IllegalStateException("flock-release retry driver is active"); + } + flockReleaseRetryThreadFactory = factory == null + ? DEFAULT_FLOCK_RELEASE_RETRY_THREAD_FACTORY + : factory; + } + } + + /** + * Installs a hook that {@link #finishClose} runs between the terminal + * cleanup and the slot-flock release. Test-only: makes the otherwise + * microsecond-wide cleanup/release window deterministic so tests can + * assert {@link #isCloseCompleted()} stays false until the release is + * confirmed. + */ + @TestOnly + public void setBeforeFlockReleaseHook(Runnable hook) { + this.beforeFlockReleaseHook = hook; + } + + /** + * Runs on a safe manager-worker handoff path when {@link #close()} moved + * cleanup ownership to either a shared manager's ring-pass completion or + * an owned manager's worker exit. + * Deliberately does NOT take the engine monitor: a retried close() can + * hold it while joining this very worker, and the thread cannot die until + * this method returns — the {@link #terminalCleanupClaimed} CAS provides + * the exactly-once exclusion instead, so the worker always exits promptly + * and the racing close() converges via {@code isWorkerReaped()}. + */ + private void completeDeferredClose() { + if (!terminalCleanupClaimed.compareAndSet(false, true)) { + // A retried close() (or an earlier duplicate handoff) already ran + // the terminal cleanup. + return; + } + finishClose(fullyDrainedForDeferredClose); + LOG.info("deferred SF engine resource cleanup completed after manager-worker quiescence; " + + "slot release confirmed: {} [slot={}]", + closeCompleted, sfDir == null ? "" : sfDir); + } + + private Runnable createDeferredClose() { + Runnable hook = beforeDeferredCloseCreationHook; + if (hook != null) { + hook.run(); + } + return this::completeDeferredClose; + } + + private boolean retryFlockReleaseIfReady() { + if (closeCompleted) { + return true; + } + if (!terminalResourcesCleaned) { + if (!terminalCleanupRetryReady + || !terminalCleanupClaimed.compareAndSet(false, true)) { + return false; + } + try { + finishClose(fullyDrainedForDeferredClose); + } catch (Throwable ignored) { + return false; + } + return closeCompleted; + } + boolean released; + if (slotLock != null) { + try { + released = slotLock.release(); + } catch (Throwable ignored) { + released = false; + } + } else { + released = true; + } + if (released) { + closeCompleted = true; + Runnable listener = slotLockReleaseListener; + if (listener != null) { try { - slotLock.close(); + listener.run(); } catch (Throwable ignored) { - // best-effort; flock is also released by kernel on process exit + // A notification failure must not invalidate a confirmed release. } } + return true; + } + startFlockReleaseRetry(); + return false; + } + + private static void runFlockReleaseRetryDriver() { + // Capped exponential backoff: a persistent unlock failure must not + // burn a fixed 10 rounds of native release syscalls per second + // forever, but a transient failure must still recover promptly. The + // ramp doubles per fully-failed round from 100 ms up to 5 s. + long parkNanos = FLOCK_RELEASE_RETRY_BASE_PARK_NANOS; + while (true) { + final int roundSize; + synchronized (FLOCK_RELEASE_RETRY_LOCK) { + roundSize = FLOCK_RELEASE_RETRY_QUEUE.size(); + if (roundSize == 0) { + flockReleaseRetryThread = null; + return; + } + } + boolean hasFailures = false; + boolean hasSuccesses = false; + for (int i = 0; i < roundSize; i++) { + final CursorSendEngine engine; + synchronized (FLOCK_RELEASE_RETRY_LOCK) { + engine = FLOCK_RELEASE_RETRY_QUEUE.pollFirst(); + } + if (engine.retryFlockReleaseIfReady()) { + engine.flockReleaseRetryStarted.set(false); + hasSuccesses = true; + } else { + synchronized (FLOCK_RELEASE_RETRY_LOCK) { + FLOCK_RELEASE_RETRY_QUEUE.addLast(engine); + } + hasFailures = true; + Runnable hook = afterFlockReleaseRetryFailureHook; + if (hook != null) { + hook.run(); + } + } + } + if (hasSuccesses) { + // Progress: the failure condition is clearing, so retry the + // remaining engines on the base cadence again. + parkNanos = FLOCK_RELEASE_RETRY_BASE_PARK_NANOS; + } + if (hasFailures) { + // Interruption must not abandon a retained flock, but clear + // the flag so subsequent parks still throttle retries. + Thread.interrupted(); + LongConsumer parkOverride = flockReleaseRetryParkOverride; + if (parkOverride != null) { + parkOverride.accept(parkNanos); + } else { + LockSupport.parkNanos(parkNanos); + } + parkNanos = Math.min(parkNanos * 2, FLOCK_RELEASE_RETRY_MAX_PARK_NANOS); + } + } + } + + private void startFlockReleaseRetry() { + if (!flockReleaseRetryStarted.compareAndSet(false, true)) { + return; + } + Throwable startFailure = null; + synchronized (FLOCK_RELEASE_RETRY_LOCK) { + FLOCK_RELEASE_RETRY_QUEUE.addLast(this); + if (flockReleaseRetryThread != null) { + // The driver may be parked on a ramped backoff; wake it so a + // freshly failed release gets its first driver retry promptly + // instead of inheriting older engines' full backoff. + LockSupport.unpark(flockReleaseRetryThread); + } else { + try { + Thread retryThread = flockReleaseRetryThreadFactory.newThread( + CursorSendEngine::runFlockReleaseRetryDriver); + if (retryThread == null) { + throw new IllegalStateException("retry thread factory returned null"); + } + retryThread.setDaemon(true); + flockReleaseRetryThread = retryThread; + retryThread.start(); + } catch (Throwable t) { + startFailure = t; + flockReleaseRetryThread = null; + CursorSendEngine queued; + while ((queued = FLOCK_RELEASE_RETRY_QUEUE.pollFirst()) != null) { + queued.flockReleaseRetryStarted.set(false); + } + } + } + } + if (startFailure == null) { + LOG.error("SF terminal cleanup or slot flock release failed during engine close; " + + "keeping closeCompleted=false and retrying on the shared driver so " + + "retired capacity recovers after the transient failure [slot={}]", + sfDir == null ? "" : sfDir); + } else { + // A later explicit close() or a pool's retired-slot probe + // (ensureFlockReleaseRetryScheduled) can still retry without + // repeating the one-time ring/watermark cleanup. The failed queue + // is cleared so the driver does not retain engines it cannot + // service. + LOG.error("Could not start SF flock-release retry driver; a retried close() " + + "or pool re-probe re-arms the retry [slot={}, error={}]", + sfDir == null ? "" : sfDir, String.valueOf(startFailure)); + } + } + + /** + * Whether {@link #close()} completed all cleanup, including a + * confirmed release of the SF slot lock — the flip is published + * strictly after explicit unlock succeeds, so observing {@code true} + * guarantees the slot dir is acquirable by a replacement engine. A false + * value after close means manager-worker quiescence could not be + * confirmed (or the flock release itself failed) and the + * worker-reachable resources were retained deliberately — either handed to the worker's exit path (owned manager), + * which flips this to true the moment the worker's in-flight pass + * finishes, or leaked until a retried close() (shared manager). Owners + * must not reuse the slot while this is false; pools may re-probe it to + * recover a retired slot's capacity once it flips. + *

    + * Deliberately unsynchronized ({@code closeCompleted} is volatile): pools + * probe this under their own capacity lock, and the deferred cleanup can + * hold the engine monitor through munmap/unlink syscalls — a synchronized + * getter would stall the pool's hot borrow path behind them. + */ + public boolean isCloseCompleted() { + return closeCompleted; + } + + /** + * Registers a callback for confirmed SF slot-lock release. If release + * already completed, invokes the callback before returning. + */ + public void setSlotLockReleaseListener(Runnable listener) { + slotLockReleaseListener = listener; + if (listener != null && closeCompleted) { + listener.run(); + } + } + + /** + * Re-arms the shared terminal retry for an engine whose final watermark + * barrier or confirmed flock release is still pending and no longer + * scheduled because the retry driver thread failed to start (e.g. OOM at + * thread creation). A close still waiting for worker quiescence cannot be + * retried here because terminal cleanup may not race that worker. + * {@code Sender.close()} is one-shot by contract, so pool probes + * ({@code QwpWebSocketSender.isSlotLockReleased()}) call this to keep a + * retired slot's capacity recoverable instead of lost until process + * exit. Cheap unless the engine is in that orphan state (volatile reads, + * then one failed CAS when the retry is already scheduled), so probes + * may call it under their capacity lock. + */ + public void ensureFlockReleaseRetryScheduled() { + if (!closeCompleted && (terminalResourcesCleaned || terminalCleanupRetryReady)) { + startFlockReleaseRetry(); } } @@ -569,6 +1272,10 @@ public MmapSegment findSegmentContaining(long fsn) { return ring.findSegmentContaining(fsn); } + MmapSegment pinSegmentContaining(long fsn) { + return ring.pinSegmentContaining(fsn); + } + /** * Pass-through to {@link SegmentRing#firstSealed()}. */ @@ -593,6 +1300,14 @@ public MmapSegment nextSealedAfter(MmapSegment current) { return ring.nextSealedAfter(current); } + MmapSegment advancePinnedSegment(MmapSegment current) { + return ring.advancePinnedSegment(current); + } + + void releasePinnedSegment(MmapSegment current) { + ring.releasePinnedSegment(current); + } + /** * I/O thread accessor: highest FSN whose frame is fully written. */ @@ -660,36 +1375,138 @@ public long recoveredOrphanTipFsn() { } /** - * Unlinks every {@code .sfa} file under {@code dir}. Called only on - * clean shutdown when the ring confirms every published FSN has been - * acked — at that moment the slot has no recoverable work and the - * files are pure noise that would mislead the next sender's recovery. - * Best-effort: logs and continues on failures, since we're already on - * the close path. + * Ascending removal rank of a segment file name for + * {@link #unlinkAllSegmentFiles(String)}. {@code sf-initial.sfa} is + * always the fresh-start segment at baseSeq 0, so it ranks first. + * Spare files carry a monotonic generation ({@code sf-.sfa}) + * assigned in creation == rotation == baseSeq order, so the parsed + * generation ranks them. Anything else with the extension is not a + * live segment and ranks last. + */ + private static long segmentCleanupRank(String name) { + if ("sf-initial.sfa".equals(name)) { + return Long.MIN_VALUE; + } + if (name.length() == 23 && name.startsWith("sf-")) { + try { + return Long.parseUnsignedLong(name.substring(3, 19), 16); + } catch (NumberFormatException ignored) { + // fall through to the unrecognized-name rank + } + } + return Long.MAX_VALUE; + } + + /** + * Unlinks every {@code .sfa} file under {@code dir}, then the SF + * manifest. Called only on clean shutdown when the ring confirms every + * published FSN has been acked — at that moment the slot has no + * recoverable work and the files are pure noise that would mislead the + * next sender's recovery. + *

    + * Crash safety comes from a single durable manifest update committed + * BEFORE the first unlink: collapsing the boundaries to + * {@code head == active} declares every frame below the active base + * trimmed, so any subset of surviving files recovers cleanly (stale + * files are discarded, a surviving active recovers as the whole chain, + * zero files with a leftover manifest is the recognized drain window + * and recovers as EMPTY). Removal still runs in ascending segment order + * and STOPS at the first failure; the retained ack watermark (== the + * final acked FSN) covers every surviving frame, so a successor replays + * nothing. Legacy slots without a manifest keep the pre-manifest + * argument: ascending-order removal preserves a contiguous top slice + * that passes FSN-contiguity. + * + * @return {@code true} only when enumeration succeeded and every + * {@code .sfa} file was confirmed removed — the caller keeps the ack + * watermark on {@code false} so residual acknowledged segments stay + * covered. */ - private static void unlinkAllSegmentFiles(String dir) { - if (!io.questdb.client.std.Files.exists(dir)) return; - long find = io.questdb.client.std.Files.findFirst(dir); + private static boolean unlinkAllSegmentFiles(FilesFacade filesFacade, String dir) { + if (!filesFacade.exists(dir)) return true; + long find = filesFacade.findFirst(dir); if (find < 0) { LOG.warn("close-time unlink could not enumerate {}; " + "any residual sf-*.sfa files will be picked up by the next recovery", dir); - return; + return false; } - if (find == 0) return; + if (find == 0) return true; + ArrayList names = new ArrayList<>(); + int rc = 1; try { - int rc = 1; while (rc > 0) { String name = io.questdb.client.std.Files.utf8ToString( - io.questdb.client.std.Files.findName(find)); - rc = io.questdb.client.std.Files.findNext(find); - if (name == null || !name.endsWith(".sfa")) continue; - String path = dir + "/" + name; - if (!io.questdb.client.std.Files.remove(path)) { - LOG.warn("Failed to unlink fully-acked segment {} on close", path); + filesFacade.findName(find)); + rc = filesFacade.findNext(find); + if (name != null && name.endsWith(".sfa")) { + names.add(name); } } } finally { - io.questdb.client.std.Files.findClose(find); + filesFacade.findClose(find); + } + if (rc < 0) { + // A partial listing must not drive any unlink: removing only the + // files we happened to see could delete the segment holding the + // highest frame while a lower one survives, leaving residual + // state the retained watermark can no longer vouch for. + LOG.warn("close-time unlink could not fully enumerate {}; " + + "leaving all segment files for the next recovery", dir); + return false; + } + names.sort((a, b) -> { + int byRank = Long.compare(segmentCleanupRank(a), segmentCleanupRank(b)); + return byRank != 0 ? byRank : a.compareTo(b); + }); + // Manifest-aware drain protocol. Before removing anything, durably + // collapse the committed boundaries to head == active. From that + // moment EVERY crash state is a valid recovery input: + // - surviving data below the active base is "stale below head" + // (acked by definition here) and is discarded by recovery; + // - a surviving active-base segment recovers as the whole chain; + // - empty spares are validated extras; + // - zero segment files with the manifest still present is the + // recognized drain window and recovers as EMPTY. + // Without this barrier, deleting the file the manifest calls "head" + // would make the next startup fail on a slot that lost nothing. + // The manifest itself is removed last, after every segment unlink. + SfManifest manifest = null; + try { + try { + manifest = SfManifest.open(filesFacade, dir); + } catch (Throwable t) { + LOG.warn("close-time unlink could not read the SF manifest in {}; leaving " + + "all files for the next recovery", dir, t); + return false; + } + if (manifest != null && !names.isEmpty()) { + try { + long activeBase = manifest.activeBase(); + manifest.update(activeBase, activeBase); + } catch (Throwable t) { + LOG.warn("close-time unlink could not commit drain boundaries in {}; " + + "leaving all files for the next recovery", dir, t); + return false; + } + } + for (int i = 0, n = names.size(); i < n; i++) { + String path = dir + "/" + names.get(i); + if (!filesFacade.remove(path)) { + LOG.warn("Failed to unlink fully-acked segment {} on close; stopping so the " + + "residual files stay a watermark-covered, manifest-consistent range", path); + return false; + } + } + } finally { + if (manifest != null) { + manifest.close(); + } + } + if (!SfManifest.removeFile(filesFacade, dir)) { + // Safe to proceed: a manifest with zero segment files is the + // recognized drain window and recovers as EMPTY. + LOG.warn("could not remove SF manifest in {} after full segment cleanup", dir); } + return true; } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 13a69f77..58c3d31e 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -49,6 +49,7 @@ import java.util.Arrays; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.LockSupport; @@ -88,6 +89,29 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { * sub-second confirmation latency once the upload completes * server-side. {@code 0} or negative disables the keepalive entirely. */ + /** + * Bounded-await backstop for {@link #close()}: the maximum time close() + * waits for the I/O thread to stop (count down {@code shutdownLatch}) + * before it loud-fails and delegates final teardown to the I/O thread's + * exit path. In the common case the round-2 in-flight connect cancellation + * (see {@link ConnectCancellation}) makes the I/O thread unwind and count + * the latch down within milliseconds, so this budget is never reached. It + * only fires in the pathological, astronomically-rare TOCTOU window where + * {@code cancel()}'s {@code closeTraffic()} lands between the pre-connect + * guard and native fd creation, so it is a no-op and the connect blocks the + * full OS SYN-retry (~60-130s per endpoint, possibly across several). + *

    + * {@code 30_000} ms is comfortably UNDER the sidecar's 120 s shutdown + * deadline (~4x headroom) yet ~1000x a healthy close's millisecond latch + * countdown, so it never prematurely abandons a legitimately draining + * close (the I/O thread only has to finish its current — now traffic-broken + * — native send/receive and run {@code ioLoop}'s finally). This bounds the + * WAIT inside close(), NOT the connect itself (rejected Option A): a + * legitimately slow SUCCESSFUL connect is unaffected — it is not concurrent + * with a close, and when close() does fire it cancels the in-flight connect + * rather than waiting it out. + */ + public static final long DEFAULT_CLOSE_SHUTDOWN_AWAIT_MILLIS = 30_000L; public static final long DEFAULT_DURABLE_ACK_KEEPALIVE_INTERVAL_MILLIS = 200L; public static final long DEFAULT_PARK_NANOS = 50_000L; // 50us idle backoff /** @@ -163,6 +187,11 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // the durable watermark can lag behind the OK watermark. private final ArrayDeque pendingDurable = new ArrayDeque<>(); private final ArrayDeque pendingDurablePool = new ArrayDeque<>(); + // Race-safe cancellation handle for an in-flight connect attempt. Passed + // into reconnectFactory.reconnect(...) on the I/O thread so close() can + // break a connect blocked mid-attempt (a black-holed native connect that + // neither unpark nor interrupt cancels). See ConnectCancellation. + private final ConnectCancellation connectCancellation = new ConnectCancellation(); // Optional reconnect plumbing. When non-null, a wire failure triggers a // reconnect attempt instead of a terminal fail(). The factory produces a // fresh, connected+upgraded WebSocketClient. @@ -200,7 +229,7 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // by category. Includes both retriable and terminal outcomes — i.e. every // server-side rejection observed regardless of how the loop reacted. private final AtomicLong totalServerErrors = new AtomicLong(); - private WebSocketClient client; + private volatile WebSocketClient client; // Optional: when non-null, every server-rejection error (retriable and // terminal alike) is offered to the dispatcher for async delivery to the user's // handler. Null disables async delivery entirely; the producer-side @@ -225,6 +254,12 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // it is engine.ackedFsn() + 1, so the first replayed frame on the new // connection is wireSeq=0 and server-side cumulative ACKs still line up. private long fsnAtZero; + // Bounded-await backstop budget for close() (see + // DEFAULT_CLOSE_SHUTDOWN_AWAIT_MILLIS). Overridable via + // setShutdownAwaitTimeoutMillis so tests can exercise the timeout branch + // deterministically without a multi-second real wait; production always + // uses the default. Read only on the owner thread inside close(). + private long shutdownAwaitTimeoutMillis = DEFAULT_CLOSE_SHUTDOWN_AWAIT_MILLIS; // Sticky flag: false until the very first time a live client is installed // (either via the constructor in SYNC/OFF mode or via swapClient on a // successful connect attempt in any mode). Once true, stays true. Used to @@ -244,11 +279,12 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // terminalError: the only writer runs on the I/O thread under the same // first-writer-wins latch. private volatile QwpDurableAckMismatchException capabilityGapTerminal; - // Failed-stop hand-off flag: set by delegateEngineClose() when an owner's - // close() could not stop the I/O thread and the engine close is therefore - // performed by the I/O thread's exit path. Write-once, owner thread only; - // read by the I/O thread strictly after its shutdown-latch countdown (see - // the handshake contract on delegateEngineClose). + // Failed-stop hand-off callback: set when an owner could not stop the I/O + // thread and must defer worker-reachable cleanup until the thread exits. + // Read strictly after shutdownLatch.countDown(); see delegateClose(). + private volatile Runnable delegatedClose; + // Engine-only hand-off retained for BackgroundDrainer, whose remaining + // resources are owned by its run method rather than by a sender. private volatile boolean engineCloseDelegated; // The latched terminal failure — THE exception every checkError() call // rethrows. Write-once for the loop's lifetime: the only writer is @@ -268,11 +304,11 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { private long nextWireSeq; private volatile SenderProgressDispatcher progressDispatcher; // Frames sent during the post-reconnect catch-up window — i.e. frames - // whose FSN was already published before the wire dropped. A non-zero + // whose FSN completed a send on an earlier connection. A non-zero // value confirms replay is working; a sustained nonzero rate means // the connection is flapping and replay is doing real work each cycle. - // Set at swapClient time to publishedFsn at that moment; cleared back - // to -1 once trySendOne has caught up past it. Used to count replay + // Snapshotted when an established connection enters reconnect; cleared + // back to -1 once trySendOne has caught up past it. Used to count replay // frames without a per-frame branch on the steady-state path. private long replayTargetFsn = -1L; // Recovered orphaned deferred tail: frames [orphanSkipStartFsn .. @@ -763,8 +799,74 @@ public synchronized void close() { // would block forever. isAlive()==false also covers the normal // post-exit case where the latch is already counted down. if (t.isAlive()) { + // Break a native send/receive before joining. Full client close + // must remain after the worker exit because it frees buffers the + // worker may still access. + WebSocketClient activeClient = client; + if (activeClient != null) { + try { + activeClient.closeTraffic(); + } catch (Throwable e) { + // A custom transport that cannot safely cancel traffic + // must not be destructively closed while the worker is + // live. Fail without joining indefinitely; the worker's + // exit path retains ownership of final cleanup. + throw new LineSenderException( + "cursor I/O thread did not stop: active transport does not support safe traffic shutdown; " + + "client/engine teardown is delegated to the I/O thread's exit path", + e + ); + } + } + // Cancel an in-flight connect attempt not yet installed as the + // `client` field: async initial connect leaves `client` null + // and a mid-flight reconnect leaves it pointing at the stale + // pre-drop client, so neither is reachable by the closeTraffic() + // above. The reconnect factory publishes the connecting client + // here before it blocks, so breaking its traffic unwinds a + // black-holed native connect (connect_timeout=0 => OS SYN-retry, + // ~60-130s) that would otherwise pin the untimed await below. + // Same loud-fail contract as the field client: a transport that + // cannot safely shut down traffic is not destructively closed; + // the worker's exit path retains ownership of final cleanup. + try { + connectCancellation.cancel(); + } catch (Throwable e) { + throw new LineSenderException( + "cursor I/O thread did not stop: active transport does not support safe traffic shutdown; " + + "client/engine teardown is delegated to the I/O thread's exit path", + e + ); + } try { - shutdownLatch.await(); + // Bounded backstop: never wait forever. The round-2 in-flight + // connect cancellation makes the I/O thread unwind and count + // the latch down in milliseconds, so this await returns true + // almost always. The timeout branch fires only in the + // pathological uninterruptible-connect TOCTOU window where + // cancel()'s closeTraffic() was a no-op and the connect blocks + // the full OS SYN-retry -- guaranteeing close() still returns + // (bounded, comfortably under the sidecar's 120s deadline) + // rather than hanging on an untimed await. + if (!shutdownLatch.await(shutdownAwaitTimeoutMillis, TimeUnit.MILLISECONDS)) { + // Latch still up after the budget: the I/O thread did not + // stop. Same failed-stop protocol as the interrupt branch + // below -- loud-fail without touching the client/engine + // (freeing native buffers under a still-live I/O thread + // risks a C5 SEGV; a quiet return would let the owner + // unmap the engine under it). The I/O thread's own exit + // path (ioLoop's finally) retains ownership of final + // cleanup; QwpWebSocketSender.close() keys its + // ioThreadStopped guard on this throw and BackgroundDrainer + // switches to delegateEngineClose(). ioThread stays set + // (not nulled below, since we throw) so a duplicate close() + // re-signals rather than silently succeeding. + throw new LineSenderException( + "cursor I/O thread did not stop: close() timed out after " + + shutdownAwaitTimeoutMillis + "ms awaiting shutdown; " + + "client/engine teardown is delegated to the I/O " + + "thread's exit path"); + } } catch (InterruptedException e) { // Re-assert the flag for the caller's stack, then decide. // If the I/O thread has genuinely not exited (latch still @@ -794,6 +896,9 @@ public synchronized void close() { } ioThread = null; } + // Covers close-before-start and start failures; normal I/O-thread + // exit already released the same pin in its finally block. + releaseSendingSegment(); // Close the current client. After a reconnect, swapClient has // replaced the original (and closed it); the owner only retains // the stale pre-reconnect reference. Without closing the live @@ -815,27 +920,20 @@ public synchronized void close() { } /** - * Failed-stop hand-off for the engine. Called by an owner whose - * {@link #close()} threw because the I/O thread would not stop: the owner - * must not free the engine (munmap/Unsafe.free of segment memory) while - * the thread may still touch it with raw {@code Unsafe} reads. Setting - * the delegation flag makes the I/O thread run {@code engine.close()} on - * its exit path, strictly after its last engine access and after the - * shutdown-latch countdown — releasing the slot lock as soon as the - * stuck wire call resolves (bounded by OS timeouts) instead of leaking - * the mapping and lock forever. - *

    - * Returns {@code true} when the I/O thread is still live and has adopted - * the engine close; {@code false} when the thread has already exited — - * the caller must close the engine itself. - *

    - * Memory model — the classic store/load handshake: this method writes the - * volatile flag, then reads the latch count; the exit path counts the - * latch down, then reads the flag. Under the sequential consistency of - * volatile (and AQS latch state) accesses, if this method observes the - * latch still up, the exit path is guaranteed to observe the flag — no - * missed close. If both sides act, {@link CursorSendEngine#close()} is - * synchronized and idempotent, so the double close is benign. + * Hands complete owner cleanup to the I/O thread when it could not be + * stopped. The callback runs after the thread's last client, buffer, and + * engine access. Returns false if the thread already exited, in which case + * the caller must run the callback. The callback itself must be idempotent: + * the latch/callback handshake guarantees execution but permits both sides + * to race after the countdown. + */ + public boolean delegateClose(Runnable closeCallback) { + delegatedClose = closeCallback; + return shutdownLatch.getCount() != 0L; + } + + /** + * Engine-only failed-stop hand-off used by BackgroundDrainer. */ public boolean delegateEngineClose() { engineCloseDelegated = true; @@ -983,6 +1081,17 @@ public void setProgressDispatcher(SenderProgressDispatcher dispatcher) { this.progressDispatcher = dispatcher; } + /** + * Test seam: shrink the {@link #close()} bounded-await backstop + * (default {@link #DEFAULT_CLOSE_SHUTDOWN_AWAIT_MILLIS}) so a test can + * exercise the timeout branch deterministically without a multi-second + * real wait. Production never calls this. Set before {@link #close()}. + */ + @TestOnly + public void setShutdownAwaitTimeoutMillis(long millis) { + this.shutdownAwaitTimeoutMillis = millis; + } + public synchronized void start() { if (ioThread != null) { throw new IllegalStateException("already started"); @@ -996,17 +1105,24 @@ public synchronized void start() { // walks back to the lowest unacked frame so sealed-segment data // actually reaches the wire — without it, start() would skip // straight to the active and orphan everything in sealed. - positionCursorForStart(); + try { + positionCursorForStart(); + } catch (Throwable t) { + running = false; + releaseSendingSegment(); + shutdownLatch.countDown(); + throw t; + } Thread t = new Thread(this::ioLoop, "qdb-cursor-ws-io"); t.setDaemon(true); try { t.start(); } catch (Throwable th) { // Thread.start() failed (e.g. native stack alloc OOM). ioLoop - // never ran, so its finally{shutdownLatch.countDown()} never - // fires. Release the latch and reset state so a subsequent - // close() doesn't block on a thread that doesn't exist. + // never ran, so its finally block cannot release the segment pin + // or count down the latch. running = false; + releaseSendingSegment(); shutdownLatch.countDown(); throw th; } @@ -1026,35 +1142,17 @@ private PendingDurableEntry acquirePendingEntry() { * else the active). Returns the same segment if it's still being written * (we're on the active and just need to wait for more publishedFsn). *

    - * Uses {@link CursorSendEngine#nextSealedAfter} so we never have to - * snapshot the full sealed list — important when the producer outpaces - * the I/O thread and the sealed list can grow to thousands of entries - * (cursor SF lets the producer fan out at memory speed; the wire path - * catches up at WebSocket speed). + * The ring switches the single I/O pin atomically with choosing the next + * segment, so trim can never unmap either side of the handoff. No sealed + * list snapshot is needed when the producer outpaces the wire path. */ private MmapSegment advanceSegment() { MmapSegment current = sendingSegment; - MmapSegment liveActive = engine.activeSegment(); - if (current == liveActive) { - // We're on the active — there's no "next", just wait for more - // bytes to be published into it. Caller's sendOne will see - // publishedOffset > sendOffset eventually and resume. - return current; + MmapSegment next = engine.advancePinnedSegment(current); + if (next != current) { + sendOffset = MmapSegment.HEADER_SIZE; } - sendOffset = MmapSegment.HEADER_SIZE; - MmapSegment next = engine.nextSealedAfter(current); - if (next != null) { - return next; - } - // current was the newest sealed (no later sealed exists). If it's - // still in the sealed list, the next segment must be the active; - // if it's been trimmed out from under us, fall back to the oldest - // remaining sealed before resorting to the active. - next = engine.firstSealed(); - if (next != null && next.baseSeq() > current.baseSeq()) { - return next; - } - return liveActive; + return next; } private void applyDurableAck() { @@ -1114,10 +1212,14 @@ private void clearDurableAckTracking() { * before the first connect attempt — see {@link #failPaced}. */ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptMillis) { - if (reconnectFactory == null || !running) { + if (!running) { + return; + } + if (reconnectFactory == null) { recordFatal(initial); return; } + snapshotReplayTarget(); LOG.warn("cursor I/O loop entering {} loop: {}", phase, initial.getMessage()); long outageStartNanos = System.nanoTime(); @@ -1156,7 +1258,7 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM attempts++; totalReconnectAttempts.incrementAndGet(); try { - WebSocketClient newClient = reconnectFactory.reconnect(); + WebSocketClient newClient = reconnectFactory.reconnect(connectCancellation); if (newClient != null) { if (!running) { // close() ran while this connect attempt was in @@ -1650,6 +1752,9 @@ private void ioLoop() { } } } finally { + // Release native-segment lifetime before publishing I/O-thread + // completion or running delegated engine cleanup. + releaseSendingSegment(); // Last act of the I/O thread: dispose of whatever client it // holds. This is the airtight half of the close()-vs-reconnect // race — when close()'s latch await is interrupted (drainer pool @@ -1670,15 +1775,14 @@ private void ioLoop() { } } shutdownLatch.countDown(); - // Failed-stop hand-off (see delegateEngineClose): the owner could - // not free the engine safely while this thread was alive, so the - // engine close — and with it the slot-lock release — happens - // here, strictly after this thread's last engine access. The flag - // is read only after the countDown: the store/load pairing with - // delegateEngineClose's flag-write-then-latch-read guarantees - // either this branch or the owner's fallback runs (or both — - // engine.close() is idempotent). - if (engineCloseDelegated) { + Runnable closeCallback = delegatedClose; + if (closeCallback != null) { + try { + closeCallback.run(); + } catch (Throwable ignored) { + // The owner callback logs individual cleanup failures. + } + } else if (engineCloseDelegated) { try { engine.close(); } catch (Throwable ignored) { @@ -1700,7 +1804,7 @@ private void ioLoop() { * loop will then wait until the producer publishes more bytes. */ private void positionCursorAt(long targetFsn) { - MmapSegment seg = engine.findSegmentContaining(targetFsn); + MmapSegment seg = engine.pinSegmentContaining(targetFsn); if (seg == null) { // No segment currently advertises targetFsn. That normally means // targetFsn is just past publishedFsn and there is nothing to @@ -1709,14 +1813,14 @@ private void positionCursorAt(long targetFsn) { // The producer is concurrent with this I/O thread, though. It can // publish targetFsn after the first findSegmentContaining() returns // null but before or during the active-tip snapshot below. - sendingSegment = engine.activeSegment(); + sendingSegment = engine.pinActiveSegment(); sendOffset = sendingSegment.publishedOffset(); // The publishedOffset read is the producer's volatile publish // barrier. If it saw the new frame bytes, the frameCount write that // makes targetFsn discoverable is also visible, so a second lookup // must now find it. If the producer publishes later, sendOffset is // still at the old tip and trySendOne() will send the frame normally. - seg = engine.findSegmentContaining(targetFsn); + seg = engine.pinSegmentContaining(targetFsn); if (seg != null) { positionCursorInSegment(seg, targetFsn); } @@ -1796,6 +1900,14 @@ private void releasePendingEntry(PendingDurableEntry e) { pendingDurablePool.addFirst(e); } + private void releaseSendingSegment() { + MmapSegment segment = sendingSegment; + if (segment != null) { + engine.releasePinnedSegment(segment); + sendingSegment = null; + } + } + /** * Send a WebSocket PING to prod the server into flushing pending * STATUS_DURABLE_ACK frames, but only when the throttle interval has @@ -1823,6 +1935,23 @@ private void sendDurableAckKeepaliveIfDue() { } } + /** + * Freezes the highest FSN that completed a send on the connection which + * just failed. Frames published before the first connection, or while a + * reconnect is in progress, have not been sent and therefore do not belong + * to the replay metric. Keep an older, higher target when another failure + * interrupts catch-up so the remaining frames still count as re-sends on + * the following connection. + */ + private void snapshotReplayTarget() { + if (hasEverConnected && nextWireSeq > 0L) { + long lastSentFsn = fsnAtZero + (nextWireSeq - 1L); + if (lastSentFsn > replayTargetFsn) { + replayTargetFsn = lastSentFsn; + } + } + } + /** * Reset wire state for a fresh connection: install the new client, * realign {@code fsnAtZero} to the next unacked FSN, restart wire @@ -1851,11 +1980,12 @@ private void swapClient(WebSocketClient newClient) { long replayStart = engine.ackedFsn() + 1L; this.fsnAtZero = replayStart; this.nextWireSeq = 0L; - // Snapshot publishedFsn at swap time — frames at FSN ≤ this value - // were already on the wire before the drop and will be replayed. - // trySendOne resets replayTargetFsn to -1 once we cross the boundary. - long pubAtSwap = engine.publishedFsn(); - this.replayTargetFsn = pubAtSwap >= replayStart ? pubAtSwap : -1L; + // snapshotReplayTarget froze the completed-send boundary when the + // outage began. ACK progress can move replayStart past that boundary; + // in that case no frame needs re-sending on this connection. + if (replayTargetFsn < replayStart) { + replayTargetFsn = -1L; + } // Drop any durable-ack tracking from the previous connection. The // new connection will re-OK every replayed batch and the server // re-emits cumulative durable-ack watermarks from scratch, so @@ -2034,6 +2164,104 @@ private boolean tryRetireOrphanTail() { @FunctionalInterface public interface ReconnectFactory { WebSocketClient reconnect() throws Exception; + + /** + * Cancellable variant of {@link #reconnect()}. The loop passes a + * per-attempt {@link ConnectCancellation} so a transport that blocks + * inside a native connect can publish the in-flight client to the + * handle BEFORE it blocks; {@link #close()} then breaks that client's + * traffic to unwind a black-holed connect promptly, rather than falling + * back to the bounded {@code shutdownLatch.await(...)} backstop (which + * only bounds the WAIT, not the connect). + *

    + * Default: ignore the handle and delegate to {@link #reconnect()} -- + * a transport that cannot publish its in-flight client is simply not + * cancellable mid-connect, and {@code close()} falls back to its + * existing field-client {@code closeTraffic()} / loud-fail path. The + * default keeps this a {@link FunctionalInterface} and preserves + * source compatibility for existing lambda / method-reference + * implementors (e.g. {@link #connectWithRetry}). + */ + default WebSocketClient reconnect(ConnectCancellation cancellation) throws Exception { + return reconnect(); + } + } + + /** + * Race-safe cancellation handle for a single in-flight connect attempt. + * Owned per-loop and passed into {@link ReconnectFactory#reconnect(ConnectCancellation)} + * on the I/O thread. The transport's connect walk {@link #publish}es the + * {@link WebSocketClient} it is about to block on before the blocking + * {@code connect()}, and {@link #clear}s it once the attempt is installed + * or disposed. {@link #close()} (owner thread) calls {@link #cancel()} to + * break the in-flight client's traffic path so a black-holed native + * connect unwinds and the I/O thread counts down the shutdown latch. + *

    + * Java 8: two volatiles, no locks. Visibility/ordering: + *

    + * The publish (write inFlight, then read cancelled) / cancel (write + * cancelled, then read inFlight) handshake covers the close-vs-connect + * race in both directions: if the guard observes {@code cancelled==false} + * then, by the volatile total order, {@code cancel()} observed the + * published client and broke its traffic; if the guard observes + * {@code cancelled==true} it skips the blocking connect entirely. + */ + public static final class ConnectCancellation { + // The client the connect walk is currently about to block / blocked + // on. Written by the I/O thread only (publish/clear); read by the + // owner thread (cancel). volatile for cross-thread visibility. + private volatile WebSocketClient inFlight; + // Latched once close() requested cancellation. Written by the owner + // thread (cancel); read by the I/O thread's pre-connect guard. + private volatile boolean cancelled; + + public boolean isCancelled() { + return cancelled; + } + + /** + * I/O-thread hook: record the client the walk is about to block on, + * BEFORE the blocking {@code connect()}. Pairs with {@link #cancel()} + * so an attempt that publishes after cancellation is caught by the + * caller's {@link #isCancelled()} guard, and an attempt already + * blocked in {@code connect()} is broken by {@code cancel()}. + */ + public void publish(WebSocketClient client) { + inFlight = client; + } + + /** + * I/O-thread hook: the walk is done with the in-flight attempt + * (installed on success, disposed on failure). Drop the reference so a + * later {@link #cancel()} cannot touch a client the walk no longer + * owns. Single writer (I/O thread) for publish/clear, so no CAS is + * needed. + */ + public void clear() { + inFlight = null; + } + + /** + * Owner-thread hook from {@link #close()}: request cancellation, then + * break the traffic path of any in-flight connect so the I/O thread's + * connect walk unwinds. {@code cancelled} is written before reading + * {@code inFlight} to pair with {@link #publish(WebSocketClient)}. May + * throw when the transport cannot safely shut down traffic -- + * {@code close()} maps that to its existing loud-fail, exactly as it + * does for the field client's {@code closeTraffic()}. + */ + void cancel() { + cancelled = true; + WebSocketClient c = inFlight; + if (c != null) { + c.closeTraffic(); + } + } } /** diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java index 78e5db9f..edf70252 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java @@ -31,9 +31,12 @@ import io.questdb.client.std.Os; import io.questdb.client.std.QuietCloseable; import io.questdb.client.std.Unsafe; +import org.jetbrains.annotations.TestOnly; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.concurrent.atomic.AtomicBoolean; + /** * One mmap-backed SF segment file. The user thread (the single producer) * appends frames into the mapping; the I/O thread (the single consumer) reads @@ -54,19 +57,27 @@ * * The mapping is sized at construction and never grows. When * {@link #tryAppend} returns -1 the caller must rotate to a fresh segment. - * Closing the segment unmaps and closes the fd; data already written is - * durable under the page cache (and recoverable across JVM restarts) — call - * {@link #msync} for OS-crash durability. + * Closing the segment unmaps and closes the fd. Dirty mapped pages normally + * survive a producer-process restart in the OS page cache, but that is not a + * host-power-loss guarantee. {@link #msync} preserves the legacy mmap-only + * flush API; use the checked {@link #syncPublished()} mapping-plus-fd barrier + * for portable power-loss durability. */ public final class MmapSegment implements QuietCloseable { public static final int FILE_MAGIC = 0x31304653; // 'SF01' little-endian public static final int FRAME_HEADER_SIZE = 8; // u32 crc + u32 payloadLen public static final int HEADER_SIZE = 24; + public static final byte MANIFEST_REQUIRED_FLAG = 1; public static final byte VERSION = 1; - private static final int[] CRC32C_TABLE = buildCrc32cTable(); private static final Logger LOG = LoggerFactory.getLogger(MmapSegment.class); + // Deduplicates the process-wide "mlock refused" warning: the refusal is a + // soft downgrade (see syncPublished) and must not spam the log once per + // barrier when RLIMIT_MEMLOCK or the platform says no. + private static final AtomicBoolean MLOCK_REFUSAL_WARNED = new AtomicBoolean(); + private static final int RECOVERY_BUFFER_SIZE = 64 * 1024; + private final FilesFacade filesFacade; private final String path; private final long sizeBytes; // memoryBacked: true when the segment buffer lives in malloc'd native @@ -82,6 +93,10 @@ public final class MmapSegment implements QuietCloseable { // segment manager pre-creates spares before the producer knows the exact // baseSeq the new active will need. private long baseSeq; + // Highest published byte offset covered by a successful data barrier. + // The manager writes this after msync+fsync; the producer reads it before + // allowing rotation to seal the segment. + private volatile long durableCursor; private int fd; // frameCount: number of frames successfully appended. Single writer (the // producer thread in tryAppend); read cross-thread by the I/O thread via @@ -95,16 +110,34 @@ public final class MmapSegment implements QuietCloseable { // because the consumer must see writes in publication order — once the // producer bumps publishedCursor, every byte before it is fully written. private volatile long publishedCursor; + // Number of failure-arm re-dirty passes performed by syncPublished. + // Cold-path only; volatile so tests observing from another thread see it. + private volatile long redirtyPasses; + // Monotonic in-memory link to the segment that immediately follows this + // one. SegmentRing publishes it before promoting the successor to active; + // close deliberately retains it so a cursor can advance after head trim. + private volatile MmapSegment successor; // Bytes between the last valid frame and the file end that look like an - // attempted-but-invalid frame write (non-zero bytes at the bail-out - // position). Zero for fresh segments and for cleanly partially-filled + // attempted-but-invalid frame write (the suffix contains non-zero bytes). + // Zero for fresh segments and for cleanly partially-filled // segments (uninitialised tail). Set only by openExisting; visible to - // recovery callers for diagnostics. Final after construction. + // recovery callers for diagnostics. openExisting only OBSERVES the + // residue -- it never mutates the file: after a mid-file tear the suffix + // can hold unreachable valid-CRC frames that are the only surviving copy + // of real payloads. Ring recovery decides after the chain fully validates + // whether to zero it (sanitizeTornTail on the resumed active or on a + // proven-dead sealed suffix) or to fail closed leaving the bytes on disk + // for operator extraction. Final after construction. private final long tornTailBytes; + // Set once sanitizeTornTail has durably zeroed the observed residue. + // Written and read only on the recovery path (single-threaded, before + // the segment is published to producer/consumer threads). + private boolean tornTailSanitized; - private MmapSegment(String path, int fd, long mmapAddress, long sizeBytes, + private MmapSegment(FilesFacade filesFacade, String path, int fd, long mmapAddress, long sizeBytes, long baseSeq, long initialCursor, long frameCount, boolean memoryBacked, long tornTailBytes) { + this.filesFacade = filesFacade; this.path = path; this.fd = fd; this.mmapAddress = mmapAddress; @@ -112,6 +145,7 @@ private MmapSegment(String path, int fd, long mmapAddress, long sizeBytes, this.baseSeq = baseSeq; this.appendCursor = initialCursor; this.publishedCursor = initialCursor; + this.durableCursor = memoryBacked ? initialCursor : HEADER_SIZE; this.frameCount = frameCount; this.memoryBacked = memoryBacked; this.tornTailBytes = tornTailBytes; @@ -145,9 +179,13 @@ public static MmapSegment create(String path, long baseSeq, long sizeBytes) { * that filesystem only. */ public static MmapSegment create(FilesFacade ff, String path, long baseSeq, long sizeBytes) { + return create(ff, path, baseSeq, sizeBytes, false); + } + + static MmapSegment create(FilesFacade ff, String path, long baseSeq, long sizeBytes, boolean manifestRequired) { long pathPtr = ff.allocNativePath(path); try { - return create(ff, pathPtr, path, baseSeq, sizeBytes); + return create(ff, pathPtr, path, baseSeq, sizeBytes, manifestRequired); } finally { ff.freeNativePath(pathPtr); } @@ -162,13 +200,17 @@ public static MmapSegment create(FilesFacade ff, String path, long baseSeq, long * per-call {@code byte[]} + native-malloc the way the String overload does. */ public static MmapSegment create(FilesFacade ff, long pathPtr, String displayPath, long baseSeq, long sizeBytes) { + return create(ff, pathPtr, displayPath, baseSeq, sizeBytes, false); + } + + static MmapSegment create(FilesFacade ff, long pathPtr, String displayPath, long baseSeq, long sizeBytes, boolean manifestRequired) { if (sizeBytes < HEADER_SIZE + FRAME_HEADER_SIZE + 1) { throw new IllegalArgumentException( "sizeBytes too small for header + one minimal frame: " + sizeBytes); } - int fd = ff.openCleanRW(pathPtr); + int fd = ff.openRWExclusive(pathPtr); if (fd < 0) { - throw new MmapSegmentException("openCleanRW failed for " + displayPath); + throw new MmapSegmentException("exclusive create failed for " + displayPath); } // Reserve real disk blocks and advance EOF to sizeBytes in one // call. ENOSPC surfaces here, before the producer thread starts @@ -186,21 +228,21 @@ public static MmapSegment create(FilesFacade ff, long pathPtr, String displayPat } long addr = Files.FAILED_MMAP_ADDRESS; try { - addr = Files.mmap(fd, sizeBytes, 0, Files.MAP_RW, MemoryTag.MMAP_DEFAULT); + addr = ff.mmap(fd, sizeBytes, 0, Files.MAP_RW, MemoryTag.MMAP_DEFAULT); if (addr == Files.FAILED_MMAP_ADDRESS) { throw new MmapSegmentException("mmap failed for " + displayPath); } // Header goes straight into the mapping — no separate write syscall. Unsafe.getUnsafe().putInt(addr, FILE_MAGIC); Unsafe.getUnsafe().putByte(addr + 4, VERSION); - Unsafe.getUnsafe().putByte(addr + 5, (byte) 0); // flags + Unsafe.getUnsafe().putByte(addr + 5, manifestRequired ? MANIFEST_REQUIRED_FLAG : (byte) 0); // flags Unsafe.getUnsafe().putShort(addr + 6, (short) 0); // reserved Unsafe.getUnsafe().putLong(addr + 8, baseSeq); Unsafe.getUnsafe().putLong(addr + 16, Os.currentTimeMicros()); - return new MmapSegment(displayPath, fd, addr, sizeBytes, baseSeq, HEADER_SIZE, 0, false, 0L); + return new MmapSegment(ff, displayPath, fd, addr, sizeBytes, baseSeq, HEADER_SIZE, 0, false, 0L); } catch (Throwable t) { if (addr != Files.FAILED_MMAP_ADDRESS) { - Files.munmap(addr, sizeBytes, MemoryTag.MMAP_DEFAULT); + ff.munmap(addr, sizeBytes, MemoryTag.MMAP_DEFAULT); } ff.close(fd); // mmap (or header writes) failed after a successful allocate — @@ -234,7 +276,7 @@ public static MmapSegment createInMemory(long baseSeq, long sizeBytes) { Unsafe.getUnsafe().putShort(addr + 6, (short) 0); Unsafe.getUnsafe().putLong(addr + 8, baseSeq); Unsafe.getUnsafe().putLong(addr + 16, Os.currentTimeMicros()); - return new MmapSegment(null, -1, addr, sizeBytes, baseSeq, HEADER_SIZE, 0, true, 0L); + return new MmapSegment(null, null, -1, addr, sizeBytes, baseSeq, HEADER_SIZE, 0, true, 0L); } catch (Throwable t) { Unsafe.free(addr, sizeBytes, MemoryTag.NATIVE_DEFAULT); throw t; @@ -242,111 +284,136 @@ public static MmapSegment createInMemory(long baseSeq, long sizeBytes) { } /** - * Opens an existing segment file for recovery. mmaps it RW, validates the - * header magic / version, then scans frames forward verifying each CRC. + * Opens an existing segment file for recovery. Validates the header and + * scans frames through positioned reads, then mmaps the validated file RW. * The first bad CRC (or a frame whose declared length runs past the file * end) is treated as a torn tail; both cursors are positioned at the * start of that frame. Returns the segment ready for further appends. * Throws {@link MmapSegmentException} on header validation failure. *

    - * If recovery observes a torn tail (the bytes at the bail-out position - * are non-zero, indicating an attempted-but-failed frame write rather - * than clean unwritten space), a {@code WARN} is emitted with the byte - * count and the bytes are exposed via {@link #tornTailBytes()} so - * operators can detect silent truncation from corruption or partial - * writes. Clean partial fills (writer never attempted to write past the - * last valid frame) do not log and report {@code 0}. + * If recovery observes a torn tail (the suffix after the last valid frame + * contains non-zero bytes, indicating an attempted-but-failed frame write + * rather than clean unwritten space), a {@code WARN} is emitted with the + * byte count and the observed size is exposed via {@link #tornTailBytes()}. + * The residue itself is NOT touched: after a mid-file tear the suffix can + * still hold unreachable frames with valid CRCs -- the only surviving copy + * of real payloads -- so whether it may be destroyed is a chain-level + * decision this method cannot make. {@link SegmentRing} recovery invokes + * {@link #sanitizeTornTail()} only once the chain has fully validated, + * and the justification differs by role: sealed suffixes are zeroed on + * PROOF (frame accounting validated complete, so the residue can hold no + * replayable frame -- a tear that cost frames fails closed instead, with + * every byte left on disk for operator extraction), while the resumed + * active's tail is zeroed by POLICY -- past a mid-file tear it can still + * hold valid-CRC frames of real unacked payloads, but they are + * unreachable by replay (the FSN sequence breaks at the tear) and + * leaving them risks the reseal-brick and stale-frame-resurrection + * hazards. Clean + * partial fills (writer never attempted to write past the last valid + * frame) do not log and report {@code 0}. */ public static MmapSegment openExisting(String path) { return openExisting(FilesFacade.INSTANCE, path); } /** - * Facade-aware variant of {@link #openExisting(String)} that takes the file - * length (and open/close) through a {@link FilesFacade} instead of straight - * off {@link Files}. Production uses {@link FilesFacade#INSTANCE}; the seam - * exists so recovery's mmap-fault guard can be regression-tested on any - * filesystem. + * Facade-aware variant of {@link #openExisting(String)}. Recovery reads the + * file through {@link FilesFacade#read(int, long, long, long)} before it + * creates the mapping, so sparse or unbacked pages cannot raise SIGBUS in + * the JVM. The descriptor length is checked before and after the scan and + * again after mmap; a short read or size change is an operational failure + * and aborts recovery. *

    - * The mapping is sized to {@code ff.length(path)} and every recovery read - * runs straight out of it, so a facade that reports a length larger - * than the real file makes the mapping extend past end-of-file. A read of a - * page beyond real EOF raises SIGBUS on every filesystem — the same - * fault an unbacked/sparse page raises on ZFS, but reproduced - * deterministically on ext4/xfs, where a within-EOF hole would instead - * zero-fill and never exercise the guard. See - * {@link FilesFacade#length(String)}. + * The caller must prevent concurrent writers from modifying the file during + * recovery and for the lifetime of the returned segment. The length checks + * detect observed truncate/extend races, but no mmap-based implementation + * can remain safe against uncoordinated mutation after the final check. */ public static MmapSegment openExisting(FilesFacade ff, String path) { - long fileSize = ff.length(path); - if (fileSize < HEADER_SIZE) { - throw new MmapSegmentException("file shorter than header: " + path + " size=" + fileSize); - } int fd = ff.openRW(path); if (fd < 0) { throw new MmapSegmentException("openRW failed for " + path); } long addr = Files.FAILED_MMAP_ADDRESS; + long fileSize = -1L; try { - addr = Files.mmap(fd, fileSize, 0, Files.MAP_RW, MemoryTag.MMAP_DEFAULT); + fileSize = ff.length(fd); + if (fileSize < 0) { + throw new MmapSegmentException( + "could not stat open segment " + path + " [errno=" + Os.errno() + ']'); + } + if (fileSize < HEADER_SIZE) { + // Corruption, not an operational error: the bytes themselves prove + // this cannot be a whole segment (a create() is never durable at a + // sub-header size — allocate() reserves the full extent up front). + throw new MmapSegmentCorruptionException( + "file shorter than header: " + path + " size=" + fileSize); + } + + RecoveryScan scan = scanForRecovery(ff, fd, path, fileSize); + long finalSize = ff.length(fd); + if (finalSize < 0) { + throw new MmapSegmentException( + "could not re-stat open segment " + path + " [errno=" + Os.errno() + ']'); + } + if (finalSize != fileSize) { + throw new MmapSegmentException( + "segment size changed during recovery: " + path + + " [before=" + fileSize + ", after=" + finalSize + ']'); + } + + addr = ff.mmap(fd, fileSize, 0, Files.MAP_RW, MemoryTag.MMAP_DEFAULT); if (addr == Files.FAILED_MMAP_ADDRESS) { throw new MmapSegmentException("mmap failed for " + path); } - int magic = Unsafe.getUnsafe().getInt(addr); - if (magic != FILE_MAGIC) { + long mappedSize = ff.length(fd); + if (mappedSize < 0) { throw new MmapSegmentException( - "bad magic in " + path + ": 0x" + Integer.toHexString(magic)); - } - byte version = Unsafe.getUnsafe().getByte(addr + 4); - if (version != VERSION) { - throw new MmapSegmentException("unsupported version in " + path + ": " + version); + "could not stat mapped segment " + path + " [errno=" + Os.errno() + ']'); } - long baseSeq = Unsafe.getUnsafe().getLong(addr + 8); - // FSNs are non-negative by construction (see SegmentRing). - // A negative baseSeq on disk means bit-rot or a malicious file — - // refuse the segment so SegmentRing.openExisting's narrow catch - // skips it like any other unreadable .sfa rather than feeding - // the bad value into Long.compareUnsigned-based contiguity - // checks (which would place the segment last in baseSeq order - // and trip the FSN-gap throw, taking the whole recovery down). - if (baseSeq < 0L) { + if (mappedSize != fileSize) { throw new MmapSegmentException( - "bad baseSeq in " + path + ": " + baseSeq); + "segment size changed while mapping: " + path + + " [before=" + fileSize + ", after=" + mappedSize + ']'); } - long lastGood = scanFrames(addr, fileSize); - long count = countFrames(addr, lastGood); - long tornTail = detectTornTail(addr, lastGood, fileSize); - if (tornTail > 0) { + if (scan.tornTailBytes > 0) { + // Observe-only: report the residue, never touch it here. A + // torn tail may be an interrupted append (dead bytes) OR a + // mid-file tear with unreachable valid-CRC frames past it -- + // the only surviving copy of real payloads. The + // destroy/preserve decision belongs to SegmentRing.recover. + // For sealed members chain validation PROVES the residue + // dead (complete frame accounting) before sanitizeTornTail + // runs, and a tear that cost frames fails closed with every + // byte left on disk. For the segment resumed as active no + // such proof exists (there is no successor to bound it): its + // residue is zeroed by policy once the rest of the chain + // validates, because replay cannot cross the tear and + // unzeroed residue risks reseal-brick and stale-frame + // resurrection. LOG.warn("SF segment {}: torn tail of {} bytes at offset {} " + "(file size {}, frames recovered {}). " - + "Recovery will overwrite this region on next append; " - + "frames past the tear (if any) are discarded. " + + "The residue is preserved pending chain validation. " + "Investigate disk health or unexpected writer crash.", - path, tornTail, lastGood, fileSize, count); + path, scan.tornTailBytes, scan.lastGood, fileSize, scan.frameCount); } - return new MmapSegment(path, fd, addr, fileSize, baseSeq, lastGood, count, false, tornTail); + return new MmapSegment( + ff, + path, + fd, + addr, + fileSize, + scan.baseSeq, + scan.lastGood, + scan.frameCount, + false, + scan.tornTailBytes + ); } catch (Throwable t) { if (addr != Files.FAILED_MMAP_ADDRESS) { - Files.munmap(addr, fileSize, MemoryTag.MMAP_DEFAULT); + ff.munmap(addr, fileSize, MemoryTag.MMAP_DEFAULT); } ff.close(fd); - // The header reads above (magic/version/baseSeq) run before - // scanFrames and are otherwise unguarded. An unbacked page 0 -- - // an unflushed header on a truncate-based-allocate filesystem - // after a crash (create() does not msync), or a file truncated - // under the mapping -- faults at the Unsafe intrinsic site as a - // catchable InternalError. Convert it to a MmapSegmentException so - // SegmentRing's per-file catch skips just this .sfa, instead of - // letting the raw InternalError escape to SegmentRing's outer catch - // and abort recovery of every sibling segment. scanFrames and - // detectTornTail already handle their own in-mapping faults; this - // covers the header block and any future reader placed ahead of the - // scan. - if (isMmapAccessFault(t)) { - throw new MmapSegmentException( - "unreadable mapped header page in " + path - + " (unbacked/sparse page 0): " + t.getMessage(), t); - } throw t; } } @@ -375,33 +442,147 @@ public void close() { if (memoryBacked) { Unsafe.free(mmapAddress, sizeBytes, MemoryTag.NATIVE_DEFAULT); } else { - Files.munmap(mmapAddress, sizeBytes, MemoryTag.MMAP_DEFAULT); + filesFacade.munmap(mmapAddress, sizeBytes, MemoryTag.MMAP_DEFAULT); } mmapAddress = 0; } if (fd >= 0) { - Files.close(fd); + filesFacade.close(fd); fd = -1; } } + boolean manifestRequired() { + return !memoryBacked && (Unsafe.getUnsafe().getByte(mmapAddress + 5) & MANIFEST_REQUIRED_FLAG) != 0; + } + + void markManifestRequired() { + if (memoryBacked || manifestRequired()) { + return; + } + Unsafe.getUnsafe().putByte(mmapAddress + 5, MANIFEST_REQUIRED_FLAG); + syncHeader(); + } + + void syncHeader() { + if (memoryBacked) { + return; + } + if (filesFacade.msync(mmapAddress, HEADER_SIZE, false) != 0 || filesFacade.fsync(fd) != 0) { + throw new MmapSegmentException("could not sync segment header " + path); + } + if (durableCursor < HEADER_SIZE) { + durableCursor = HEADER_SIZE; + } + } + public boolean isFull() { return capacityRemaining() <= 0; } + public boolean isPublishedDurable() { + return durableCursor >= publishedCursor; + } + /** - * Synchronously flushes dirty pages of {@code [HEADER_SIZE, publishedOffset())} - * to disk via {@code msync(MS_SYNC)}. Off the hot path — call only when - * the user has opted into OS-crash durability (e.g. {@code sf_msync_on_flush=on}). + * Preserves the original explicit mmap-flush behavior for callers that use + * this low-level API directly. Periodic durability uses + * {@link #syncPublished()}, which adds checked error handling and an fd + * barrier. */ public void msync() { - if (memoryBacked) return; // no on-disk pages to flush - long pub = publishedCursor; - if (pub > HEADER_SIZE) { - Files.msync(mmapAddress, pub, false); + if (memoryBacked) { + return; + } + long published = publishedCursor; + if (published > HEADER_SIZE) { + filesFacade.msync(mmapAddress, published, false); + } + } + + /** + * Synchronously flushes every complete frame published when this method + * captures {@link #publishedCursor}. A concurrent producer may publish + * more bytes while the barrier runs; those bytes remain outside the + * returned durable boundary until a later call. + *

    + * The not-yet-durable range is pinned with a best-effort {@code mlock} + * for the duration of the barrier and, on failure, re-dirtied before the + * pin is released. Rationale (fsyncgate): after a failed writeback the + * kernel marks the affected pages clean and reports the error once per + * open file description, so a naive retry would msync+fsync clean pages + * and return a vacuous 0 without persisting anything -- and an unpinned + * clean page can be reclaimed and later re-faulted from stale disk + * content, silently replacing the only good copy of the frames. The pin + * guarantees the failure arm re-dirties the genuine in-memory bytes; the + * re-dirty guarantees the next barrier performs real writeback and + * reports real errors, so the manager's unlatch-on-success stays honest. + * A refused mlock (RLIMIT_MEMLOCK, missing capability, stale native + * library) never affects the barrier outcome; it only widens the + * microseconds-scale window between the failed syscall and the re-dirty. + * + * @return the captured byte offset covered by the successful barrier + */ + public long syncPublished() { + long published = publishedCursor; + if (memoryBacked) { + durableCursor = published; + return published; + } + if (published <= durableCursor) { + return durableCursor; + } + long lockOffset = durableCursor & -Files.PAGE_SIZE; + long lockAddr = mmapAddress + lockOffset; + long lockLen = published - lockOffset; + boolean locked = filesFacade.mlock(lockAddr, lockLen) == 0; + if (!locked && MLOCK_REFUSAL_WARNED.compareAndSet(false, true)) { + LOG.warn("mlock refused for SF barrier range in {} (degrading to re-dirty-only retry protection); " + + "raise RLIMIT_MEMLOCK or grant CAP_IPC_LOCK to close the post-failure reclaim window", path); + } + boolean durable = false; + try { + if (filesFacade.msync(mmapAddress, published, false) != 0) { + throw new MmapSegmentException("could not sync segment data " + path); + } + // FlushViewOfFile alone is not power-loss durable on Windows. Keep the + // fd barrier on every platform so this method has one portable contract. + if (filesFacade.fsync(fd) != 0) { + throw new MmapSegmentException("could not sync segment file " + path); + } + durable = true; + durableCursor = published; + return published; + } finally { + if (!durable) { + redirtyRange(lockAddr, mmapAddress + published); + } + if (locked) { + filesFacade.munlock(lockAddr, lockLen); + } } } + /** + * Marks every page overlapping {@code [fromAddr, endAddr)} dirty again by + * re-storing one byte per page. Every touched offset lies inside the + * published prefix (or the immutable header magic after aligning down), + * so the same-value store races with nothing; kernel dirty tracking is + * value-blind, so the store is a real dirtying event that forces the next + * writeback to re-submit the whole page. + */ + private void redirtyRange(long fromAddr, long endAddr) { + for (long addr = fromAddr; addr < endAddr; addr += Files.PAGE_SIZE) { + Unsafe.getUnsafe().putByte(addr, Unsafe.getUnsafe().getByte(addr)); + } + redirtyPasses++; + } + + @TestOnly + public long redirtyPassesForTest() { + return redirtyPasses; + } + /** * Bytes safely written and visible to the consumer. Reading any byte at * offset {@code >= publishedOffset()} from the mapping is undefined — @@ -438,6 +619,21 @@ public long sizeBytes() { return sizeBytes; } + void linkSuccessor(MmapSegment next) { + if (next == null) { + throw new IllegalArgumentException("successor must not be null"); + } + MmapSegment existing = successor; + if (existing != null && existing != next) { + throw new IllegalStateException("segment successor already linked"); + } + successor = next; + } + + MmapSegment successor() { + return successor; + } + /** * Appends one frame: writes {@code [crc32c | u32 payloadLen | payload]} * starting at the current append cursor, then advances both cursors @@ -458,16 +654,17 @@ public long tryAppend(long payloadAddr, int payloadLen) { return -1L; } // CRC32C over the (payloadLen, payload) pair. Recovery scans validate - // each frame by recomputing this CRC over the on-disk bytes. + // each frame by recomputing this CRC over the on-disk bytes. The + // payloadLen field (4 bytes at lenAddr) and the payload (at lenAddr+4) + // are physically contiguous, so a single CRC pass covers both -- one + // native call per frame, byte-identical to the chained form the + // recovery scanner recomputes (see scanForRecovery). long lenAddr = mmapAddress + offset + 4; Unsafe.getUnsafe().putInt(lenAddr, payloadLen); if (payloadLen > 0) { Unsafe.getUnsafe().copyMemory(payloadAddr, mmapAddress + offset + FRAME_HEADER_SIZE, payloadLen); } - int crc = Crc32c.update(Crc32c.INIT, lenAddr, 4); - if (payloadLen > 0) { - crc = Crc32c.update(crc, mmapAddress + offset + FRAME_HEADER_SIZE, payloadLen); - } + int crc = Crc32c.update(Crc32c.INIT, lenAddr, 4L + payloadLen); Unsafe.getUnsafe().putInt(mmapAddress + offset, crc); appendCursor = offset + total; // Plain read + write of the volatile field. `frameCount++` would @@ -538,204 +735,286 @@ public long frameCount() { * recovery observes non-zero bytes past the bail-out point. {@code 0} for * fresh segments, memory-backed segments, and cleanly partially-filled * recovered segments. Operators / tests can read this to tell silent - * truncation (corruption) from a normal partial fill (no incident). - *

    - * One case this does NOT count: when the scan stops because the bail-out - * region is itself an unbacked mapped page (an in-mapping fault, not a - * backed torn header), that region cannot be probed, so this returns - * {@code 0} even though frames may have been discarded. That outcome is - * surfaced by the {@code WARN} in {@link #scanFrames} instead -- see it for - * the benign-tail-vs-media-error caveat. + * truncation (corruption) from a normal partial fill (no incident). This + * is the open-time observation and is never updated: a later + * {@link #sanitizeTornTail()} zeroes the on-disk residue (after which a + * re-open reports {@code 0}) but leaves this diagnostic intact. Sparse + * holes read back as zeroes; an actual positioned-read failure aborts + * recovery instead of being classified as a clean tail. */ public long tornTailBytes() { return tornTailBytes; } /** - * True when {@code t} is the JVM's recoverable signal for a fault while - * accessing a memory-mapped region -- a SIGBUS/SIGSEGV that HotSpot - * translates into an {@code InternalError} at an {@code Unsafe} intrinsic - * site instead of aborting the process. It surfaces when a mapped page is - * not backed by real file blocks: a sparse {@code .sfa} tail on a - * filesystem whose pre-allocation leaves holes (e.g. ZFS, where a - * truncate-based {@code allocate} does not materialize blocks), or a file - * truncated under the mapping. Recovery treats this as an I/O boundary -- - * the same way MappedByteBuffer readers do -- not a fatal VM error. + * Durably zeroes the torn-tail residue this segment was opened with: + * {@code [sizeBytes - tornTailBytes, sizeBytes)} is set to zero and made + * durable with an msync+fsync barrier. Restores the invariant every fresh + * segment already has -- all bytes past the append cursor are zero -- so + * residue that resumed appends do not fully overwrite cannot survive into + * a reseal (where sealed-suffix recovery would treat it as fatal + * corruption on every subsequent startup) and a byte-aligned stale frame + * with a valid CRC cannot be resurrected at a recycled FSN by a later + * scan (the frame envelope binds neither position nor FSN). *

    - * The message is matched on the fragment {@code "unsafe memory access - * operation"}, which is common to both HotSpot wordings and NOT - * version-stable as a whole: pre-21 JDKs (including the shipping/CI JDK 8) - * emit {@code "a fault occurred in a recent unsafe memory access operation - * in compiled Java code"}, while JDK 21+ shortened it to {@code "a fault - * occurred in an unsafe memory access operation"}. Matching the shared - * fragment keeps the guard effective on JDK 8/11/17 as well as 21+, while - * still being specific enough that a genuine VirtualMachineError (real OOM, - * StackOverflow) is never swallowed. + * DESTRUCTIVE by design and therefore never called by + * {@link #openExisting}: after a mid-file tear the residue can hold + * unreachable valid-CRC frames that are the only surviving copy of + * unacked payloads. The caller ({@link SegmentRing} recovery) must first + * establish the residue is not load-bearing, and the strength of that + * claim differs by role: a sealed suffix is PROVEN dead (frame + * accounting validated complete against the chain), while the resumed + * active's tail past a mid-file tear is discarded by POLICY, not proof + * -- it can hold valid-CRC frames of real unacked payloads that are + * unreachable by replay (the FSN sequence breaks at the tear), and + * leaving them risks the two hazards above. Must run before any append + * resumes; appending first would put live frames inside the zeroed range, + * so that ordering is rejected. Idempotent; no-op when no residue was + * observed. The fsync is load-bearing: in MEMORY durability mode rotation + * does not sync the sealed predecessor's data pages, so an unflushed + * zero-write plus a crash would regenerate the exact poisoned state this + * sanitization removes. A failed barrier throws and the caller fails + * closed; the zeroes may still reach disk via the page cache, which is + * safe: a retry either observes a clean tail or re-zeroes. */ - private static boolean isMmapAccessFault(Throwable t) { - if (!(t instanceof InternalError)) { - return false; + public void sanitizeTornTail() { + if (tornTailBytes == 0 || tornTailSanitized) { + return; + } + long residueStart = sizeBytes - tornTailBytes; + if (appendCursor != residueStart) { + throw new MmapSegmentException( + "torn tail must be sanitized before appends resume in " + path + + " [appendCursor=" + appendCursor + + ", residueStart=" + residueStart + ']'); } - String msg = t.getMessage(); - return msg != null && msg.contains("unsafe memory access operation"); + Unsafe.getUnsafe().setMemory(mmapAddress + residueStart, tornTailBytes, (byte) 0); + if (filesFacade.msync(mmapAddress, sizeBytes, false) != 0 || filesFacade.fsync(fd) != 0) { + throw new MmapSegmentException( + "could not sync zeroed torn tail in " + path + + " [errno=" + Os.errno() + ']'); + } + tornTailSanitized = true; + LOG.warn("SF segment {}: zeroed {} bytes of torn-tail residue at offset {}; " + + "frames past the tear (if any) are discarded", + path, tornTailBytes, residueStart); } /** - * Forward scan that returns the offset just past the last frame whose - * CRC verifies. A torn-tail frame (declared length runs past EOF, or - * CRC mismatch) leaves both cursors at the start of that frame; the - * next {@link #tryAppend} will overwrite it. The scan only reads from - * the mapping — no syscalls. + * True when recovery observed torn-tail residue that + * {@link #sanitizeTornTail()} has not yet durably zeroed -- i.e. the + * on-disk suffix still carries the observed bytes. */ - private static long scanFrames(long addr, long fileSize) { - long pos = HEADER_SIZE; - try { - while (pos + FRAME_HEADER_SIZE <= fileSize) { - int crcRead = Unsafe.getUnsafe().getInt(addr + pos); - int payloadLen = Unsafe.getUnsafe().getInt(addr + pos + 4); - // Defensive: a corrupt length field could be enormous or negative, - // both of which would otherwise overrun the mapping. - if (payloadLen < 0 || pos + FRAME_HEADER_SIZE + payloadLen > fileSize) { - return pos; + public boolean hasUnsanitizedTornTail() { + return tornTailBytes > 0 && !tornTailSanitized; + } + + /** + * Scans the header and frames through a bounded positioned-read buffer. + * Sparse file holes are returned by {@code pread}/{@code ReadFile} as zero + * bytes, while media errors and concurrent truncation surface as ordinary + * read failures. No recovery byte is dereferenced through mmap. + */ + private static RecoveryScan scanForRecovery( + FilesFacade ff, + int fd, + String path, + long fileSize + ) { + try (RecoveryReader reader = new RecoveryReader(ff, fd, path, fileSize)) { + int magic = reader.getInt(0L); + if (magic != FILE_MAGIC) { + throw new MmapSegmentCorruptionException( + "bad magic in " + path + ": 0x" + Integer.toHexString(magic)); + } + byte version = reader.getByte(4L); + if (version != VERSION) { + // Deliberately NOT the corruption subtype: an unsupported + // version is a well-formed segment written by a different + // client build (e.g. after a downgrade). Quarantine-renaming + // it would strand its frames for the writer that CAN read it; + // failing recovery keeps the slot intact for that writer. + throw new MmapSegmentException("unsupported version in " + path + ": " + version); + } + long baseSeq = reader.getLong(8L); + // FSNs are non-negative by construction (see SegmentRing). A + // negative value on disk is positively identified corruption. + if (baseSeq < 0L) { + throw new MmapSegmentCorruptionException( + "bad baseSeq in " + path + ": " + baseSeq); + } + + long frameCount = 0L; + long pos = HEADER_SIZE; + while (fileSize - pos >= FRAME_HEADER_SIZE) { + int crcRead = reader.getInt(pos); + int payloadLen = reader.getInt(pos + 4L); + // Avoid addition overflow while rejecting a negative or + // beyond-EOF frame length as a torn tail. + if (payloadLen < 0 || payloadLen > fileSize - pos - FRAME_HEADER_SIZE) { + break; } - // CRC over the contiguous (payloadLen, payload) pair, folded - // via Unsafe reads rather than the native Crc32c.update. - // Recovery maps to the file's stat length, but a page inside - // that range can be unbacked: a sparse pre-allocation tail (a - // truncate-based allocate that never materialized blocks, as on - // ZFS), or -- via a torn write, since tryAppend writes the - // length field before copying the payload -- a real positive - // payloadLen whose payload spans into an unwritten hole. A raw - // read of an unbacked page raises SIGBUS; HotSpot translates - // that into a catchable InternalError ONLY at an Unsafe - // intrinsic site, NEVER inside JNI native code, so a native - // Crc32c.update over such a page aborts the whole JVM (and, - // empirically on pre-21 JDKs, a preceding Unsafe pre-touch does - // not reliably fault first once an earlier native CRC ran in - // the same scan). Folding over Unsafe keeps every fault - // catchable -- handled below as the boundary of recoverable - // data; a page that instead reads back as zeroes just fails the - // CRC check and ends the scan. Recovery is cold, so the slower - // table CRC here is immaterial. - int crcCalc = crc32cRecovery(addr + pos + 4, 4L + payloadLen); + int crcCalc = reader.crc32c(pos + 4L, 4L + payloadLen); if (crcCalc != crcRead) { - return pos; + break; } pos += FRAME_HEADER_SIZE + payloadLen; + frameCount++; } - } catch (InternalError e) { - // The read at `pos` hit a mapped page that is not backed by real - // file blocks: the JVM translates the underlying SIGBUS into a - // recoverable InternalError instead of aborting the process. This - // happens when a prior session left a sparse segment tail (a - // truncate-based pre-allocation that does not materialize blocks, - // as on ZFS) or the file was truncated under the mapping. Every - // frame below `pos` already verified; treat the unreadable region - // exactly like unwritten space or a torn tail -- the boundary of - // recoverable data -- rather than letting the error abort recovery - // of the whole slot. Anything that is not the documented mmap - // access fault is a genuine VM error, so rethrow it. - if (!isMmapAccessFault(e)) { - throw e; - } - LOG.warn("SF segment recovery: unreadable mapped page at offset {} (file size {}); " - + "treating it as the end of recoverable data -- any frames beyond this " - + "offset are discarded. The usual cause is a benign sparse pre-allocation " - + "tail (e.g. a truncate-based allocate on ZFS) left by a prior session, " - + "but a mid-file media error (bad sector) is indistinguishable here; " - + "check disk health if this segment was expected to be fully written or " - + "if this recurs.", - pos, fileSize); + long tornTailBytes = detectTornTail(reader, pos, fileSize); + return new RecoveryScan(baseSeq, frameCount, pos, tornTailBytes); } - return pos; } /** - * CRC-32C (Castagnoli) of {@code [addr, addr + len)} read through - * {@link Unsafe}, seeded like {@code Crc32c.update(Crc32c.INIT, addr, len)} - * and bit-identical to it (verified) -- but every byte load is an Unsafe - * intrinsic, so a fault on an unbacked mapped page is a catchable - * {@link InternalError} instead of the uncatchable JNI SIGBUS the native - * {@link Crc32c} would raise. Byte-at-a-time via a precomputed table - * ({@code ~0.5 GiB/s}); used only on the cold recovery scan, never on the - * append hot path (which stays on the native, hardware-friendly path). + * Distinguishes attempted frame data from clean zero-filled space after the + * recovery boundary. The entire suffix is inspected: an all-zero corrupt + * frame header must not hide a non-zero payload or later frames and make a + * data-bearing segment look like a reusable empty spare. Positioned-read + * failures are intentionally not swallowed; they are operational failures, + * not proof of an unwritten tail. */ - private static int crc32cRecovery(long addr, long len) { - int crc = ~Crc32c.INIT; - for (long i = 0; i < len; i++) { - crc = (crc >>> 8) ^ CRC32C_TABLE[(crc ^ Unsafe.getUnsafe().getByte(addr + i)) & 0xFF]; + private static long detectTornTail(RecoveryReader reader, long lastGood, long fileSize) { + if (lastGood >= fileSize) { + return 0L; } - return ~crc; + return reader.hasNonZero(lastGood, fileSize - lastGood) + ? fileSize - lastGood + : 0L; } - /** - * Standard reflected CRC-32C byte table (polynomial {@code 0x82F63B78}), - * matching {@code crc32c_table[0]} in the native {@code crc32c.c}. Computed - * at class init to avoid 256 hand-transcribed literals; drives - * {@link #crc32cRecovery}. - */ - private static int[] buildCrc32cTable() { - int[] table = new int[256]; - for (int n = 0; n < 256; n++) { - int c = n; - for (int k = 0; k < 8; k++) { - c = (c & 1) != 0 ? (0x82F63B78 ^ (c >>> 1)) : (c >>> 1); + private static final class RecoveryReader implements QuietCloseable { + private final long bufferAddress; + private final int fd; + private final FilesFacade filesFacade; + private final long fileSize; + private final String path; + private int bufferLength; + private long bufferOffset = -1L; + + private RecoveryReader(FilesFacade filesFacade, int fd, String path, long fileSize) { + this.filesFacade = filesFacade; + this.fd = fd; + this.path = path; + this.fileSize = fileSize; + this.bufferAddress = Unsafe.malloc(RECOVERY_BUFFER_SIZE, MemoryTag.NATIVE_DEFAULT); + } + + @Override + public void close() { + Unsafe.free(bufferAddress, RECOVERY_BUFFER_SIZE, MemoryTag.NATIVE_DEFAULT); + } + + private int crc32c(long offset, long len) { + int crc = Crc32c.INIT; + while (len > 0L) { + ensure(offset, 1); + int bufferIndex = (int) (offset - bufferOffset); + long chunk = Math.min(len, bufferLength - bufferIndex); + crc = Crc32c.update(crc, bufferAddress + bufferIndex, chunk); + offset += chunk; + len -= chunk; } - table[n] = c; + return crc; } - return table; - } - /** - * Distinguishes "torn tail" (writer attempted a write past the last valid - * frame and failed — partial write, mid-stream corruption, bit rot) from - * clean unwritten space (manager-allocated segment with zero-filled tail). - * Returns the byte count from {@code lastGood} to {@code fileSize} when - * the bytes at the bail-out frame header are non-zero, else {@code 0}. - *

    - * Heuristic but robust for the common cases: {@link #create} truncates the - * file to size, leaving the tail zero-filled; the writer only writes - * non-zero bytes via {@link #tryAppend}, which writes the CRC and length - * fields together. So a non-zero byte at the failed-frame position - * implies an attempted write — exactly the case operators want flagged. - */ - private static long detectTornTail(long addr, long lastGood, long fileSize) { - if (lastGood >= fileSize) { - return 0L; + private void ensure(long offset, int requiredBytes) { + if (offset >= bufferOffset + && offset - bufferOffset <= bufferLength - requiredBytes) { + return; + } + if (offset < 0L || requiredBytes < 0 || offset > fileSize - requiredBytes) { + throw new MmapSegmentException( + "recovery read outside segment " + path + + " [offset=" + offset + ", required=" + requiredBytes + + ", fileSize=" + fileSize + ']'); + } + int len = (int) Math.min((long) RECOVERY_BUFFER_SIZE, fileSize - offset); + readFully(bufferAddress, len, offset); + bufferOffset = offset; + bufferLength = len; } - long probe = Math.min(FRAME_HEADER_SIZE, fileSize - lastGood); - try { - for (long i = 0; i < probe; i++) { - if (Unsafe.getUnsafe().getByte(addr + lastGood + i) != 0) { - return fileSize - lastGood; + + private byte getByte(long offset) { + ensure(offset, 1); + return Unsafe.getUnsafe().getByte(bufferAddress + offset - bufferOffset); + } + + private int getInt(long offset) { + ensure(offset, Integer.BYTES); + return Unsafe.getUnsafe().getInt(bufferAddress + offset - bufferOffset); + } + + private long getLong(long offset) { + ensure(offset, Long.BYTES); + return Unsafe.getUnsafe().getLong(bufferAddress + offset - bufferOffset); + } + + private boolean hasNonZero(long offset, long len) { + boolean nonZero = false; + while (len > 0L) { + ensure(offset, 1); + int bufferIndex = (int) (offset - bufferOffset); + int chunk = (int) Math.min(len, bufferLength - bufferIndex); + long address = bufferAddress + bufferIndex; + int i = 0; + while (i < chunk && ((address + i) & (Long.BYTES - 1L)) != 0L) { + nonZero |= Unsafe.getUnsafe().getByte(address + i++) != 0; } + while (i <= chunk - Long.BYTES) { + nonZero |= Unsafe.getUnsafe().getLong(address + i) != 0L; + i += Long.BYTES; + } + while (i < chunk) { + nonZero |= Unsafe.getUnsafe().getByte(address + i++) != 0; + } + offset += chunk; + len -= chunk; } - } catch (InternalError e) { - // The bail-out region is an unbacked (sparse) mapped page -- see - // scanFrames for the mechanism. An unbacked hole was never written, - // so it is clean unwritten space, not a torn write. Rethrow any - // error that is not the recoverable mmap access fault. - if (!isMmapAccessFault(e)) { - throw e; + return nonZero; + } + + private void readFully(long address, long len, long offset) { + long read = 0L; + while (read < len) { + long n = filesFacade.read(fd, address + read, len - read, offset + read); + if (n < 0L) { + throw new MmapSegmentException( + "could not read SF segment " + path + + " [offset=" + (offset + read) + + ", remaining=" + (len - read) + + ", errno=" + Os.errno() + ']'); + } + if (n == 0L) { + throw new MmapSegmentException( + "short read while recovering SF segment " + path + + " [offset=" + (offset + read) + + ", remaining=" + (len - read) + + ", fileSize=" + fileSize + ']'); + } + if (n > len - read) { + throw new MmapSegmentException( + "invalid read length while recovering SF segment " + path + + " [offset=" + (offset + read) + + ", requested=" + (len - read) + + ", actual=" + n + ']'); + } + read += n; } - return 0L; } - return 0L; } - /** - * Counts frames in {@code [HEADER_SIZE, lastGood)}. Walks the framing in - * lockstep with {@link #scanFrames} (which already validated CRCs); so - * this is just length-driven traversal, no CRC re-check. - */ - private static long countFrames(long addr, long lastGood) { - long pos = HEADER_SIZE; - long count = 0; - while (pos < lastGood) { - int payloadLen = Unsafe.getUnsafe().getInt(addr + pos + 4); - pos += FRAME_HEADER_SIZE + payloadLen; - count++; - } - return count; + private static final class RecoveryScan { + private final long baseSeq; + private final long frameCount; + private final long lastGood; + private final long tornTailBytes; + + private RecoveryScan(long baseSeq, long frameCount, long lastGood, long tornTailBytes) { + this.baseSeq = baseSeq; + this.frameCount = frameCount; + this.lastGood = lastGood; + this.tornTailBytes = tornTailBytes; + } } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegmentCorruptionException.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegmentCorruptionException.java new file mode 100644 index 00000000..8a72a026 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegmentCorruptionException.java @@ -0,0 +1,50 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client.sf.cursor; + +/** + * Positively-identified segment corruption: the file's own bytes prove it is + * not (or no longer) a readable SF segment — truncated below the fixed header, + * wrong magic, or a negative {@code baseSeq}. Distinct from its parent + * {@link MmapSegmentException}, which recovery treats as an operational + * failure (open/read/mmap error on a file + * whose contents may be perfectly intact) and must therefore be fatal. + *

    + * Recovery quarantines corruption (rename to {@code .corrupt}) and + * relies on manifest boundaries / FSN contiguity to decide whether the + * quarantined file was load-bearing; operational failures always abort + * startup so a transient {@code EMFILE}/{@code ENOMEM} can never silently + * drop durable frames. + */ +public final class MmapSegmentCorruptionException extends MmapSegmentException { + + public MmapSegmentCorruptionException(String message) { + super(message); + } + + public MmapSegmentCorruptionException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegmentException.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegmentException.java index eec0c0d9..371d980c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegmentException.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegmentException.java @@ -28,9 +28,19 @@ * Hard failure of the MmapSegment layer — bad header, mmap rejection, file * too short for header, etc. Indicates the segment is unusable, not that * the disk is full (the latter surfaces as backpressure on the producer - * via {@link io.questdb.client.cutlass.qwp.client.LineSenderException}). + * via {@link io.questdb.client.cutlass.line.LineSenderException}). + *

    + * Recovery distinguishes three flavors: this base type marks operational + * failures (open/read/mmap/enumeration errors — the file's contents may be fine, + * so recovery must fail closed); {@link MmapSegmentCorruptionException} marks + * positively-identified corruption in one file's own bytes, which + * recovery may quarantine when the surviving chain proves safe; and + * {@link SfRecoveryException} marks a terminal chain failure that needs + * operator intervention (with one self-healed refinement, + * {@link SfSanitizedResidueException}, that unattended callers retry once + * instead of quarantining). */ -public final class MmapSegmentException extends RuntimeException { +public class MmapSegmentException extends RuntimeException { public MmapSegmentException(String message) { super(message); } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java index 3b7cff60..e4a23090 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentManager.java @@ -34,8 +34,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; import java.util.concurrent.locks.LockSupport; +import java.util.function.LongSupplier; /** * Background worker that keeps every registered {@link SegmentRing} supplied @@ -49,27 +52,32 @@ * in a JVM). Polls each ring on a configurable tick (default 1 ms) — short * enough that a producer rarely sees {@link SegmentRing#BACKPRESSURE_NO_SPARE} * in the steady state, long enough that an idle JVM doesn't burn CPU. - *

    - * baseSeq race window: the spare is created with - * {@code baseSeq = ring.nextSeqHint()} as observed by the manager. If the - * producer thread appends more frames before the rotation actually fires, - * the spare's baseSeq will be stale and {@link SegmentRing#appendOrFsn} will - * throw on the mismatch check. In practice this is benign — by the time - * {@link SegmentRing#needsHotSpare()} returns true the producer has very - * little room left in the active segment, and the manager polls fast enough - * to install before the producer fills the rest. Hardening to make the race - * impossible (lazy header write at rotation time) is a separate refinement - * deferred to PR2. */ public final class SegmentManager implements QuietCloseable { public static final long DEFAULT_POLL_NANOS = 1_000_000L; // 1 ms public static final long DISK_FULL_LOG_THROTTLE_NANOS = 30_000_000_000L; // 30 s public static final long UNLIMITED_TOTAL_BYTES = Long.MAX_VALUE; + private static final int ENTRY_DEREGISTERED = 3; + private static final int ENTRY_DEREGISTERED_IN_SERVICE = 2; + private static final int ENTRY_IN_SERVICE = 1; + private static final int ENTRY_REGISTERED = 0; + private static final AtomicReferenceFieldUpdater ENTRY_CLEANUP_UPDATER = + AtomicReferenceFieldUpdater.newUpdater(RingEntry.class, Runnable.class, "quiescenceCleanup"); + private static final AtomicIntegerFieldUpdater ENTRY_STATE_UPDATER = + AtomicIntegerFieldUpdater.newUpdater(RingEntry.class, "state"); private static final Logger LOG = LoggerFactory.getLogger(SegmentManager.class); + private static final int MAX_TRIMS_PER_RING_PASS = 64; + private static final long TRIM_RETRY_INITIAL_NANOS = 4_000_000L; + private static final long TRIM_RETRY_MAX_NANOS = 1_024_000_000L; + private static final int TRIM_RETRY_NONE = 0; + private static final int TRIM_RETRY_POST_BARRIER = 3; + private static final int TRIM_RETRY_PRE_BARRIER = 1; + private static final int TRIM_RETRY_UNLINK = 2; private static final long WORKER_JOIN_TIMEOUT_MILLIS = 5_000L; private final AtomicLong fileGeneration = new AtomicLong(); + private final FilesFacade filesFacade; private final Object lock = new Object(); private final long maxTotalBytes; // Reused by the manager worker thread to build spare-segment paths @@ -86,6 +94,25 @@ public final class SegmentManager implements QuietCloseable { private final ObjList ringSnapshot = new ObjList<>(); private final ObjList rings = new ObjList<>(); private final long segmentSizeBytes; + // Reused by the manager worker while it checkpoints one ring. The entry + // in-service state keeps these segment mappings alive until the pass ends. + private final ObjList syncScratch = new ObjList<>(); + private final LongSupplier ticks; + // Reused by the worker for one bounded trim quantum. Keeping the unlinked + // prefix here until the post-unlink directory barrier succeeds avoids both + // per-pass allocation and publishing logical removal before it is durable. + private final MmapSegment[] trimBatch = new MmapSegment[MAX_TRIMS_PER_RING_PASS]; + // Test seam: runs after a deferred ring-pass cleanup returns. Null in + // production; public sender/pool lifecycle tests use it to observe exact + // callback completion without sleeps or polling. + private volatile Runnable afterRingCleanupHook; + // Test seam: runs at the top of deferUntilWorkerExit, before the + // worker-liveness check. Null in production; registration-failure tests + // throw from it to simulate an allocation failure (OOM building the + // cleanup lambda or growing exitCleanups) while the worker is still + // live. Callers must treat such a throw as "worker state unknown", + // never as the exact false return meaning the worker loop has exited. + private volatile Runnable beforeExitCleanupRegistrationHook; // Test seam: runs on the worker thread just before the install path's // synchronized(lock) entry (the one that performs installHotSpare + the // totalBytes += segmentSize commit). Null in production; tests use it to @@ -93,33 +120,75 @@ public final class SegmentManager implements QuietCloseable { // but before ownership/accounting commit. Callers may inject a deregister // or hold this stale worker snapshot while caller-side cleanup runs. private volatile Runnable beforeInstallSyncHook; + // Test seam: records entry into the per-ring quiescence wait. Null in + // production; owned-engine close tests use it to prove they take only the + // stronger whole-manager join path, not two sequential timeout budgets. + private volatile Runnable beforeRingQuiescenceAwaitHook; // Test seam: runs on the worker thread just before the trim block's // synchronized(lock) entry. Null in production; only // SegmentManagerTrimDeregisterRaceTest installs it, to deterministically // inject a deregister(ring) call into the exact race window that the - // registered flag check inside the trim block closes for watermark writes - // and totalBytes accounting. + // entry-state check inside the trim block closes for watermark writes and + // totalBytes accounting. private volatile Runnable beforeTrimSyncHook; + // Test seam invoked exactly when a retry transition/recovery is logged. + // Null in production; persistent-failure tests use it to prove log bounds + // without binding to a particular SLF4J backend. + private volatile Runnable retryLogHook; + // Entry the worker is claiming or servicing, or null between passes. + // Volatile publication lets teardown find the entry after deregister() + // removes it from rings. RingEntry.state is the authoritative barrier: + // the worker publishes this reference before its REGISTERED->IN_SERVICE + // CAS and only touches ring resources after that CAS succeeds. + private volatile RingEntry inService; + // Cleanup actions handed to the worker's exit block by an owning engine + // whose close() found the worker still mid service pass after the bounded + // join timed out (see deferUntilWorkerExit). Guarded by {@link #lock}; + // consumed exactly once by the worker-loop finally, which runs them + // OUTSIDE `lock`: they perform syscalls (munmap/unlink/flock release), + // and no caller-facing lock may ever be held while running third-party + // cleanup code. + private ObjList exitCleanups; + // A private manager belongs to exactly one CursorSendEngine. Its callback + // is preallocated by that engine and stored directly here, so the critical + // timed-out-close handoff never allocates an ObjList or grows a backing + // array. Guarded by lock and consumed on worker-loop exit outside lock. + private Runnable ownedEngineExitCleanup; private long lastDiskFullLogNs; private volatile boolean running; + // pathScratch free-exactly-once coordination between a timed-out close() + // and the worker's exit path. All three are guarded by {@link #lock}. + // When close() gives up on the join while the worker loop has not yet + // exited, it hands scratch ownership to the worker + // (scratchHandedToWorker=true) and the worker frees the buffer in its + // exit block; in every other case close() frees it. Without the handoff, + // a worker that outlives the bounded join leaks the native scratch + // buffer forever, because nobody retries manager cleanup after close() + // returns. + private boolean scratchFreed; + private boolean scratchHandedToWorker; + private volatile long shortestSyncIntervalNanos = Long.MAX_VALUE; + private boolean workerLoopExited; // Total bytes currently allocated across every segment owned by every // registered ring (active + sealed + hot-spare). Mutated by the manager // thread on provision/trim and by register/deregister callers under // {@link #lock}; the lock covers both paths so the counter stays // consistent across registration boundaries. private long totalBytes; - private long workerJoinTimeoutMillis = WORKER_JOIN_TIMEOUT_MILLIS; + // volatile: read by awaitRingQuiescence() from arbitrary caller threads + // while the @TestOnly setter may run on another. + private volatile long workerJoinTimeoutMillis = WORKER_JOIN_TIMEOUT_MILLIS; // volatile because wakeWorker() reads workerThread without holding the // monitor; the synchronized start()/close() pair handles the // start-vs-close ordering. private volatile Thread workerThread; public SegmentManager(long segmentSizeBytes) { - this(segmentSizeBytes, DEFAULT_POLL_NANOS, UNLIMITED_TOTAL_BYTES); + this(segmentSizeBytes, DEFAULT_POLL_NANOS, UNLIMITED_TOTAL_BYTES, FilesFacade.INSTANCE, System::nanoTime); } public SegmentManager(long segmentSizeBytes, long pollNanos) { - this(segmentSizeBytes, pollNanos, UNLIMITED_TOTAL_BYTES); + this(segmentSizeBytes, pollNanos, UNLIMITED_TOTAL_BYTES, FilesFacade.INSTANCE, System::nanoTime); } /** @@ -145,6 +214,22 @@ public SegmentManager(long segmentSizeBytes, long pollNanos) { * hold an initial active plus one hot spare. */ public SegmentManager(long segmentSizeBytes, long pollNanos, long maxTotalBytes) { + this(segmentSizeBytes, pollNanos, maxTotalBytes, FilesFacade.INSTANCE, System::nanoTime); + } + + @TestOnly + public SegmentManager(long segmentSizeBytes, long pollNanos, long maxTotalBytes, FilesFacade filesFacade) { + this(segmentSizeBytes, pollNanos, maxTotalBytes, filesFacade, System::nanoTime); + } + + @TestOnly + public SegmentManager( + long segmentSizeBytes, + long pollNanos, + long maxTotalBytes, + FilesFacade filesFacade, + LongSupplier ticks + ) { // The pathScratch field initializer has already allocated its native // buffer by the time this body runs, so a validation throw must free // it or every failed construction leaks 256 bytes of native memory @@ -159,9 +244,15 @@ public SegmentManager(long segmentSizeBytes, long pollNanos, long maxTotalBytes) "maxTotalBytes (" + maxTotalBytes + ") must allow at least one segment of " + segmentSizeBytes + " bytes"); } + this.filesFacade = filesFacade; this.segmentSizeBytes = segmentSizeBytes; this.pollNanos = pollNanos; this.maxTotalBytes = maxTotalBytes; + this.ticks = ticks; + } + + FilesFacade filesFacade() { + return filesFacade; } @Override @@ -194,18 +285,251 @@ public synchronized void close() { } } if (t.isAlive()) { - LOG.warn("SegmentManager worker did not stop before close wait completed; " - + "leaving worker-owned resources allocated"); - return; + synchronized (lock) { + if (!workerLoopExited) { + // Hand pathScratch ownership to the worker: its exit + // block frees the buffer under the same lock, so the + // native allocation is reclaimed even though this + // close() could not confirm termination. workerThread + // stays set so isWorkerReaped() reports the incomplete + // shutdown and a later close() can retry the join. + scratchHandedToWorker = true; + LOG.warn("SegmentManager worker did not stop before close wait completed; " + + "worker frees its native scratch buffer on exit"); + return; + } + } + // The worker has left its service loop (workerLoopExited) and + // is now running only its finite exit cleanups -- the owning + // engine's finishClose(), which releases the slot flock and + // only THEN publishes closeCompleted. It can no longer wedge in + // a service pass, so give it a second bounded join to finish + // before reaping. Without this, a bounded-join timeout that + // lands mid-finishClose reaps the worker (workerThread=null => + // isWorkerReaped()) while the flock release is still in flight, + // so a caller reading isCloseCompleted() observes a stale false + // and a spurious flock-release retry gets scheduled. This reuses + // the same join-under-monitor pattern as the bounded join above + // -- the worker uses a lock-free CAS for exactly-once cleanup + // and never blocks on this monitor -- and still reaps on + // timeout, so a pathologically slow cleanup cannot hang close(). + // + // Budget with the fixed WORKER_JOIN_TIMEOUT_MILLIS, NOT the + // tunable workerJoinTimeoutMillis: the first join bounds a + // possibly-wedged SERVICE pass (tests shrink it to force the + // timed-out path), but the exit cleanups are finite and must be + // allowed to finish regardless of that tuning. In production + // both values are equal, so this only matters under a shrunk + // test override. + boolean cleanupInterrupted = Thread.interrupted(); + long cleanupDeadlineNanos = System.nanoTime() + WORKER_JOIN_TIMEOUT_MILLIS * 1_000_000L; + try { + while (t.isAlive()) { + long remainingMillis = (cleanupDeadlineNanos - System.nanoTime()) / 1_000_000L; + if (remainingMillis <= 0) { + break; + } + try { + t.join(remainingMillis); + } catch (InterruptedException ignored) { + cleanupInterrupted = true; + } + } + } finally { + if (cleanupInterrupted) { + Thread.currentThread().interrupt(); + } + } + // If the cleanups still have not finished (pathologically slow + // syscall), fall through and reap anyway -- identical to the + // best-effort behaviour before this second join existed. } workerThread = null; } // Free the rotation-path native scratch buffer only after worker - // termination has been observed. The worker is the only thread that - // touches the buffer, but close() uses a bounded join; if the worker is - // still alive, leaking this one scratch allocation is safer than freeing - // native memory it may still read or write. - pathScratch.close(); + // termination (or worker-loop exit) has been observed. The worker is + // the only thread that touches the buffer; the scratchFreed flag + // (shared with the worker's exit block) makes the free exactly-once + // no matter which side runs last. + synchronized (lock) { + if (!scratchFreed) { + scratchFreed = true; + pathScratch.close(); + } + } + } + + /** + * Hands the single owning engine's preallocated close callback to the + * worker exit block. Registration only assigns a reference under + * {@link #lock}; it cannot allocate in the timed-out teardown path. + * Returns {@code false} only after the worker loop has exited (or when it + * never started), so the caller may then clean up inline safely. + */ + public boolean deferOwnedEngineCloseUntilWorkerExit(Runnable cleanup) { + synchronized (lock) { + if (workerLoopExited || workerThread == null) { + return false; + } + if (ownedEngineExitCleanup != null && ownedEngineExitCleanup != cleanup) { + throw new IllegalStateException("owned manager already has an engine-exit cleanup"); + } + ownedEngineExitCleanup = cleanup; + return true; + } + } + + /** + * Hands a cleanup action to the current service pass for {@code ring}. + * The worker runs it outside the manager lock immediately after that pass + * finishes. Returns {@code false} when no pass for the ring remains in + * flight, in which case the caller already owns a quiescent ring and must + * run the cleanup itself. + *

    + * Registration and pass completion coordinate through the entry's atomic + * state and callback fields, so there is no gap without an owner: either + * this method attaches the cleanup while the pass remains active, the + * worker claims an attached cleanup while completing, or this method + * observes completed state and rejects the handoff. A repeated registration + * for the same pass returns {@code true} without replacing its existing + * owner. + */ + public boolean deferUntilRingQuiescent(SegmentRing ring, Runnable cleanup) { + RingEntry e = inService; + if (e == null || e.ring != ring || !e.isInService()) { + return false; + } + Runnable existing = ENTRY_CLEANUP_UPDATER.get(e); + if (existing == null && ENTRY_CLEANUP_UPDATER.compareAndSet(e, null, cleanup)) { + if (e.isInService()) { + return true; + } + // Completion changed the state before observing the callback. + // Remove our callback and clean inline, unless the worker already + // claimed it (in which case that worker remains the owner). + return !ENTRY_CLEANUP_UPDATER.compareAndSet(e, cleanup, null); + } + // A callback already attached to this pass is an owner. The sole + // production caller always supplies the engine's preallocated bound + // callback; duplicates intentionally do not replace it. + return true; + } + + /** + * Hands a cleanup action to the worker thread's exit block, to run + * strictly after the worker loop has finished its final service pass -- + * i.e. after the last point where the worker can create, write or unlink + * anything under a registered ring's slot directory. Returns {@code false} + * when the worker loop has already exited (or the worker never started): + * the caller must run the cleanup itself, which is equally safe for the + * same reason -- no further worker access to the slot is possible. + *

    + * This is the slot-ownership transfer used by an owning engine's close() + * when the bounded worker join timed out: instead of retiring the slot + * until process exit, ring/watermark/flock release moves to the worker, + * which is provably the last thread able to touch the slot directory. + * The registration here and the exit block's {@code workerLoopExited} + * flip share {@link #lock}, so the cleanup runs exactly once: either it + * is registered before the flip and the worker runs it, or the flip won + * and this method rejects the handoff. + *

    + * May throw on allocation failure (the lambda at the call site, the + * {@code ObjList}, or its growth). A throw carries NO liveness + * information: the worker was never observed, so the caller must treat + * it as "worker possibly still live" and retain resources — never as + * the exact {@code false} return above. + */ + public boolean deferUntilWorkerExit(Runnable cleanup) { + Runnable hook = beforeExitCleanupRegistrationHook; + if (hook != null) { + hook.run(); + } + synchronized (lock) { + if (workerLoopExited || workerThread == null) { + return false; + } + if (exitCleanups == null) { + exitCleanups = new ObjList<>(); + } + exitCleanups.add(cleanup); + return true; + } + } + + /** + * Quiescence barrier for {@link #deregister(SegmentRing)}. Blocks until + * the worker thread is provably no longer executing a service pass for + * {@code ring}, or the worker-join timeout elapses. After this returns + * {@code true}, the worker will never again touch the ring, its + * watermark, or path names under its slot directory: deregister has + * removed the entry from the registry, a stale snapshot entry that has + * not started its pass is skipped by the registration check at the top + * of {@link #serviceRing(RingEntry)}, and this method has observed the + * end of any in-flight pass. Only then may the caller release dependent + * resources (ring, watermark, segment files, slot lock). + *

    + * Returns {@code true} immediately when no worker is running or when + * called from the worker thread itself (test hooks inject deregister + * calls there; waiting would self-deadlock). Returns {@code false} when + * the in-flight pass did not finish within the timeout — the caller must + * treat the worker as still live and leak rather than release. + *

    + * A pending caller interrupt is preserved but does not abort the wait, + * mirroring {@link #close()}. + */ + public boolean awaitRingQuiescence(SegmentRing ring) { + Runnable hook = beforeRingQuiescenceAwaitHook; + if (hook != null) { + hook.run(); + } + Thread t = workerThread; + if (t == null || t == Thread.currentThread()) { + return true; + } + long deadlineNanos = System.nanoTime() + workerJoinTimeoutMillis * 1_000_000L; + boolean interrupted = Thread.interrupted(); + try { + RingEntry e = inService; + if (e == null || e.ring != ring || !e.isInService()) { + return true; + } + synchronized (lock) { + e.quiescenceWaiters++; + try { + while (e.isInService()) { + long remainingNanos = deadlineNanos - System.nanoTime(); + if (remainingNanos <= 0) { + return false; + } + try { + // Round up so a sub-millisecond remainder still waits + // instead of spinning through wait(0) == wait-forever. + lock.wait(Math.max(1L, remainingNanos / 1_000_000L)); + } catch (InterruptedException ignored) { + interrupted = true; + } + } + } finally { + e.quiescenceWaiters--; + } + } + return true; + } finally { + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + } + + /** + * True when no manager worker thread can be running: either + * {@link #start()} was never called, or a {@link #close()} confirmed + * worker termination and reaped the thread. Owners use this as a + * stronger fallback barrier when {@link #awaitRingQuiescence(SegmentRing)} + * times out but a subsequent {@code close()} join succeeded. + */ + public synchronized boolean isWorkerReaped() { + return workerThread == null; } /** @@ -213,6 +537,13 @@ public synchronized void close() { * created after this returns, but already-installed spares stay with * the ring (the ring closes them on its own {@link SegmentRing#close}). * Idempotent; safe to call from any thread. + *

    + * Non-blocking: a worker service pass already in flight for this ring + * may still be running when this returns. Callers about to release + * resources the worker can reach (the ring itself, its watermark, its + * segment files, or the slot lock guarding its directory) MUST follow + * up with {@link #awaitRingQuiescence(SegmentRing)} and only release on + * a {@code true} result. */ public void deregister(SegmentRing ring) { synchronized (lock) { @@ -226,7 +557,7 @@ public void deregister(SegmentRing ring) { // single subtraction covers both the initial seed // and the net manager activity (provisions minus // trims) for this ring. - e.registered = false; + e.deregister(); totalBytes -= ring.totalSegmentBytes(); rings.remove(i); return; @@ -247,7 +578,7 @@ public void deregister(SegmentRing ring) { * the high-water mark — no waiting on the next tick. */ public void register(SegmentRing ring, String dir) { - register(ring, dir, null); + register(ring, dir, null, 0L); } /** @@ -260,6 +591,20 @@ public void register(SegmentRing ring, String dir) { * {@code lowestSurvivingBaseSeq - 1}. */ public void register(SegmentRing ring, String dir, AckWatermark watermark) { + register(ring, dir, watermark, 0L); + } + + /** + * Registers a ring with an optional periodic data-checkpoint interval. + * A positive interval requires disk-backed store-and-forward mode. + */ + public void register(SegmentRing ring, String dir, AckWatermark watermark, long syncIntervalNanos) { + if (syncIntervalNanos < 0L) { + throw new IllegalArgumentException("syncIntervalNanos must not be negative"); + } + if (syncIntervalNanos > 0L && dir == null) { + throw new IllegalArgumentException("periodic sync requires a segment directory"); + } // Account for bytes the ring already owns when it joins. A recovered // ring (post-restart, orphan adoption) can come up at-or-above the cap; // without this seed, totalBytes stays at 0 and the per-tick cap check @@ -273,7 +618,7 @@ public void register(SegmentRing ring, String dir, AckWatermark watermark) { // the in-flight mmap. Memory-mode rings have no dir; nothing to scan. long minNextGeneration = dir == null ? -1L : scanMaxGeneration(dir) + 1L; Runnable managerWakeup = this::wakeWorker; - RingEntry e = new RingEntry(ring, dir, watermark); + RingEntry e = new RingEntry(ring, dir, watermark, syncIntervalNanos, ticks.getAsLong()); // ObjList.add either throws before storing e or makes the entry visible. // Once visible, only non-throwing state commits may remain. synchronized (lock) { @@ -282,6 +627,12 @@ public void register(SegmentRing ring, String dir, AckWatermark watermark) { } rings.add(e); totalBytes += ringBytes; + if (syncIntervalNanos > 0L) { + ring.enablePeriodicSync(); + if (syncIntervalNanos < shortestSyncIntervalNanos) { + shortestSyncIntervalNanos = syncIntervalNanos; + } + } } ring.setManagerWakeup(managerWakeup); // Nudge the worker so it picks up the new ring on its very next @@ -297,16 +648,82 @@ public void register(SegmentRing ring, String dir, AckWatermark watermark) { wakeWorker(); } + @TestOnly + public SegmentRing getInServiceRingForTesting() { + RingEntry entry = inService; + return entry == null ? null : entry.ring; + } + + @TestOnly + public long getTotalBytesForTesting() { + synchronized (lock) { + return totalBytes; + } + } + + @TestOnly + public Thread getWorkerThreadForTesting() { + return workerThread; + } + + @TestOnly + public boolean isPathScratchAllocatedForTesting() { + synchronized (lock) { + return !scratchFreed; + } + } + + @TestOnly + public boolean serviceRingForTesting(SegmentRing ring) { + if (workerThread != null) { + throw new IllegalStateException("test service requires a stopped manager"); + } + RingEntry selected = null; + synchronized (lock) { + for (int i = 0, n = rings.size(); i < n; i++) { + RingEntry candidate = rings.getQuick(i); + if (candidate.ring == ring) { + selected = candidate; + break; + } + } + } + if (selected == null) { + throw new IllegalArgumentException("ring is not registered"); + } + return serviceRing(selected); + } + + @TestOnly + public void setAfterRingCleanupHook(Runnable hook) { + this.afterRingCleanupHook = hook; + } + + @TestOnly + public void setBeforeExitCleanupRegistrationHook(Runnable hook) { + this.beforeExitCleanupRegistrationHook = hook; + } + @TestOnly public void setBeforeInstallSyncHook(Runnable hook) { this.beforeInstallSyncHook = hook; } + @TestOnly + public void setBeforeRingQuiescenceAwaitHook(Runnable hook) { + this.beforeRingQuiescenceAwaitHook = hook; + } + @TestOnly public void setBeforeTrimSyncHook(Runnable hook) { this.beforeTrimSyncHook = hook; } + @TestOnly + public void setRetryLogHook(Runnable hook) { + this.retryLogHook = hook; + } + @TestOnly public void setWorkerJoinTimeoutMillis(long millis) { this.workerJoinTimeoutMillis = millis; @@ -341,21 +758,19 @@ public void wakeWorker() { * files in {@code dir}, or {@code -1} if none exist. Skips files that * don't match the pattern (e.g. the legacy {@code sf-initial.sfa}). */ - private static long scanMaxGeneration(String dir) { + private long scanMaxGeneration(String dir) { long max = -1L; - if (!Files.exists(dir)) return max; - long find = Files.findFirst(dir); + if (!filesFacade.exists(dir)) return max; + long find = filesFacade.findFirst(dir); if (find < 0) { - LOG.warn("scanMaxGeneration could not enumerate {}; " - + "next spare may collide with an existing on-disk segment", dir); - return max; + throw new SfOperationalException("could not enumerate SF segment directory " + dir); } if (find == 0) return max; try { int rc = 1; while (rc > 0) { - String name = Files.utf8ToString(Files.findName(find)); - rc = Files.findNext(find); + String name = Files.utf8ToString(filesFacade.findName(find)); + rc = filesFacade.findNext(find); if (name == null || !name.startsWith("sf-") || !name.endsWith(".sfa")) { continue; } @@ -368,8 +783,11 @@ private static long scanMaxGeneration(String dir) { // sf-initial.sfa or non-hex — skip } } + if (rc < 0) { + throw new SfOperationalException("could not fully enumerate SF segment directory " + dir); + } } finally { - Files.findClose(find); + filesFacade.findClose(find); } return max; } @@ -411,14 +829,62 @@ private String nextSparePath(String dir) { return displayPath; } - private void serviceRing(RingEntry e) { + private boolean serviceRing(RingEntry e) { + // Publish before the CAS so deregister + await can always find the + // entry. The state CAS is the ownership decision: if deregister won, + // this stale snapshot pass skips the ring without touching it. + inService = e; + if (!ENTRY_STATE_UPDATER.compareAndSet(e, ENTRY_REGISTERED, ENTRY_IN_SERVICE)) { + inService = null; + return false; + } + boolean hasMoreTrimmable; + try { + hasMoreTrimmable = serviceRing0(e); + } finally { + e.finishService(); + Runnable cleanup = e.quiescenceCleanup == null + ? null + : ENTRY_CLEANUP_UPDATER.getAndSet(e, null); + inService = null; + // A normal pass performs only the two state CAS operations above. + // The manager monitor is entered here only for an actual close + // waiter; the recheck under lock prevents lost wakeups. + if (e.quiescenceWaiters > 0) { + synchronized (lock) { + if (e.quiescenceWaiters > 0) { + lock.notifyAll(); + } + } + } + if (cleanup != null) { + try { + cleanup.run(); + } catch (Throwable t) { + LOG.error("deferred engine cleanup failed after manager-worker ring pass", t); + } finally { + Runnable hook = afterRingCleanupHook; + if (hook != null) { + hook.run(); + } + } + } + } + return hasMoreTrimmable; + } + + private boolean serviceRing0(RingEntry e) { + boolean memoryMode = e.dir == null; + if (!memoryMode) { + servicePeriodicSync(e, ticks.getAsLong()); + } + // 1. Provision a hot spare if the ring needs one AND we have headroom // under the disk-total cap. Cap check is per-tick; if we're capped // here, the ring stays in BACKPRESSURE_NO_SPARE until trim (step 2) // on this or a subsequent tick frees space. Logged at most once per // DISK_FULL_LOG_THROTTLE_NANOS so a sustained-disk-full state // doesn't drown the log. - boolean memoryMode = e.dir == null; if (e.ring.needsHotSpare()) { // Snapshot totalBytes under lock — register/deregister can mutate // it from caller threads. Heavy provisioning I/O happens outside @@ -455,14 +921,21 @@ private void serviceRing(RingEntry e) { // via its long-ptr overload, bypassing the byte[] + native // malloc that the String overload would incur on every // rotation. - spare = MmapSegment.create(FilesFacade.INSTANCE, + spare = MmapSegment.create(filesFacade, pathScratch.ptr(), path, - e.ring.nextSeqHint(), segmentSizeBytes); + e.ring.nextSeqHint(), segmentSizeBytes, true); } Runnable installHook = beforeInstallSyncHook; if (installHook != null) { installHook.run(); } + if (!memoryMode) { + spare.syncHeader(); + if (filesFacade.fsyncDir(e.dir) != 0) { + throw new MmapSegmentException( + "could not sync hot-spare directory " + e.dir); + } + } // Install + commit atomically under the manager lock. // If `e.ring` was deregistered between the snapshot // above and now, abandoning the spare here is the only @@ -476,7 +949,7 @@ private void serviceRing(RingEntry e) { // observe the spare in the ring (and subtract it) or // run before installation (so no install happens). synchronized (lock) { - if (e.registered) { + if (e.isRegistered()) { e.ring.installHotSpare(spare); totalBytes += segmentSizeBytes; installed = true; @@ -493,110 +966,367 @@ private void serviceRing(RingEntry e) { } catch (Throwable ignored) { } } - // Remove the file even when spare is null (i.e. when - // MmapSegment.create itself threw): MmapSegment.create's - // catch already best-effort removes, but if anything - // before mmap (e.g. an exception thrown by the JVM - // between openCleanRW and the try block) leaves a file - // on disk, this is the second-line defense. Repeated - // unlink on an already-removed path is a harmless no-op. - if (path != null) { - Files.remove(path); + // Only remove the file when the spare object exists, i.e. + // MmapSegment.create succeeded and ownership is ours but + // installation was rejected (ring deregistered/closed). + // When create() itself threw, its catch already removed + // anything it put on disk -- and with exclusive create + // (O_EXCL) a failure can also mean the path was ALREADY + // occupied by a file some other lifecycle owns, which a + // blanket unlink here would destroy. + if (path != null && spare != null) { + filesFacade.remove(path); } } } } - // 2. Trim any segments that the ring says are fully acked. For - // memory-mode rings, "trim" is just close() (Unsafe.free) — no - // file to unlink. - // - // The watermark write and totalBytes commit are registration-gated - // under `lock` so stale worker snapshots cannot touch the - // engine-owned watermark or mutate accounting after deregister() - // returns. drainTrimmable still runs for stale snapshots: it - // transfers ownership of fully-acked sealed segments to this - // worker, preserving the old close + unlink behavior. - // - // munmap + unlink stay outside the lock — they can be slow - // and shouldn't block register/deregister or sibling rings. - ObjList trim; + // 2. Trim fully ACKed segments. The ring first transfers one bounded, + // unpinned prefix out of live traversal and into hidden pending + // ownership. Only then may this worker unmap it. Disk mode keeps the + // pending prefix until unlink + directory fsync are durable; memory + // mode commits each successfully freed prefix immediately. No + // syscall runs under the manager lock. Runnable hook = beforeTrimSyncHook; if (hook != null) { hook.run(); } - synchronized (lock) { - boolean registered = e.registered; - // Persist the current ackedFsn watermark BEFORE the trim runs. - // On host crash between the persist and the unlinks below, the - // segments survive and the watermark is correct. On crash AFTER - // the unlinks but before next tick, the segments are gone and - // the watermark is stale, but recovery clamps with - // max(lowestSurvivingBaseSeq - 1, watermark) so either ordering - // is safe. Memory-mode rings (and callers that didn't supply a - // watermark) skip the write. - // Persist only on advance to avoid pointless mmap stores when - // ackedFsn is steady. The store is a single 8-byte put against - // an already-mapped region -- no syscall, no allocation -- but - // the gate keeps the dirty-page footprint minimal under - // steady-state load with no new acks arriving. - if (registered && e.watermark != null) { - long currentAck = e.ring.ackedFsn(); - if (currentAck > e.lastPersistedAck) { - e.watermark.write(currentAck); - e.lastPersistedAck = currentAck; - } - } - trim = e.ring.drainTrimmable(); - if (registered && trim != null) { - for (int i = 0, n = trim.size(); i < n; i++) { - totalBytes -= trim.get(i).sizeBytes(); - } - } - } - if (trim != null) { - for (int i = 0, n = trim.size(); i < n; i++) { - MmapSegment s = trim.get(i); - String path = s.path(); - try { - s.close(); - if (path != null && !Files.remove(path)) { - LOG.warn("Failed to unlink trimmed segment {}", path); + int pendingCount = e.ring.pendingTrimCount(); + MmapSegment first = pendingCount == 0 ? e.ring.firstTrimmable() : null; + if (pendingCount == 0 && first == null) { + // Preserve the cheap mmap-only watermark cadence for ACKs that do + // not yet cover a complete, unpinned sealed segment. + synchronized (lock) { + if (e.isRegistered() && e.watermark != null) { + long currentAck = e.ring.ackedFsn(); + if (currentAck > e.lastPersistedAck) { + e.watermark.write(currentAck); + e.lastPersistedAck = currentAck; } + } + } + return false; + } + + if (memoryMode) { + int batchSize = pendingCount > 0 + ? e.ring.copyPendingTrims(trimBatch) + : e.ring.stagePendingTrims( + trimBatch, MAX_TRIMS_PER_RING_PASS, e.ring.ackedFsn()); + int closed = 0; + Throwable closeFailure = null; + while (closed < batchSize) { + try { + trimBatch[closed].close(); + closed++; } catch (Throwable t) { - LOG.warn("Failed to trim segment {}", path == null ? "" : path, t); + closeFailure = t; + break; } } + if (closed > 0) { + synchronized (lock) { + long removedBytes = e.ring.commitPendingTrims(trimBatch, closed); + if (e.isRegistered()) { + totalBytes -= removedBytes; + } + } + } + for (int i = 0; i < batchSize; i++) { + trimBatch[i] = null; + } + if (closeFailure != null) { + LOG.warn("Failed to trim memory segment", closeFailure); + return false; + } + return e.ring.firstTrimmable() != null; + } + + // A deferred disk retry does no sync, unlink, or logging work. Signed + // subtraction is the standard wrap-safe deadline comparison because + // the bounded delay is many orders of magnitude below half the long + // range, even when the monotonic clock wraps. + long now = ticks.getAsLong(); + if (e.trimRetryDelayNanos != 0 && now - e.trimRetryAtNanos < 0) { + return false; + } + + // Every attempt repeats the cheap covering barrier. Besides keeping + // the latest cumulative ACK durable, this preserves the same strict + // pre-unlink/post-unlink directory ordering on pending retries. + if (e.watermark == null) { + if (!e.missingWatermarkLogged) { + e.missingWatermarkLogged = true; + LOG.warn("Cannot durably trim acknowledged segments in {} without an ack watermark", e.dir); + } + return false; + } + long durableAck; + synchronized (lock) { + if (!e.isRegistered()) { + return false; + } + durableAck = e.ring.ackedFsn(); + if (durableAck > e.lastPersistedAck) { + e.watermark.write(durableAck); + e.lastPersistedAck = durableAck; + } + } + try { + e.watermark.sync(); + if (filesFacade.fsyncDir(e.dir) != 0) { + recordTrimFailure(e, TRIM_RETRY_PRE_BARRIER, now, null); + return false; + } + } catch (Throwable t) { + recordTrimFailure(e, TRIM_RETRY_PRE_BARRIER, now, t); + return false; + } + + int batchSize; + if (pendingCount > 0) { + batchSize = e.ring.copyPendingTrims(trimBatch); + } else { + try { + // Under the ring monitor: advance the manifest past the last + // eligible member, then atomically hide the batch. No I/O pin + // can appear between the eligibility check and live removal. + batchSize = e.ring.stagePendingTrims( + trimBatch, MAX_TRIMS_PER_RING_PASS, durableAck); + } catch (Throwable t) { + for (int i = 0; i < trimBatch.length; i++) { + trimBatch[i] = null; + } + recordTrimFailure(e, TRIM_RETRY_UNLINK, now, t); + return false; + } + if (batchSize == 0) { + return false; + } + } + + boolean trimFailed = false; + Throwable trimFailure = null; + int unlinked = 0; + while (unlinked < batchSize) { + MmapSegment trimming = trimBatch[unlinked]; + String path = trimming.path(); + try { + trimming.close(); + if (!filesFacade.remove(path) && filesFacade.exists(path)) { + trimFailed = true; + break; + } + unlinked++; + } catch (Throwable t) { + trimFailed = true; + trimFailure = t; + break; + } + } + + if (unlinked > 0) { + try { + if (filesFacade.fsyncDir(e.dir) != 0) { + for (int i = 0; i < batchSize; i++) { + trimBatch[i] = null; + } + recordTrimFailure(e, TRIM_RETRY_POST_BARRIER, now, null); + return false; + } + } catch (Throwable t) { + for (int i = 0; i < batchSize; i++) { + trimBatch[i] = null; + } + recordTrimFailure(e, TRIM_RETRY_POST_BARRIER, now, t); + return false; + } + try { + synchronized (lock) { + long removedBytes = e.ring.commitPendingTrims(trimBatch, unlinked); + if (e.isRegistered()) { + totalBytes -= removedBytes; + } + } + } catch (Throwable t) { + for (int i = 0; i < batchSize; i++) { + trimBatch[i] = null; + } + recordTrimFailure(e, TRIM_RETRY_POST_BARRIER, now, t); + return false; + } + } + for (int i = 0; i < batchSize; i++) { + trimBatch[i] = null; + } + if (trimFailed) { + recordTrimFailure(e, TRIM_RETRY_UNLINK, now, trimFailure); + return false; + } + recordTrimSuccess(e); + return e.ring.firstTrimmable() != null; + } + + private void recordTrimFailure(RingEntry e, int failureKind, long now, Throwable failure) { + long delay = e.trimRetryDelayNanos == 0 + ? TRIM_RETRY_INITIAL_NANOS + : Math.min(e.trimRetryDelayNanos << 1, TRIM_RETRY_MAX_NANOS); + e.trimRetryAtNanos = now + delay; + e.trimRetryDelayNanos = delay; + if (e.trimRetryFailureKind != failureKind) { + e.trimRetryFailureKind = failureKind; + if (failure == null) { + LOG.warn("Durable segment trim failed in {} during {} (retry delayed)", + e.dir, trimFailureName(failureKind)); + } else { + LOG.warn("Durable segment trim failed in {} during {} (retry delayed)", + e.dir, trimFailureName(failureKind), failure); + } + Runnable hook = retryLogHook; + if (hook != null) { + hook.run(); + } + } + } + + private void recordTrimSuccess(RingEntry e) { + if (e.trimRetryDelayNanos != 0) { + e.trimRetryAtNanos = 0; + e.trimRetryDelayNanos = 0; + e.trimRetryFailureKind = TRIM_RETRY_NONE; + LOG.info("Durable segment trim recovered in {}", e.dir); + Runnable hook = retryLogHook; + if (hook != null) { + hook.run(); + } + } + } + + private static String trimFailureName(int failureKind) { + switch (failureKind) { + case TRIM_RETRY_PRE_BARRIER: + return "covering barrier"; + case TRIM_RETRY_UNLINK: + return "segment unlink"; + default: + return "directory commit barrier"; + } + } + + private void servicePeriodicSync(RingEntry e, long now) { + if (e.syncIntervalNanos <= 0L + || (!e.ring.isSyncRequested() && now - e.nextDataSyncNanos < 0L)) { + return; + } + try { + e.ring.copyLiveSegmentsForSync(syncScratch); + for (int i = 0, n = syncScratch.size(); i < n; i++) { + syncScratch.getQuick(i).syncPublished(); + } + e.ring.clearSyncRequestIfActiveDurable(); + // The pass above covered every live segment's published range, and + // a failed barrier re-dirties its range under an mlock pin (see + // MmapSegment.syncPublished), so a success here is a genuine + // re-persist -- not a vacuous retry over pages a failed writeback + // marked clean (fsyncgate). Unlatch so a transient disk fault + // doesn't permanently brick the producer. + e.ring.clearDurabilityFailure(); + e.nextDataSyncNanos = now + e.syncIntervalNanos; + if (e.syncFailureLogged) { + e.syncFailureLogged = false; + LOG.info("Periodic SF data sync recovered for {}", e.dir); + } + } catch (Throwable failure) { + e.ring.recordDurabilityFailure(failure); + if (!e.syncFailureLogged) { + e.syncFailureLogged = true; + LOG.error("Periodic SF data sync failed for {}", e.dir, failure); + } + long retry = Math.min(e.syncIntervalNanos, 1_000_000_000L); + e.nextDataSyncNanos = now + retry; + } finally { + syncScratch.clear(); } } private void workerLoop() { - while (running) { - // Snapshot the rings under the lock so we don't hold it through the - // (potentially slow) syscalls during creation/unlink. ringSnapshot - // is a thread-confined field — no per-tick allocation. - ringSnapshot.clear(); + try { + while (running) { + // Snapshot the rings under the lock so we don't hold it through the + // (potentially slow) syscalls during creation/unlink. ringSnapshot + // is a thread-confined field — no per-tick allocation. + ringSnapshot.clear(); + synchronized (lock) { + for (int i = 0, n = rings.size(); i < n; i++) { + ringSnapshot.add(rings.getQuick(i)); + } + } + boolean hasMoreTrimmable = false; + for (int i = 0, n = ringSnapshot.size(); i < n; i++) { + if (!running) break; + if (serviceRing(ringSnapshot.getQuick(i))) { + hasMoreTrimmable = true; + } + } + // Drop strong refs so a deregistered ring becomes collectable + // before the next tick (otherwise the snapshot pins it for up + // to pollNanos after deregister). + ringSnapshot.clear(); + if (!running) break; + if (!hasMoreTrimmable) { + LockSupport.parkNanos(Math.min(pollNanos, shortestSyncIntervalNanos)); + } + } + } finally { + // If a timed-out close() abandoned the reap, it handed + // pathScratch ownership to this thread (see close()). Freeing it + // here reclaims the native buffer even when the worker outlives + // every close() attempt — nobody else retries manager cleanup. + ObjList cleanups; + Runnable ownedEngineCleanup; synchronized (lock) { - for (int i = 0, n = rings.size(); i < n; i++) { - ringSnapshot.add(rings.getQuick(i)); + workerLoopExited = true; + if (scratchHandedToWorker && !scratchFreed) { + scratchFreed = true; + pathScratch.close(); } + cleanups = exitCleanups; + exitCleanups = null; + ownedEngineCleanup = ownedEngineExitCleanup; + ownedEngineExitCleanup = null; } - for (int i = 0, n = ringSnapshot.size(); i < n; i++) { - if (!running) break; - serviceRing(ringSnapshot.getQuick(i)); + // Deferred engine cleanups run OUTSIDE + // `lock`: they perform syscalls (munmap, unlink, flock release) + // and must never execute under a lock that close()/register/ + // deregister callers contend on. Running them after the loop body + // is what makes the handoff safe: this thread can no longer touch + // any slot path. They must also never block on a caller-held + // monitor — a retried engine.close() joins this thread while + // holding the engine monitor, which is why the engine side uses a + // lock-free claim (CAS), not synchronization, for exactly-once. + if (ownedEngineCleanup != null) { + try { + ownedEngineCleanup.run(); + } catch (Throwable t) { + LOG.error("deferred owned-engine cleanup failed on manager-worker exit", t); + } + } + if (cleanups != null) { + for (int i = 0, n = cleanups.size(); i < n; i++) { + try { + cleanups.getQuick(i).run(); + } catch (Throwable t) { + LOG.error("deferred engine cleanup failed on manager-worker exit", t); + } + } } - // Drop strong refs so a deregistered ring becomes collectable - // before the next tick (otherwise the snapshot pins it for up - // to pollNanos after deregister). - ringSnapshot.clear(); - if (!running) break; - LockSupport.parkNanos(pollNanos); } } private static final class RingEntry { final String dir; final SegmentRing ring; + final long syncIntervalNanos; // Engine-owned ack watermark for this slot, or null in memory // mode and for callers that didn't supply one. Manager-thread // only after register; never closed here (owner closes). @@ -607,16 +1337,77 @@ private static final class RingEntry { // Survives across multiple serviceRing ticks and avoids a // write-storm when ackedFsn is steady. long lastPersistedAck = -1L; - // Guarded by SegmentManager.lock. A worker snapshot may retain this - // entry after deregister() removes it from rings; registered=false is - // the O(1) ownership check that prevents post-deregister writes through - // the engine-owned watermark, hot-spare installs, and accounting. - boolean registered = true; + // Prevents a legacy disk registration without a watermark from + // flooding the log on every manager tick. + boolean missingWatermarkLogged; + long nextDataSyncNanos; + boolean syncFailureLogged; + // Zero-allocation manager-thread-only retry state. The deadline uses + // the manager's monotonic clock and the delay doubles to a fixed cap. + long trimRetryAtNanos; + long trimRetryDelayNanos; + int trimRetryFailureKind; + // Updated lock-free by deferUntilRingQuiescent and pass completion. + // The field updater avoids one AtomicReference allocation per ring. + volatile Runnable quiescenceCleanup; + // Waiters mutate this under SegmentManager.lock; pass completion reads + // it before taking the otherwise-cold notification path. + volatile int quiescenceWaiters; + // REGISTERED -> IN_SERVICE -> REGISTERED on a normal pass. + // Deregistration changes either registered state to its corresponding + // deregistered state; completion then changes DEREGISTERED_IN_SERVICE + // to DEREGISTERED. The field updater avoids per-entry allocation. + volatile int state = ENTRY_REGISTERED; - RingEntry(SegmentRing ring, String dir, AckWatermark watermark) { + RingEntry( + SegmentRing ring, + String dir, + AckWatermark watermark, + long syncIntervalNanos, + long now + ) { this.ring = ring; this.dir = dir; this.watermark = watermark; + this.syncIntervalNanos = syncIntervalNanos; + // Run the first periodic pass immediately. This establishes a + // durable baseline for segments recovered from MEMORY mode. + this.nextDataSyncNanos = now; + } + + void deregister() { + while (true) { + int current = state; + int next = current == ENTRY_IN_SERVICE + ? ENTRY_DEREGISTERED_IN_SERVICE + : ENTRY_DEREGISTERED; + if (current == ENTRY_DEREGISTERED || current == ENTRY_DEREGISTERED_IN_SERVICE + || ENTRY_STATE_UPDATER.compareAndSet(this, current, next)) { + return; + } + } + } + + void finishService() { + while (true) { + int current = state; + int next = current == ENTRY_DEREGISTERED_IN_SERVICE + ? ENTRY_DEREGISTERED + : ENTRY_REGISTERED; + if (ENTRY_STATE_UPDATER.compareAndSet(this, current, next)) { + return; + } + } + } + + boolean isInService() { + int current = state; + return current == ENTRY_IN_SERVICE || current == ENTRY_DEREGISTERED_IN_SERVICE; + } + + boolean isRegistered() { + int current = state; + return current == ENTRY_REGISTERED || current == ENTRY_IN_SERVICE; } } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java index c8516c4c..d366fbc5 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java @@ -25,12 +25,15 @@ package io.questdb.client.cutlass.qwp.client.sf.cursor; import io.questdb.client.std.Files; +import io.questdb.client.std.FilesFacade; import io.questdb.client.std.ObjList; import io.questdb.client.std.QuietCloseable; import org.jetbrains.annotations.TestOnly; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.IdentityHashMap; + /** * Chain of {@link MmapSegment}s presented to the user thread as one logical * append-only log keyed by frame sequence number (FSN). Owns segment @@ -40,10 +43,10 @@ *

  • Producer thread (single user thread): {@link #appendOrFsn}, * {@link #installHotSpare}, {@link #publishedFsn}.
    • *
  • I/O thread: {@link #publishedFsn} (read-only), {@link #acknowledge} - * (single writer), {@link #drainTrimmable} (single reader).
    • + * (single writer), and one pinned segment cursor for native reads. *
  • Segment manager: polls {@link #needsHotSpare}, hands new - * segments via {@link #installHotSpare}, drains trim-eligible segments - * via {@link #drainTrimmable} on its own cadence.
    • + * segments via {@link #installHotSpare}, and stages trim-eligible + * segments into hidden cleanup ownership on its own cadence. * * No locks; the only cross-thread state is {@link #publishedFsn} (volatile, * single-writer) and {@link #ackedFsn} (volatile, single-writer). Hot-spare @@ -62,15 +65,31 @@ public final class SegmentRing implements QuietCloseable { public static final long BACKPRESSURE_NO_SPARE = -1L; /** Sentinel: append failed because the payload doesn't fit in a fresh segment. */ public static final long PAYLOAD_TOO_LARGE = -2L; + private static final RetainedSegmentMembershipMode DEFAULT_MEMBERSHIP_MODE = + RetainedSegmentMembershipMode.IDENTITY; private static final Logger LOG = LoggerFactory.getLogger(SegmentRing.class); + private static final int MAX_PENDING_TRIMS = 64; + // Tally of sealed-list entries inspected by nextSealedAfter(). Test-only + // operation count for deterministic traversal-complexity assertions. + private static long nextSealedComparisons; // Tally of baseSeq comparisons performed by sortByBaseSeq across every // openExisting() recovery on this JVM. Used by SegmentRingTest to // assert the sort stays O(N log N) without relying on wall-clock time // (CI runner variance makes elapsed-millisecond bounds flaky). Cheap - // in production: one volatile-free add per partition pass, dwarfed by - // the mmap I/O the recovery does on every segment. + // in production: one volatile-free add per partition pass (plus two per + // sift level in the rare heapsort fallback), dwarfed by the mmap I/O + // the recovery does on every segment. private static long sortComparisons; + // References copied while compacting the logical sealed-segment head. + // Test-only operation count for deterministic trim-complexity assertions. + private static long trimMovedReferences; private final long maxBytesPerSegment; + private final SfManifest manifest; + // ACKed segments leave live traversal under the ring monitor before the + // manager unmaps them. They remain owned here until close + unlink + the + // directory barrier succeed, so failures stay retryable and accounted. + // At most one bounded manager batch is pending at a time. + private final ObjList pendingTrims = new ObjList<>(MAX_PENDING_TRIMS); // Sealed segments in baseSeq order, oldest first. Active is held separately. // Single-writer (producer thread, on rotation); single-reader at trim time // (the segment manager). For now, both sides synchronize via the single- @@ -78,6 +97,9 @@ public final class SegmentRing implements QuietCloseable { // looks at sealedSegments after observing a higher ackedFsn, by which // point the producer thread's add to sealedSegments has retired. private final ObjList sealedSegments = new ObjList<>(); + // Logical head of sealedSegments. Head removal nulls one entry and advances + // this index; occasional compaction bounds unused prefix slots. + private int sealedHead; // High-water byte offset within the active segment at which we proactively // ask the segment manager to provision a spare (if one isn't already // installed). Computed once as 3/4 of segment capacity -- leaves the manager @@ -96,18 +118,28 @@ public final class SegmentRing implements QuietCloseable { // call installHotSpare on a closed ring (whose hotSpare was just // zeroed by close()) -- the spare's mmap + fd would never be reclaimed. private boolean closed; + // Latest periodic data-barrier failure. The manager latches it and the + // producer observes it before its next append; the manager clears it + // again once a subsequent periodic sync pass succeeds, so a transient + // disk fault does not permanently brick the producer. + private volatile MmapSegmentException durabilityFailure; // hotSpare: written by segment manager (installHotSpare), read+cleared by // producer thread on rotation. Volatile so the producer sees fresh installs. private volatile MmapSegment hotSpare; + // Segment whose native mapping the single I/O consumer may dereference. + // Guarded by this monitor: cursor lookup/switch and manager trim staging + // are atomic, so a pinned segment cannot be hidden or unmapped. + private MmapSegment ioPinnedSegment; // Optional callback the segment manager registers via setManagerWakeup // so the producer can wake the manager out of its poll-park the moment - // a spare is needed (rotation just consumed one, or active crossed the - // high-water mark while no spare is installed). Without this, the - // manager only notices on its next polling tick -- fine on average, - // but the worst-case wait is the full poll interval. Producer-thread-only. + // a spare is needed, and the I/O thread can wake it after releasing a + // segment that may now be trimmable. Without this, the manager only + // notices on its next polling tick. private Runnable managerWakeup; private long nextSeq; + private boolean periodicSyncEnabled; private volatile long publishedFsn; + private volatile boolean syncRequested; // Plain (producer-thread-only) flag; set to true the first time we ask // the manager for a spare for the current active segment, cleared on // every rotation. Coalesces multiple high-water-mark crossings into a @@ -120,10 +152,15 @@ public final class SegmentRing implements QuietCloseable { * frameCount == 0); typically supplied by the segment manager at startup. */ public SegmentRing(MmapSegment initialActive, long maxBytesPerSegment) { + this(initialActive, maxBytesPerSegment, null); + } + + SegmentRing(MmapSegment initialActive, long maxBytesPerSegment, SfManifest manifest) { if (initialActive == null) { throw new IllegalArgumentException("initialActive must not be null"); } this.active = initialActive; + this.manifest = manifest; this.maxBytesPerSegment = maxBytesPerSegment; // 3/4 of capacity gives the manager a full quarter-segment of producer // runway before backpressure kicks in. Long math, no float, no alloc. @@ -133,190 +170,720 @@ public SegmentRing(MmapSegment initialActive, long maxBytesPerSegment) { } /** - * Recovers a ring from segments already on disk in {@code sfDir}. Used at - * sender startup when the user's previous session left durable but - * not-yet-acked frames behind. Walks every {@code *.sfa} file in the - * directory, opens each via {@link MmapSegment#openExisting}, and - * arranges them by baseSeq: - *
      - *
    • Highest-baseSeq segment becomes the active (further appends land - * there until it fills, at which point normal rotation kicks in).
      • - *
    • All others become sealed segments awaiting ACK and trim.
      • - *
    - * Returns {@code null} if the directory is empty or contains no - * recognizable {@code .sfa} files -- the caller should then construct a - * fresh ring with {@link #SegmentRing(MmapSegment, long)} and a freshly - * created initial segment. - *

    - * Recovery is best-effort: a single bad-magic file is silently skipped - * (logged-then-ignored is the right call here; a stray unrelated file in - * the SF dir shouldn't take the whole sender down). A failure to open - * an otherwise-valid segment IS fatal -- the caller's data integrity - * depends on every segment being readable. + * Compatibility wrapper using the production facade. New startup code uses + * {@link #recover(FilesFacade, String, long)} so EMPTY is explicit. */ public static SegmentRing openExisting(String sfDir, long maxBytesPerSegment) { - if (!Files.exists(sfDir)) { - return null; + return openExisting(FilesFacade.INSTANCE, sfDir, maxBytesPerSegment); + } + + /** + * Facade-aware variant of {@link #openExisting(String, long)}: every + * filesystem touch (enumeration, open, mmap, quarantine rename, manifest + * I/O) goes through {@code filesFacade} so recovery's fail-closed behavior + * can be fault-injected in tests. Returns {@code null} when the slot holds + * nothing recoverable; throws {@link MmapSegmentException} when the slot's + * state cannot be proven safe. + */ + public static SegmentRing openExisting(FilesFacade filesFacade, String sfDir, long maxBytesPerSegment) { + Recovery recovery = recover(filesFacade, sfDir, maxBytesPerSegment); + return recovery.status == RecoveryStatus.EMPTY ? null : recovery.ring; + } + + /** + * Exhaustively discovers and validates an SF slot without mutating frame + * data. Only after enumeration, opens, CRC scans, contiguity and manifest + * boundaries all succeed does it migrate a legacy chain or discard + * validated spares. Torn-tail residue is handled destroy-last: + * {@link MmapSegment#openExisting} only observes it, and recovery zeroes + * it exclusively where validation has proven it non-load-bearing -- the + * resumed active's tail (about to be reclaimed by appends) and sealed + * suffixes whose frame accounting validated complete against the chain. + * A tear that cost frames (e.g. a mid-file tear in a sealed member) + * fails closed BEFORE any of that, leaving every byte on disk for + * operator extraction. + */ + static Recovery recover(FilesFacade filesFacade, String sfDir, long maxBytesPerSegment) { + return recover(filesFacade, sfDir, maxBytesPerSegment, null); + } + + static Recovery recover( + FilesFacade filesFacade, + String sfDir, + long maxBytesPerSegment, + MembershipObserver membershipObserver + ) { + if (!filesFacade.exists(sfDir)) { + return Recovery.empty(); } - ObjList opened = new ObjList<>(); - long find = Files.findFirst(sfDir); + ObjList names = new ObjList<>(); + long find = filesFacade.findFirst(sfDir); if (find < 0) { - LOG.warn("openExisting could not enumerate {} - treating as empty, " - + "but this may indicate a permission or transient error", sfDir); - return null; + throw new MmapSegmentException("could not enumerate SF directory " + sfDir); } - if (find == 0) { - return null; - } - // Outer try-catch: anything escaping the recovery body -- IOOBE from - // ObjList growth, OOM from native mmap during MmapSegment.openExisting, - // unforeseen RuntimeException from the contiguity check, etc. -- must - // not leave fds + mmaps owned by `opened` orphaned. Close every - // recovered segment and rethrow so the engine surfaces the failure. - try { + if (find > 0) { + int rc = 1; try { - int rc = 1; while (rc > 0) { - String name = Files.utf8ToString(Files.findName(find)); + String name = Files.utf8ToString(filesFacade.findName(find)); if (name != null && name.endsWith(".sfa")) { - String path = sfDir + "/" + name; - MmapSegment seg = null; - try { - seg = MmapSegment.openExisting(path); - // Filter out empty leftovers -- typically hot-spare - // segments the manager pre-allocated for a prior - // session that never got rotated into active. They - // carry the provisional baseSeq=0 and frameCount=0, - // which would otherwise collide with the real - // baseSeq=0 segment and trip the contiguity check - // below. No data to recover; close and unlink. - // Without the unlink the file persists across crash - // cycles and the disk leak compounds with every - // unclean shutdown. - // - // CAUTION: only unlink when the file is genuinely - // empty past the header. If frame[0] failed CRC - // (bit-rot, partial-page-write at crash, etc.) but - // valid frames followed, scanFrames returns - // lastGood=HEADER_SIZE and frameCount=0 -- yet - // tornTailBytes is non-zero. Treating that as - // "empty hot-spare" would silently destroy every - // surviving frame. Quarantine to .corrupt - // instead so a postmortem can recover what's left. - if (seg.frameCount() == 0) { - long torn = seg.tornTailBytes(); - seg.close(); - seg = null; - if (torn > 0) { - Files.rename(path, path + ".corrupt"); - } else { - Files.remove(path); - } - } else { - opened.add(seg); - seg = null; + names.add(name); + } + rc = filesFacade.findNext(find); + } + if (rc < 0) { + throw new MmapSegmentException("could not fully enumerate SF directory " + sfDir); + } + } finally { + filesFacade.findClose(find); + } + } + + ObjList all = new ObjList<>(); + // Files whose own bytes prove corruption (bad magic, sub-header size, + // negative baseSeq). They are excluded from the chain and quarantined + // to .corrupt — but only AFTER the surviving chain validates (or + // resolves to EMPTY). The precise invariant: a failed recovery never + // mutates COMMITTED CHAIN BYTES. It is not "never mutates the slot" + // -- several windows durably mutate before a later step can still + // fail: proven-dead sealed residue is zeroed just before the + // fail-closed first-sight throw in sanitizeSealedResidue; the + // legacy-migration sanitize zeroes proven-dead residue before + // SfManifest.create (or any later step) can fail; and validated- + // extra cleanup plus corrupt-file quarantine run before the + // active-sanitize barrier or ring construction can fail. Every such + // window is confined to bytes the already-validated chain proves no + // replay can ever need, or to preserve-by-rename quarantines that + // keep the bytes on disk. + // Whether a quarantined file was load-bearing is decided by the + // manifest-boundary / contiguity checks below, not by the skip itself. + // Operational open/stat/read/mmap errors, observed size instability, + // and unsupported versions are NOT in this bucket: they throw the + // plain MmapSegmentException type and abort recovery, because the + // underlying file may be perfectly intact and silently dropping it + // could lose durable frames. + ObjList corruptPaths = null; + SfManifest manifest = null; + try { + for (int i = 0, n = names.size(); i < n; i++) { + String path = sfDir + "/" + names.get(i); + try { + all.add(MmapSegment.openExisting(filesFacade, path)); + } catch (MmapSegmentCorruptionException e) { + LOG.warn("recovery: {} is not a readable SF segment; excluding it and " + + "deferring quarantine until the surviving chain validates -- {}", + path, e.toString()); + if (corruptPaths == null) { + corruptPaths = new ObjList<>(); + } + corruptPaths.add(path); + } catch (MmapSegmentException e) { + throw new MmapSegmentException("recovery failed for recognized segment " + path, e); + } + } + manifest = SfManifest.open(filesFacade, sfDir); + if (all.size() == 0) { + if (corruptPaths != null) { + // Nothing valid survived. With a manifest this is still a + // hole we can prove (boundaries reference segments that are + // now unreadable) -- fail without mutating. Without one, + // legacy semantics apply: quarantine and start fresh. + if (manifest != null) { + throw new SfRecoveryException("every SF segment in " + sfDir + + " is corrupt but " + SfManifest.FILE_NAME + + " references durable data"); + } + quarantineCorrupt(filesFacade, corruptPaths); + return Recovery.empty(); + } + if (manifest != null) { + // No .sfa files at all. Only two protocols legitimately + // produce this, and both leave head == active: the + // close-time drain durably collapses the boundaries + // BEFORE its first unlink, and a fresh-start crash can + // leave a boundary-less (0,0) manifest behind. Uncollapsed + // boundaries therefore prove durable, never-declared-acked + // frames existed in [headBase, activeBase] whose files + // vanished outside the protocol (manual wipe, partial + // restore) -- fail closed and keep the manifest as + // evidence instead of silently starting fresh. + long manifestHeadBase = manifest.headBase(); + long manifestActiveBase = manifest.activeBase(); + if (manifestHeadBase != manifestActiveBase) { + throw new SfRecoveryException(SfManifest.FILE_NAME + " in " + sfDir + + " references durable data (headBase=" + manifestHeadBase + + ", activeBase=" + manifestActiveBase + + ") but no segment files exist"); + } + // Collapsed boundaries: nothing recoverable exists, so + // accept EMPTY -- but shout, because a manual wipe of a + // fully-acked slot looks identical and the operator + // should know. + LOG.warn("SF manifest exists in {} with collapsed boundaries ({}) and " + + "no segment files (clean-drain or fresh-start crash window, or " + + "manual segment removal); discarding it and starting fresh", + sfDir, manifestActiveBase); + manifest.close(); + manifest = null; + if (!SfManifest.removeFile(filesFacade, sfDir)) { + throw new MmapSegmentException( + "could not remove stale SF manifest in " + sfDir); + } + } + return Recovery.empty(); + } + + ObjList data = new ObjList<>(); + boolean requiresManifest = false; + for (int i = 0, n = all.size(); i < n; i++) { + MmapSegment segment = all.get(i); + requiresManifest |= segment.manifestRequired(); + if (segment.frameCount() > 0) { + data.add(segment); + } + } + sortByBaseSeq(data, 0, data.size()); + if (manifest == null && requiresManifest) { + throw new SfRecoveryException("new-format SF segment exists but " + + SfManifest.FILE_NAME + " is missing"); + } + + MmapSegment active; + ObjList chain = new ObjList<>(); + long headBase; + long activeBase; + if (manifest == null) { + if (data.size() > 0) { + validateContiguous(data); + for (int i = 0, n = data.size(); i < n; i++) { + chain.add(data.get(i)); + } + active = chain.get(chain.size() - 1); + headBase = chain.get(0).baseSeq(); + activeBase = active.baseSeq(); + // Legacy sealed members carrying pre-manifest torn + // residue: contiguity just proved their frame accounting + // complete, so the residue is dead bytes. Zero it now + // (silently -- legacy slots predate the fail-closed + // sealed-suffix contract) so the migrated chain presents + // the all-zero sealed suffixes the manifest-era + // invariant requires. + sanitizeSealedResidue(chain, false); + } else { + active = chooseEmptyInitial(all, sfDir); + if (active == null) { + // Legacy slot holding only empty leftovers, every one + // of them torn (a clean empty would have been chosen). + // Nothing recoverable: quarantine the torn evidence, + // drop the clean debris, start fresh -- exactly the + // pre-manifest behavior. + for (int i = 0, n = all.size(); i < n; i++) { + MmapSegment segment = all.get(i); + String path = segment.path(); + long torn = segment.tornTailBytes(); + segment.close(); + if (torn > 0) { + quarantineFile(filesFacade, path); + } else if (!filesFacade.remove(path)) { + LOG.warn("could not remove empty SF leftover {}", path); } - } catch (MmapSegmentException t) { - // Per-file data error (bad magic, bad header, - // unsupported version, mmap rejection on this one - // file). Don't take down recovery for one corrupt - // .sfa -- log and skip so siblings still recover. - // Resource exhaustion (OOM) and programmer errors - // (IOOBE) deliberately propagate to the outer - // catch, which closes every already-recovered - // segment and rethrows: continuing the loop after - // an OOM would just fail again on the next file - // while silently leaking the segments we managed - // to recover before it. - LOG.warn("openExisting: skipping {} -- {}", path, t.toString()); - } finally { - // Close any seg whose ownership wasn't transferred - // (either to opened, or via the empty-branch close - // above). Fires on a propagating throw between - // open and transfer -- most importantly an OOM - // from ObjList.add growing its backing array - // after the mmap+fd were already acquired. - if (seg != null) { - try { - seg.close(); - } catch (Throwable closeErr) { - LOG.warn("openExisting: error closing in-flight segment {}", - path, closeErr); - } + } + all.clear(); + quarantineCorrupt(filesFacade, corruptPaths); + return Recovery.empty(); + } + chain.add(active); + headBase = active.baseSeq(); + activeBase = active.baseSeq(); + } + manifest = SfManifest.create(filesFacade, sfDir, headBase, activeBase); + for (int i = 0, n = chain.size(); i < n; i++) { + chain.get(i).markManifestRequired(); + } + } else { + headBase = manifest.headBase(); + activeBase = manifest.activeBase(); + for (int i = 0, n = data.size(); i < n; i++) { + MmapSegment segment = data.get(i); + long end = segment.baseSeq() + segment.frameCount(); + if (segment.baseSeq() < headBase) { + if (end > headBase) { + throw new SfRecoveryException("segment overlaps committed SF head boundary"); + } + continue; // acknowledged stale file after manifest-before-unlink crash + } + if (segment.baseSeq() > activeBase) { + throw new SfRecoveryException("segment exists beyond committed SF active boundary"); + } + chain.add(segment); + } + if (chain.size() > 0) { + validateContiguous(chain); + if (chain.get(0).baseSeq() != headBase) { + throw new SfRecoveryException("missing expected SF head segment at base " + headBase); + } + } + active = findActive(all, activeBase); + if (active == null) { + if (chain.size() == 0 && headBase == activeBase && corruptPaths == null) { + // Clean-drain crash window: the close-time drain first + // durably collapses the boundaries to head == active + // (declaring every frame acked), then unlinks segments + // in ascending order -- so dying between the active's + // unlink and the spare's/manifest's leaves exactly + // this state: no data frame anywhere, no file at the + // committed active base, only empty spares and/or + // acked stale files. Nothing recoverable exists; + // accept EMPTY and clear the debris. Guarded on + // corruptPaths because an unreadable .sfa of unknown + // identity could be the real active -- in that case + // fail closed instead of guessing. + LOG.warn("SF manifest in {} has collapsed boundaries ({}) with no " + + "segment at the active base and no recovered frames; " + + "accepting the clean-drain crash window as empty", + sfDir, activeBase); + for (int i = 0, n = all.size(); i < n; i++) { + MmapSegment segment = all.get(i); + String path = segment.path(); + long torn = segment.tornTailBytes(); + segment.close(); + if (torn > 0) { + quarantineFile(filesFacade, path); + } else if (!filesFacade.remove(path)) { + LOG.warn("could not remove drained SF leftover {}", path); } } + all.clear(); + manifest.close(); + manifest = null; + if (!SfManifest.removeFile(filesFacade, sfDir)) { + throw new MmapSegmentException( + "could not remove stale SF manifest in " + sfDir); + } + return Recovery.empty(); } - rc = Files.findNext(find); + throw new SfRecoveryException("missing expected SF active segment at base " + activeBase); } - } finally { - Files.findClose(find); - } - if (opened.size() == 0) { - return null; - } - // Sort by baseSeq ascending. Worst-case segment count is - // sf_max_total_bytes / sf_max_bytes -- at the documented ceiling - // (1 TiB / 64 MiB) that is ~16K entries, where an O(N²) sort spends - // multiple seconds in compares + shifts before the I/O thread can - // start. In-place quicksort with median-of-three pivot keeps the - // no-allocation discipline of the surrounding code; median-of-three - // is required because readdir on many filesystems returns entries - // in lexicographic (== baseSeq-hex) order and a naive first-element - // pivot would degrade back to O(N²) on exactly that common case. - sortByBaseSeq(opened, 0, opened.size()); - // Sanity: the recovered segments must form a contiguous FSN range. - // Detect gaps so they don't silently produce duplicate or missing - // FSNs after recovery. A gap means a segment went missing (a - // manual deletion) or a sealed segment under-recovered -- its tail - // was cut short by a sparse/unbacked page or a mid-file media error - // (bad sector), the same class of fault scanFrames tolerates on the - // active segment but which corrupts the range on a sealed one. - for (int i = 1, n = opened.size(); i < n; i++) { - MmapSegment prev = opened.get(i - 1); - MmapSegment curr = opened.get(i); - long expected = prev.baseSeq() + prev.frameCount(); - if (curr.baseSeq() != expected) { - throw new MmapSegmentException( - "FSN gap in recovered segments: prev baseSeq=" + prev.baseSeq() - + " frameCount=" + prev.frameCount() - + " expected next baseSeq=" + expected - + " but got " + curr.baseSeq() - + " -- a segment was deleted, or a sealed segment's tail was" - + " truncated (sparse/unbacked page or disk media error);" - + " check disk health"); + if (chain.size() == 0) { + if (headBase != activeBase || active.frameCount() != 0 || corruptPaths != null) { + // corruptPaths guard: with an unreadable .sfa in the + // slot, the innocent-looking empty at the active base + // could be a leftover spare coincidentally carrying + // the same provisional baseSeq as a corrupted real + // active -- accepting it would quarantine unacked + // frames and re-issue their FSNs. Fail closed. + throw new SfRecoveryException( + "missing SF chain between committed boundaries" + + (corruptPaths != null + ? " (a corrupt segment prevents proving the empty state)" : "")); + } + chain.add(active); + } else if (chain.get(chain.size() - 1) != active) { + MmapSegment tail = chain.get(chain.size() - 1); + long chainEnd = tail.baseSeq() + tail.frameCount(); + if (corruptPaths == null && active.frameCount() == 0 && active.baseSeq() == chainEnd) { + // Rotation committed (manifest fsync'd, promoted spare's + // header synced) but the process/OS died before a single + // frame of the new active reached disk: the sealed chain + // ends exactly where the empty active begins. Legal + // crash state -- resume appending into it. Refused when + // corrupt segments exist (same stand-in hazard as the + // empty-chain acceptance above). + chain.add(active); + } else { + throw new SfRecoveryException( + "missing expected SF active/tail segment at base " + activeBase); + } + } + // Sealed members must present an all-zero suffix: fresh + // segments are zero-allocated and this recovery sanitizes + // the resumed active before it takes appends, so a reseal + // can never carry residue forward. Reaching this point + // proves every sealed member's frame accounting is complete + // (validateContiguous plus the head/active boundary matching + // above), so any suffix residue here is dead bytes -- legacy + // pre-sanitization poison or an out-of-protocol scribble + // past the last frame, never lost frames. Zero it durably, + // then still fail closed on first sight so the incident is + // surfaced; the restart then proves the chain clean. A tear + // that DID cost frames never gets here: the contiguity or + // boundary checks above throw first, with every byte left + // on disk for operator extraction. + sanitizeSealedResidue(chain, true); + for (int i = 0, n = chain.size(); i < n; i++) { + chain.get(i).markManifestRequired(); } } - // The newest segment becomes the active. Even if it's full, that's OK: - // the next appendOrFsn returns BACKPRESSURE_NO_SPARE, the manager - // installs a hot spare, the producer rotates. Same fast path as a - // mid-life ring. - int last = opened.size() - 1; - MmapSegment active = opened.get(last); - opened.remove(last); - SegmentRing ring = new SegmentRing(active, maxBytesPerSegment); - // Older segments become sealed in baseSeq order. - for (int i = 0, n = opened.size(); i < n; i++) { - ring.sealedSegments.add(opened.get(i)); + + // Build membership and clean validated extras while every mmap and + // the manifest still have one local owner. In particular, an OOM + // while allocating the identity map falls through to the outer + // failure cleanup instead of stranding extras after transfer. + RetainedSegmentMembership retained = newDefaultMembership(chain, membershipObserver); + for (int i = 0, n = all.size(); i < n; i++) { + MmapSegment segment = all.get(i); + if (!retained.contains(segment)) { + String path = null; + long torn = 0; + Throwable inspectionError = null; + try { + path = segment.path(); + torn = segment.tornTailBytes(); + } catch (Throwable error) { + inspectionError = error; + } + // A close failure is not best-effort: keep local ownership + // and fail through the outer cleanup rather than transfer a + // ring while an extra may still own an mmap or fd. + segment.close(); + all.setQuick(i, null); + if (inspectionError != null) { + warnPostRecoveryCleanupFailure(inspectionError); + continue; + } + try { + cleanupClosedExtra(filesFacade, path, torn); + } catch (Throwable cleanupError) { + warnPostRecoveryCleanupFailure(cleanupError); + } + } } - return ring; + quarantineCorrupt(filesFacade, corruptPaths); + + // The resumed active is the only segment that takes appends and + // the only one a later rotation reseals, so recovery zeroes ITS + // residue exactly here -- every chain/manifest check has passed + // and the ring is about to be exposed. Resumed appends restart + // at lastGood but stop wherever the last payload fits, so + // unzeroed residue would survive a seal-via-rotation (bricking + // the next startup's sealed-suffix check) and a byte-aligned + // stale frame with a valid CRC could be resurrected at a + // recycled FSN by a later scan. The durable barrier inside + // sanitizeTornTail is load-bearing in MEMORY durability mode, + // where rotation does not sync the sealed predecessor's data + // pages. A failed barrier aborts recovery (fail closed); the + // retry re-observes the same residue because openExisting never + // mutates. Unlike sealed suffixes, the active's residue is NOT + // proven dead -- past a mid-file tear it can hold valid-CRC + // frames of real unacked payloads. Zeroing is policy: replay + // cannot cross the tear, and preserving the bytes would trade + // the two hazards above for data no recovery path can use (see + // sanitizeTornTail). + active.sanitizeTornTail(); + for (int i = 1, n = chain.size(); i < n; i++) { + chain.get(i - 1).linkSuccessor(chain.get(i)); + } + SegmentRing ring = new SegmentRing(active, maxBytesPerSegment, manifest); + for (int i = 0, n = chain.size() - 1; i < n; i++) { + ring.sealedSegments.add(chain.get(i)); + } + // Allocate the return wrapper before transfer. Until this succeeds, + // the outer catch remains the sole owner and can close all segments + // and the manifest if construction or sealed-list growth fails. + Recovery recovery = Recovery.recovered(ring); + manifest = null; + all.clear(); + return recovery; } catch (Throwable t) { - // Close every recovered MmapSegment that's still in `opened`. - // After the success path, `opened` no longer contains the active - // segment (removed above), but the sealed segments transferred to - // ring.sealedSegments are still owned by the ring once it's - // returned -- so this catch only fires before the return statement. - for (int i = 0, n = opened.size(); i < n; i++) { + for (int i = 0, n = all.size(); i < n; i++) { + MmapSegment segment = all.get(i); + if (segment != null) { + try { + segment.close(); + } catch (Throwable closeError) { + warnRecoveryCloseFailure(closeError); + } + } + } + if (manifest != null) { try { - opened.get(i).close(); - } catch (Throwable closeErr) { - LOG.warn("openExisting: error closing recovered segment during cleanup", - closeErr); + manifest.close(); + } catch (Throwable closeError) { + warnRecoveryCloseFailure(closeError); } } throw t; } } + /** + * Durably advances the manifest head past {@code trimming} (the LAST + * sealed segment of the bounded batch the manager is about to unlink). + * One durable commit covers every earlier batch member: head values are + * segment boundaries and the batch is a contiguous prefix of the sealed + * chain, so recovery discards each member as "stale below head" + * regardless of how far the unlink loop got. The successor and the + * current active are both read under the ring monitor, so a concurrent + * rotation (which also mutates the manifest under this monitor) can never + * make the head leapfrog a still-live sealed segment: if rotation sealed + * the old active after the caller's snapshot, {@code trimming.successor()} + * now points at that sealed segment, not at the new active. + */ + private synchronized void advanceManifestHeadPast(MmapSegment trimming) { + if (manifest == null) { + return; + } + MmapSegment successor = trimming.successor(); + long newHeadBase = (successor == null || successor == active) + ? active.baseSeq() + : successor.baseSeq(); + manifest.update(newHeadBase, active.baseSeq()); + } + + /** + * Picks the clean (untorn) empty segment to reuse as a legacy slot's + * initial active, preferring {@code sf-initial.sfa}. Returns {@code null} + * when no clean empty exists; torn empties are never reused here because + * their bytes are quarantine evidence, not blank space. + */ + private static MmapSegment chooseEmptyInitial(ObjList all, String sfDir) { + String initialPath = sfDir + "/sf-initial.sfa"; + MmapSegment selected = null; + for (int i = 0, n = all.size(); i < n; i++) { + MmapSegment segment = all.get(i); + if (segment.frameCount() != 0 || segment.tornTailBytes() > 0) { + continue; + } + if (selected == null || initialPath.equals(segment.path())) { + selected = segment; + } + } + return selected; + } + + private static void cleanupClosedExtra(FilesFacade filesFacade, String path, long torn) { + if (torn > 0) { + quarantineFile(filesFacade, path); + } else if (!filesFacade.remove(path)) { + LOG.warn("could not remove validated stale/empty SF segment {}", path); + } + } + + private static RetainedSegmentMembership newDefaultMembership( + final ObjList chain, + final MembershipObserver observer + ) { + if (observer != null) { + observer.beforeMembershipAllocation(); + } + if (DEFAULT_MEMBERSHIP_MODE == RetainedSegmentMembershipMode.LINEAR) { + if (observer == null) { + return new RetainedSegmentMembership() { + @Override + public boolean contains(MmapSegment segment) { + for (int i = 0, n = chain.size(); i < n; i++) { + if (chain.get(i) == segment) { + return true; + } + } + return false; + } + }; + } + return new RetainedSegmentMembership() { + @Override + public boolean contains(MmapSegment segment) { + for (int i = 0, n = chain.size(); i < n; i++) { + observer.onMembershipOperation(); + if (chain.get(i) == segment) { + return true; + } + } + return false; + } + }; + } + + final IdentityHashMap retained = new IdentityHashMap<>(chain.size()); + for (int i = 0, n = chain.size(); i < n; i++) { + retained.put(chain.get(i), Boolean.TRUE); + } + if (observer == null) { + return new RetainedSegmentMembership() { + @Override + public boolean contains(MmapSegment segment) { + return retained.containsKey(segment); + } + }; + } + return new RetainedSegmentMembership() { + @Override + public boolean contains(MmapSegment segment) { + observer.onMembershipOperation(); + return retained.containsKey(segment); + } + }; + } + + /** Renames every collected corrupt path to {@code .corrupt}, best-effort. */ + private static void quarantineCorrupt(FilesFacade filesFacade, ObjList corruptPaths) { + if (corruptPaths == null) { + return; + } + for (int i = 0, n = corruptPaths.size(); i < n; i++) { + try { + quarantineFile(filesFacade, corruptPaths.get(i)); + } catch (Throwable cleanupError) { + warnPostRecoveryCleanupFailure(cleanupError); + } + } + } + + private static void quarantineFile(FilesFacade filesFacade, String path) { + if (filesFacade.rename(path, path + ".corrupt") != 0) { + LOG.warn("could not quarantine {} to {}.corrupt; it will be re-examined " + + "on the next recovery", path, path); + } + } + + /** + * Locates the segment the manifest's {@code activeBase} refers to. + * Preference order among same-base candidates: + *

      + *
    1. a segment with recovered frames (the durable chain tail);
      2. + *
    2. an empty segment with a torn tail (the promoted active whose + * first frame write was cut short — an attempted write marks it as + * the one rotation actually exposed);
      3. + *
    3. a clean empty segment.
      4. + *
    + * Multiple equivalent empties at the same base are NOT an error: a fresh + * start or a rotation crash routinely leaves both the initial/promoted + * segment and a provisioned hot spare carrying the same provisional + * baseSeq. They are interchangeable blanks — pick one deterministically + * and let the extras cleanup discard the rest. Bricking startup on this + * state would turn every "kill -9 shortly after start" into a manual + * repair. + */ + private static MmapSegment findActive(ObjList all, long activeBase) { + MmapSegment tornEmpty = null; + MmapSegment cleanEmpty = null; + for (int i = 0, n = all.size(); i < n; i++) { + MmapSegment segment = all.get(i); + if (segment.baseSeq() != activeBase) { + continue; + } + if (segment.frameCount() > 0) { + return segment; + } + if (segment.tornTailBytes() > 0) { + if (tornEmpty == null) { + tornEmpty = segment; + } + } else if (cleanEmpty == null) { + cleanEmpty = segment; + } + } + return tornEmpty != null ? tornEmpty : cleanEmpty; + } + + private static void validateContiguous(ObjList segments) { + for (int i = 1, n = segments.size(); i < n; i++) { + MmapSegment previous = segments.get(i - 1); + MmapSegment current = segments.get(i); + long expected = previous.baseSeq() + previous.frameCount(); + if (current.baseSeq() != expected) { + throw new SfRecoveryException("FSN gap in recovered segments: expected " + + expected + " but got " + current.baseSeq()); + } + } + } + + /** + * Durably zeroes torn-tail residue on the chain's sealed members (every + * element but the last, which is the active). Callers must have already + * proven each sealed member's frame accounting complete -- contiguity + * plus head/active boundary matching -- which is exactly what makes the + * residue provably dead: a tear that cost frames breaks those checks and + * fails recovery before any mutation, preserving the bytes (potentially + * the only copy of unreachable valid-CRC frames) for operator + * extraction. With {@code failClosedOnSight} the incident is still + * surfaced as a first-sight {@link SfSanitizedResidueException} after + * sanitizing: the residue is already durably zeroed when it propagates, + * so a retry proves the chain clean (attended callers get that via + * restart; unattended callers key off the distinct type to retry + * instead of quarantining a just-healed slot). Without the flag the + * chain proceeds immediately (legacy migration, which predates the + * sealed-suffix contract). + */ + private static void sanitizeSealedResidue(ObjList chain, boolean failClosedOnSight) { + String firstTornPath = null; + for (int i = 0, n = chain.size() - 1; i < n; i++) { + MmapSegment sealed = chain.get(i); + if (sealed.tornTailBytes() > 0) { + sealed.sanitizeTornTail(); + if (firstTornPath == null) { + firstTornPath = sealed.path(); + } + } + } + if (failClosedOnSight && firstTornPath != null) { + throw new SfSanitizedResidueException("corrupt torn tail in sealed SF segment " + firstTornPath); + } + } + + private static void warnPostRecoveryCleanupFailure(Throwable cleanupError) { + try { + LOG.warn("post-recovery cleanup failed; leftover files will be " + + "re-examined on the next startup", cleanupError); + } catch (Throwable ignored) { + // Cleanup diagnostics must not invalidate a recovered ring. + } + } + + private static void warnRecoveryCloseFailure(Throwable closeError) { + try { + LOG.warn("error closing SF resource after recovery failure", closeError); + } catch (Throwable ignored) { + // Preserve the original recovery failure and continue closing. + } + } + + interface MembershipObserver { + void beforeMembershipAllocation(); + + void onMembershipOperation(); + } + + static final class Recovery { + private final SegmentRing ring; + private final RecoveryStatus status; + + private Recovery(RecoveryStatus status, SegmentRing ring) { + this.status = status; + this.ring = ring; + } + + static Recovery empty() { + return new Recovery(RecoveryStatus.EMPTY, null); + } + + SegmentRing ring() { + return ring; + } + + static Recovery recovered(SegmentRing ring) { + return new Recovery(RecoveryStatus.RECOVERED, ring); + } + + RecoveryStatus status() { + return status; + } + } + + enum RecoveryStatus { + EMPTY, + RECOVERED + } + + interface RetainedSegmentMembership { + boolean contains(MmapSegment segment); + } + + private enum RetainedSegmentMembershipMode { + IDENTITY, + LINEAR + } + /** * Highest FSN that the server has ACK'd. Read by the segment manager to * decide which sealed segments are safe to munmap + unlink. @@ -367,6 +934,7 @@ public boolean acknowledge(long seq) { * {@link #needsHotSpare}) to prepare the next spare. */ public long appendOrFsn(long payloadAddr, int payloadLen) { + checkDurability(); long offset = active.tryAppend(payloadAddr, payloadLen); if (offset == -1L) { // Active is full. Try to rotate. @@ -374,20 +942,61 @@ public long appendOrFsn(long payloadAddr, int payloadLen) { if (spare == null) { return BACKPRESSURE_NO_SPARE; } + // Periodic mode must make the predecessor's complete published + // range durable before the manifest can name its successor. The + // manager performs the barrier; the producer uses the existing + // backpressure path while it waits. + MmapSegment previous = active; + if (requestSyncBeforeRotation(previous)) { + wakeManager(); + return BACKPRESSURE_NO_SPARE; + } // Pin the spare's baseSeq to whatever the active's nextSeq actually // is right now. This is the right moment because (a) the active is // full, so its frameCount is stable, and (b) the spare hasn't been // appended to yet (rebaseSeq enforces that). The segment manager's // earlier guess at baseSeq is irrelevant. - long actualBase = active.baseSeq() + active.frameCount(); + long actualBase = previous.baseSeq() + previous.frameCount(); spare.rebaseSeq(actualBase); - // Mutate sealedSegments under the same monitor used by - // snapshotSealedSegments -- the I/O thread reads through that - // path and must not see a half-resized ObjList. + if (manifest != null) { + // Make the spare's rebased identity durable BEFORE the manifest + // references it. Without this barrier an OS crash could leave a + // durable manifest pointing at baseSeq=actualBase while the + // spare's on-disk header still carries the manager's + // provisional guess -- recovery would then find no segment at + // the committed active boundary and fail a startup that lost + // nothing. One msync per rotation, amortized over a whole + // segment of appends; runs outside the monitor because the + // spare is not yet visible to any other thread. + // + // MEMORY mode deliberately does not sync the predecessor's + // data pages. PERIODIC mode reached this point only after the + // manager covered the predecessor's complete published range. + spare.syncHeader(); + } + // Publish the successor before the volatile active promotion. The + // same monitor protects the sealed list and nextSealedAfter's trim + // fallback, while the volatile link also remains readable from a + // current segment after the manager removes and closes it. synchronized (this) { - sealedSegments.add(active); + if (manifest != null) { + // Inside the monitor: serialized with the trim path's + // advanceManifestHeadPast so neither writer publishes a + // boundary computed from a state the other has already + // moved past (SfManifest additionally clamps monotonic). + // BEFORE any ring mutation: if the manifest fsync throws, + // the rotation never happened -- previous stays active, + // the spare stays installed, and the producer's retry + // re-runs this block from a consistent state. + long headBase = sealedHead < sealedSegments.size() + ? sealedSegments.get(sealedHead).baseSeq() + : previous.baseSeq(); + manifest.update(headBase, actualBase); + } + previous.linkSuccessor(spare); + sealedSegments.add(previous); + active = spare; } - active = spare; hotSpare = null; // Fresh active just consumed the spare → ask the manager to start // making the next one immediately, before this segment fills. @@ -422,6 +1031,67 @@ public long appendOrFsn(long payloadAddr, int payloadLen) { return fsn; } + public void checkDurability() { + MmapSegmentException failure = durabilityFailure; + if (failure != null) { + throw failure; + } + } + + /** + * Clears a latched periodic data-barrier failure. Called by the segment + * manager after a subsequent periodic sync pass over every live segment + * succeeds: the published range the failed barrier was meant to cover is + * durable now, so the producer may resume. The monitor serializes the + * clear with {@link #recordDurabilityFailure(Throwable)}; the volatile + * write publishes it to producer threads calling {@link #checkDurability()}. + */ + void clearDurabilityFailure() { + if (durabilityFailure != null) { + synchronized (this) { + durabilityFailure = null; + } + } + } + + synchronized void clearSyncRequestIfActiveDurable() { + if (active != null && active.isPublishedDurable()) { + syncRequested = false; + } + } + + synchronized void copyLiveSegmentsForSync(ObjList target) { + target.clear(); + for (int i = sealedHead, n = sealedSegments.size(); i < n; i++) { + target.add(sealedSegments.get(i)); + } + if (active != null) { + target.add(active); + } + } + + void enablePeriodicSync() { + periodicSyncEnabled = true; + syncRequested = true; + } + + boolean isSyncRequested() { + return syncRequested; + } + + void recordDurabilityFailure(Throwable failure) { + if (durabilityFailure == null) { + MmapSegmentException wrapped = failure instanceof MmapSegmentException + ? (MmapSegmentException) failure + : new MmapSegmentException("periodic SF data sync failed", failure); + synchronized (this) { + if (durabilityFailure == null) { + durabilityFailure = wrapped; + } + } + } + } + @Override public synchronized void close() { // Marking closed BEFORE freeing fields ensures any concurrent @@ -431,6 +1101,7 @@ public synchronized void close() { // / firstSealed / findSegmentContaining, so they don't iterate // half-freed state. closed = true; + ioPinnedSegment = null; if (active != null) { active.close(); active = null; @@ -439,13 +1110,18 @@ public synchronized void close() { hotSpare.close(); hotSpare = null; } - for (int i = 0, n = sealedSegments.size(); i < n; i++) { - MmapSegment s = sealedSegments.get(i); - if (s != null) { - s.close(); - } + for (int i = sealedHead, n = sealedSegments.size(); i < n; i++) { + sealedSegments.get(i).close(); } sealedSegments.clear(); + sealedHead = 0; + for (int i = 0, n = pendingTrims.size(); i < n; i++) { + pendingTrims.getQuick(i).close(); + } + pendingTrims.clear(); + if (manifest != null) { + manifest.close(); + } } /** @@ -461,10 +1137,11 @@ public synchronized ObjList drainTrimmable() { ObjList out = null; // Sealed segments are in baseSeq order, oldest first; once we hit one // that isn't fully acked, none of the later ones can be either. - // Synchronized so the I/O thread's snapshotSealedSegments() can't - // race against the remove(0) shuffling slots underneath it. - while (sealedSegments.size() > 0) { - MmapSegment s = sealedSegments.get(0); + while (sealedHead < sealedSegments.size()) { + MmapSegment s = sealedSegments.get(sealedHead); + if (s == ioPinnedSegment) { + break; + } long lastSeq = s.baseSeq() + s.frameCount() - 1; if (lastSeq > acked) { break; @@ -473,7 +1150,7 @@ public synchronized ObjList drainTrimmable() { out = new ObjList<>(); } out.add(s); - sealedSegments.remove(0); + removeSealedHead(); } return out; } @@ -485,26 +1162,26 @@ public synchronized ObjList drainTrimmable() { * to position the I/O thread's cursor at the first unacked frame for * replay. *

    - * Walks sealed first (oldest → newest) then the active. The sealed list - * is small enough -- and reconnects are rare enough -- that the linear - * scan cost doesn't matter. + * Binary-searches the sealed list, then falls back to the active + * segment: O(log live-sealed) per call, so repositioning stays cheap + * even when a producer-outpaces-drain backlog has grown the sealed + * list without bound. */ public synchronized MmapSegment findSegmentContaining(long fsn) { - for (int i = 0, n = sealedSegments.size(); i < n; i++) { - MmapSegment s = sealedSegments.get(i); - long base = s.baseSeq(); - if (fsn >= base && fsn < base + s.frameCount()) { - return s; - } - } - MmapSegment a = active; - if (a != null) { - long base = a.baseSeq(); - if (fsn >= base && fsn < base + a.frameCount()) { - return a; - } + return findSegmentContaining0(fsn); + } + + /** + * Atomically finds and pins the segment containing {@code fsn}. The pin + * remains until the I/O cursor switches or releases it, preventing trim + * staging from hiding or unmapping the returned segment meanwhile. + */ + synchronized MmapSegment pinSegmentContaining(long fsn) { + MmapSegment segment = findSegmentContaining0(fsn); + if (segment != null) { + switchIoPin(segment); } - return null; + return segment; } /** @@ -513,7 +1190,25 @@ public synchronized MmapSegment findSegmentContaining(long fsn) { * fallback -- see {@link #nextSealedAfter(MmapSegment)}. */ public synchronized MmapSegment firstSealed() { - return sealedSegments.size() > 0 ? sealedSegments.get(0) : null; + return sealedHead < sealedSegments.size() ? sealedSegments.get(sealedHead) : null; + } + + /** + * Returns the oldest fully acknowledged, unpinned live segment without + * removing it. Staging later transfers it to hidden ring-owned cleanup + * state, where unlink/barrier failures remain retryable without exposing + * an unmapped segment to traversal. + */ + public synchronized MmapSegment firstTrimmable() { + if (sealedHead == sealedSegments.size()) { + return null; + } + MmapSegment segment = sealedSegments.get(sealedHead); + if (segment == ioPinnedSegment) { + return null; + } + long lastSeq = segment.baseSeq() + segment.frameCount() - 1; + return lastSeq <= ackedFsn ? segment : null; } /** Active segment -- exposed for the I/O thread's "send next batch" path. */ @@ -536,7 +1231,7 @@ public synchronized MmapSegment firstSealed() { */ public synchronized long findLastFsnWithoutPayloadFlag(int flagsOffset, int flagMask, int headerMagic, int minPayloadLen) { long best = -1L; - for (int i = 0, n = sealedSegments.size(); i < n; i++) { + for (int i = sealedHead, n = sealedSegments.size(); i < n; i++) { long fsn = sealedSegments.get(i).findLastFrameFsnWithoutPayloadFlag(flagsOffset, flagMask, headerMagic, minPayloadLen); if (fsn > best) { best = fsn; @@ -550,13 +1245,21 @@ public MmapSegment getActive() { return active; } + /** Atomically pins and returns the current active segment for I/O. */ + synchronized MmapSegment pinActiveSegment() { + MmapSegment segment = active; + switchIoPin(segment); + return segment; + } + /** * Direct view of sealed segments (oldest first). NOT thread-safe -- use * only from the producer thread, or alongside a lock that excludes * concurrent rotation. Cross-thread readers (typically the I/O loop) * should use {@link #snapshotSealedSegments(MmapSegment[])} instead. */ - public ObjList getSealedSegments() { + public synchronized ObjList getSealedSegments() { + compactSealedSegments(); return sealedSegments; } @@ -591,6 +1294,30 @@ public boolean needsHotSpare() { return hotSpare == null; } + /** + * Atomically advances the I/O pin from {@code current} to the next live + * sealed segment, or to the active segment when no sealed successor + * remains. An active cursor stays pinned in place until rotation seals it. + */ + synchronized MmapSegment advancePinnedSegment(MmapSegment current) { + assert ioPinnedSegment == current; + MmapSegment liveActive = active; + if (current == liveActive) { + return current; + } + MmapSegment next = nextSealedAfter(current); + if (next == null) { + MmapSegment first = sealedHead < sealedSegments.size() + ? sealedSegments.get(sealedHead) + : null; + next = first != null && first.baseSeq() > current.baseSeq() + ? first + : liveActive; + } + switchIoPin(next); + return next; + } + /** * Returns the sealed segment whose {@code baseSeq} immediately follows * {@code current.baseSeq()}, or {@code null} if no such segment exists @@ -600,21 +1327,28 @@ public boolean needsHotSpare() { * outpaces the I/O thread and sealed segments accumulate well beyond * any reasonable snapshot-array size. *

    - * Identity match is intentionally avoided: we compare {@code baseSeq} - * so the loop is robust against the case where {@code current} was - * trimmed out from under us (already ACK'd before the I/O thread - * advanced) -- we still return the next segment in baseSeq order rather - * than failing. Synchronized against rotation. + * Each segment publishes its successor once, before rotation exposes that + * successor as active. A constant-time head check detects when trimming + * removed the immediate successor and falls forward to the oldest live + * sealed segment. Synchronized against rotation and head removal. */ public synchronized MmapSegment nextSealedAfter(MmapSegment current) { - long currentBase = current.baseSeq(); - for (int i = 0, n = sealedSegments.size(); i < n; i++) { - MmapSegment s = sealedSegments.get(i); - if (s.baseSeq() > currentBase) { - return s; - } + nextSealedComparisons++; + MmapSegment successor = current.successor(); + if (successor == null) { + return null; } - return null; + if (successor == active) { + return null; + } + MmapSegment first = sealedHead < sealedSegments.size() ? sealedSegments.get(sealedHead) : null; + if (first != null && successor.baseSeq() >= first.baseSeq()) { + return successor; + } + // Head trimming may have removed the immediate successor while the + // I/O cursor still held an older segment. Trims only remove a prefix, + // so the current head is the first live segment after that prefix. + return first; } /** @@ -634,6 +1368,147 @@ public long publishedFsn() { return publishedFsn; } + /** + * Drops a directory-durable prefix from hidden trim ownership and returns + * its exact byte total for manager accounting. + */ + synchronized long commitPendingTrims(MmapSegment[] expected, int count) { + if (count < 0 || count > pendingTrims.size()) { + throw new IllegalArgumentException("invalid pending trim count: " + count); + } + long bytes = 0; + for (int i = 0; i < count; i++) { + MmapSegment segment = pendingTrims.getQuick(i); + if (segment != expected[i]) { + throw new IllegalStateException("pending trim prefix changed"); + } + bytes += segment.sizeBytes(); + } + if (count == pendingTrims.size()) { + pendingTrims.clear(); + } else if (count > 0) { + pendingTrims.remove(0, count - 1); + } + return bytes; + } + + /** Copies the hidden retry batch into manager-thread scratch storage. */ + synchronized int copyPendingTrims(MmapSegment[] target) { + int count = pendingTrims.size(); + if (target.length < count) { + throw new IllegalArgumentException("pending trim target is too small"); + } + for (int i = 0; i < count; i++) { + target[i] = pendingTrims.getQuick(i); + } + return count; + } + + /** Number of hidden trim entries retained for retry. */ + synchronized int pendingTrimCount() { + return pendingTrims.size(); + } + + @TestOnly + public synchronized MmapSegment getHotSpareForTesting() { + return hotSpare; + } + + @TestOnly + public synchronized int getPendingTrimCount() { + return pendingTrims.size(); + } + + @TestOnly + public synchronized MmapSegment pinSegmentContainingForTest(long fsn) { + return pinSegmentContaining(fsn); + } + + @TestOnly + public void recordDurabilityFailureForTesting(Throwable failure) { + recordDurabilityFailure(failure); + } + + @TestOnly + public synchronized void releasePinnedSegmentForTest(MmapSegment expected) { + releasePinnedSegment(expected); + } + + private synchronized boolean requestSyncBeforeRotation(MmapSegment previous) { + if (periodicSyncEnabled && !previous.isPublishedDurable()) { + syncRequested = true; + return true; + } + return false; + } + + /** Releases the I/O cursor pin and wakes trim if it still names {@code expected}. */ + synchronized void releasePinnedSegment(MmapSegment expected) { + if (ioPinnedSegment == expected) { + ioPinnedSegment = null; + wakeManager(); + } + } + + /** + * Commits removal of the segment returned by {@link #firstTrimmable()}. + * Returns false if concurrent lifecycle activity changed the head. + */ + public synchronized boolean removeTrimmable(MmapSegment segment) { + if (sealedHead == sealedSegments.size() + || sealedSegments.get(sealedHead) != segment + || segment == ioPinnedSegment) { + return false; + } + long lastSeq = segment.baseSeq() + segment.frameCount() - 1; + if (lastSeq > ackedFsn) { + return false; + } + removeSealedHead(); + return true; + } + + /** + * Durably advances the manifest past, then hides, one bounded ACKed and + * unpinned sealed prefix. Once this returns, I/O lookup cannot discover + * any staged segment; the manager owns only the physical cleanup attempt. + */ + synchronized int stagePendingTrims(MmapSegment[] target, int maxCount, long coveredAck) { + if (pendingTrims.size() != 0) { + return 0; + } + if (maxCount < 0 || maxCount > MAX_PENDING_TRIMS || target.length < maxCount) { + throw new IllegalArgumentException("invalid trim batch size: " + maxCount); + } + int count = 0; + for (int i = sealedHead, n = sealedSegments.size(); i < n && count < maxCount; i++) { + MmapSegment segment = sealedSegments.get(i); + long lastSeq = segment.baseSeq() + segment.frameCount() - 1L; + if (segment == ioPinnedSegment || lastSeq > coveredAck) { + break; + } + target[count++] = segment; + } + if (count == 0) { + return 0; + } + try { + advanceManifestHeadPast(target[count - 1]); + } catch (Throwable t) { + for (int i = 0; i < count; i++) { + target[i] = null; + } + throw t; + } + for (int i = 0; i < count; i++) { + MmapSegment segment = target[i]; + assert sealedSegments.get(sealedHead) == segment; + pendingTrims.add(segment); + removeSealedHead(); + } + return count; + } + /** * Registers a wakeup callback that the producer thread will invoke when * a hot spare is needed -- either right after a rotation has consumed the @@ -663,17 +1538,21 @@ public void setManagerWakeup(Runnable wakeup) { * the I/O loop is about to do. */ public synchronized int snapshotSealedSegments(MmapSegment[] target) { - int n = sealedSegments.size(); - if (n > target.length) { - for (int i = 0; i < target.length; i++) { - target[i] = sealedSegments.get(i); - } - return -1; + int n = sealedSegments.size() - sealedHead; + int copyCount = Math.min(n, target.length); + for (int i = 0; i < copyCount; i++) { + target[i] = sealedSegments.get(sealedHead + i); } - for (int i = 0; i < n; i++) { - target[i] = sealedSegments.get(i); + return n > target.length ? -1 : n; + } + + public void syncAllLiveSegments() { + ObjList segments = new ObjList<>(); + copyLiveSegmentsForSync(segments); + for (int i = 0, n = segments.size(); i < n; i++) { + segments.getQuick(i).syncPublished(); } - return n; + clearSyncRequestIfActiveDurable(); } /** @@ -689,21 +1568,101 @@ public synchronized long totalSegmentBytes() { if (a != null) total += a.sizeBytes(); MmapSegment hs = hotSpare; if (hs != null) total += hs.sizeBytes(); - for (int i = 0, n = sealedSegments.size(); i < n; i++) { - MmapSegment s = sealedSegments.get(i); - if (s != null) total += s.sizeBytes(); + for (int i = sealedHead, n = sealedSegments.size(); i < n; i++) { + total += sealedSegments.get(i).sizeBytes(); + } + for (int i = 0, n = pendingTrims.size(); i < n; i++) { + total += pendingTrims.getQuick(i).sizeBytes(); } return total; } + private MmapSegment findSegmentContaining0(long fsn) { + // The sealed list is strictly ascending and contiguous in baseSeq: + // rotation seals the predecessor exactly where the promoted spare's + // rebased baseSeq starts, and recovery sorts the chain then rejects + // any gap via validateContiguous. The only sealed segment that can + // cover fsn is therefore the rightmost one whose baseSeq is at or + // below fsn -- binary-search for it (unsigned, matching + // sortByBaseSeq's key order) instead of walking a list that grows + // without bound while the producer outpaces the drain. + int lo = sealedHead; + int hi = sealedSegments.size() - 1; + while (lo <= hi) { + int mid = (lo + hi) >>> 1; + if (Long.compareUnsigned(sealedSegments.get(mid).baseSeq(), fsn) <= 0) { + lo = mid + 1; + } else { + hi = mid - 1; + } + } + if (hi >= sealedHead) { + MmapSegment segment = sealedSegments.get(hi); + long base = segment.baseSeq(); + if (fsn >= base && fsn < base + segment.frameCount()) { + return segment; + } + } + MmapSegment liveActive = active; + if (liveActive != null) { + long base = liveActive.baseSeq(); + if (fsn >= base && fsn < base + liveActive.frameCount()) { + return liveActive; + } + } + return null; + } + + private void compactSealedSegments() { + if (sealedHead > 0) { + int liveCount = sealedSegments.size() - sealedHead; + trimMovedReferences += liveCount; + sealedSegments.remove(0, sealedHead - 1); + sealedHead = 0; + } + } + + private void removeSealedHead() { + sealedSegments.setQuick(sealedHead++, null); + int size = sealedSegments.size(); + if (sealedHead == size) { + sealedSegments.clear(); + sealedHead = 0; + } else if (sealedHead >= 64 && sealedHead >= size - sealedHead) { + compactSealedSegments(); + } + } + + private void switchIoPin(MmapSegment segment) { + MmapSegment previous = ioPinnedSegment; + ioPinnedSegment = segment; + if (previous != null && previous != segment) { + wakeManager(); + } + } + + private void wakeManager() { + Runnable wakeup = managerWakeup; + if (wakeup != null) { + wakeup.run(); + } + } + + /** Returns the sealed-list operation count used by traversal tests. */ + @TestOnly + public static long getNextSealedComparisons() { + return nextSealedComparisons; + } + /** * Returns the cumulative count of baseSeq comparisons performed by * {@link #sortByBaseSeq} since the last {@link #resetSortComparisons()} * (or process start). The count is incremented once per partition pass * for the median-of-three pivot pick plus once per element compared - * against the pivot, so a clean run on N segments adds roughly - * {@code 3 + (hi - lo - 1)} per recursive frame, summing to O(N log N). - * Exposed for {@code SegmentRingTest} to detect O(N²) regressions + * against the pivot ({@code 3 + (hi - lo - 1)} per pass), and by two per + * sift-down level when a range falls back to heapsort, so it strictly + * upper-bounds the true compare count and sums to O(N log N) on every + * input. Exposed for {@code SegmentRingTest} to detect O(N²) regressions * deterministically. */ @TestOnly @@ -711,22 +1670,71 @@ public static long getSortComparisons() { return sortComparisons; } + /** Returns the references moved by sealed-list compaction. */ + @TestOnly + public static long getTrimMovedReferences() { + return trimMovedReferences; + } + + /** Zeroes the counter exposed via {@link #getNextSealedComparisons()}. */ + @TestOnly + public static void resetNextSealedComparisons() { + nextSealedComparisons = 0; + } + /** Zeroes the counter exposed via {@link #getSortComparisons()}. */ @TestOnly public static void resetSortComparisons() { sortComparisons = 0; } + /** Zeroes the counter exposed via {@link #getTrimMovedReferences()}. */ + @TestOnly + public static void resetTrimMovedReferences() { + trimMovedReferences = 0; + } + /** - * In-place quicksort over {@code list[lo, hi)} keyed by ascending - * {@code baseSeq}. Median-of-three pivot avoids the pathological O(N²) - * on already-sorted input that lexicographic readdir produces (our - * filenames are zero-padded hex of {@code baseSeq}). Recursion depth is - * bounded by ~2 log₂(N) -- for the documented 16K-segment ceiling, well - * under the JVM default stack. + * Drives the recovery-time baseSeq sort over the whole list. Exposed so + * {@code SegmentRingTest} can feed adversarial orders (organ-pipe, mass + * duplicates, median-of-three killer, unsigned-boundary keys) straight + * into the sort and assert comparison bounds without staging thousands + * of segment files on disk. + */ + @TestOnly + public static void sortByBaseSeqForTest(ObjList list) { + sortByBaseSeq(list, 0, list.size()); + } + + /** + * In-place introsort over {@code list[lo, hi)} keyed by ascending + * unsigned {@code baseSeq}. Median-of-three quicksort handles the readdir + * orders a healthy slot produces (lexicographic enumeration of the + * generation-numbered filenames yields already-sorted baseSeqs; hashed + * directory order is effectively random), and a partition-pass budget of + * 2·⌊log₂(N)⌋ demotes any range that keeps splitting badly to in-place + * heapsort. Without that budget, Lomuto with a median-of-three pivot is + * O(N²) on organ-pipe, duplicate-heavy and median-of-three-killer orders + * -- reachable only through corrupted-yet-parseable or operator-copied + * headers, but at the documented 16K-segment ceiling that is 10⁷..10⁸ + * comparisons of startup stall before recovery validation gets to + * reject the slot, so the fallback makes O(N log N) unconditional. + * Recursion depth stays under log₂(N) (recurse on the smaller side, + * loop on the larger), well within the JVM default stack. */ private static void sortByBaseSeq(ObjList list, int lo, int hi) { + int n = hi - lo; + if (n > 1) { + sortByBaseSeq(list, lo, hi, 2 * (31 - Integer.numberOfLeadingZeros(n))); + } + } + + private static void sortByBaseSeq(ObjList list, int lo, int hi, int budget) { while (hi - lo > 1) { + if (budget-- == 0) { + heapSortByBaseSeq(list, lo, hi); + return; + } int mid = (lo + hi) >>> 1; long a = list.get(lo).baseSeq(); long b = list.get(mid).baseSeq(); @@ -756,17 +1764,61 @@ private static void sortByBaseSeq(ObjList list, int lo, int hi) { } swap(list, store, hi - 1); // Recurse on the smaller partition; loop on the larger to keep - // recursion depth bounded by log₂(N). + // recursion depth bounded by log₂(N). Children inherit the + // remaining pass budget: it counts passes along a root-to-leaf + // path, so a chain of bad splits exhausts it after ~2 log₂(N) + // levels no matter how the work is divided. if (store - lo < hi - store - 1) { - sortByBaseSeq(list, lo, store); + sortByBaseSeq(list, lo, store, budget); lo = store + 1; } else { - sortByBaseSeq(list, store + 1, hi); + sortByBaseSeq(list, store + 1, hi, budget); hi = store; } } } + /** + * In-place heapsort over {@code list[lo, hi)} keyed by ascending unsigned + * {@code baseSeq}: the introsort fallback for ranges whose partition-pass + * budget ran out. Guaranteed O(N log N) for any key distribution and any + * initial order; no allocation. + */ + private static void heapSortByBaseSeq(ObjList list, int lo, int hi) { + int n = hi - lo; + for (int root = (n >>> 1) - 1; root >= 0; root--) { + siftDownByBaseSeq(list, lo, root, n); + } + for (int end = n - 1; end > 0; end--) { + swap(list, lo, lo + end); + siftDownByBaseSeq(list, lo, 0, end); + } + } + + private static void siftDownByBaseSeq(ObjList list, int lo, int root, int heapSize) { + while (true) { + int child = (root << 1) + 1; + if (child >= heapSize) { + return; + } + // At most two unsigned compares per level (sibling pick + parent + // test); bump the counter by the constant 2 up front -- same + // cheap-upper-bound convention as the partition pass. + sortComparisons += 2; + if (child + 1 < heapSize + && Long.compareUnsigned(list.get(lo + child).baseSeq(), + list.get(lo + child + 1).baseSeq()) < 0) { + child++; + } + if (Long.compareUnsigned(list.get(lo + root).baseSeq(), + list.get(lo + child).baseSeq()) >= 0) { + return; + } + swap(list, lo + root, lo + child); + root = child; + } + } + private static void swap(ObjList list, int i, int j) { if (i == j) return; MmapSegment tmp = list.get(i); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderConnectionDispatcher.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderConnectionDispatcher.java index 2dec7668..541f7dcf 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderConnectionDispatcher.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderConnectionDispatcher.java @@ -27,6 +27,7 @@ import io.questdb.client.SenderConnectionEvent; import io.questdb.client.SenderConnectionListener; import io.questdb.client.std.QuietCloseable; +import org.jetbrains.annotations.TestOnly; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -146,6 +147,11 @@ public long getTotalDelivered() { return totalDelivered.get(); } + @TestOnly + public Thread getWorkerThreadForTesting() { + return dispatcherThread; + } + /** * Non-blocking enqueue. Always admits the new event unless the dispatcher * is closed or {@code event} is null. Returns {@code true} when the new diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderErrorDispatcher.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderErrorDispatcher.java index aaa0e4f9..fb796712 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderErrorDispatcher.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderErrorDispatcher.java @@ -27,6 +27,7 @@ import io.questdb.client.SenderError; import io.questdb.client.SenderErrorHandler; import io.questdb.client.std.QuietCloseable; +import org.jetbrains.annotations.TestOnly; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -209,6 +210,11 @@ public long getTotalDelivered() { return totalDelivered.get(); } + @TestOnly + public Thread getWorkerThreadForTesting() { + return dispatcherThread; + } + /** * True if at least one error has been delivered to a user-installed * (non-default) handler since this dispatcher started. Used by diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderProgressDispatcher.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderProgressDispatcher.java index 2c0e90a1..9e01a630 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderProgressDispatcher.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SenderProgressDispatcher.java @@ -26,6 +26,7 @@ import io.questdb.client.SenderProgressHandler; import io.questdb.client.std.QuietCloseable; +import org.jetbrains.annotations.TestOnly; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -172,6 +173,11 @@ public long getTotalDelivered() { return totalDelivered.get(); } + @TestOnly + public Thread getWorkerThreadForTesting() { + return dispatcherThread; + } + /** * Replace the user-supplied handler. Effective immediately for any * subsequent delivery. Pass {@code null} to install the no-op default. diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SfManifest.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SfManifest.java new file mode 100644 index 00000000..b6afda99 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SfManifest.java @@ -0,0 +1,308 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.std.Crc32c; +import io.questdb.client.std.FilesFacade; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.QuietCloseable; +import io.questdb.client.std.Unsafe; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Crash-safe boundary record for an SF segment chain. Two fixed-size, + * independently CRC-protected records at offsets 0 and 4096 alternate on + * update. Recovery selects the valid record with the greatest generation. + * The separate 4 KiB slots prevent one aligned 512-byte or 4 KiB sector tear + * from erasing both the update and the previous committed boundary. + *

    + * Public within this exported-internal package (like {@link MmapSegment} and + * {@link SegmentRing}) so the boundary contract -- notably the monotonic + * clamp in {@link #update} -- can be pinned by direct unit tests; it is not + * part of the supported client API. + */ +public final class SfManifest implements QuietCloseable { + static final String FILE_NAME = "sf-manifest.bin"; + private static final Logger LOG = LoggerFactory.getLogger(SfManifest.class); + private static final int CRC_OFFSET = 60; + private static final long FILE_SIZE = 8 * 1024; + private static final int MAGIC = 0x314d4653; // SFM1 little-endian + private static final int RECORD_SIZE = 64; + private static final int RECORD_SLOT_SIZE = 4 * 1024; + private static final int VERSION = 1; + private final int fd; + private final FilesFacade filesFacade; + private final String path; + // Preallocated record scratch for update(): the trim path calls update() + // once per batch and must not malloc/free per call. Guarded by this + // object's monitor (update() and close() are both synchronized). + private final long writeScratch; + private long activeBase; + private boolean closed; + private long generation; + private long headBase; + + private SfManifest(FilesFacade filesFacade, String path, int fd, + long generation, long headBase, long activeBase) { + this.filesFacade = filesFacade; + this.path = path; + this.fd = fd; + this.generation = generation; + this.headBase = headBase; + this.activeBase = activeBase; + this.writeScratch = Unsafe.malloc(RECORD_SIZE, MemoryTag.NATIVE_DEFAULT); + } + + public static SfManifest create(FilesFacade filesFacade, String dir, long headBase, long activeBase) { + String path = dir + "/" + FILE_NAME; + int fd = filesFacade.openRWExclusive(path); + if (fd < 0) { + throw new MmapSegmentException("exclusive create failed for SF manifest " + path); + } + boolean success = false; + SfManifest manifest = null; + try { + if (!filesFacade.allocate(fd, FILE_SIZE)) { + throw new MmapSegmentException("could not allocate SF manifest " + path); + } + manifest = new SfManifest(filesFacade, path, fd, 0, -1, -1); + manifest.update(headBase, activeBase); + if (filesFacade.fsyncDir(dir) != 0) { + throw new MmapSegmentException("could not sync SF manifest directory " + dir); + } + success = true; + return manifest; + } finally { + if (!success) { + if (manifest != null) { + // close() frees the constructor-owned scratch buffer as + // well as the fd; closing only the raw fd would leak it. + manifest.close(); + } else { + filesFacade.close(fd); + } + filesFacade.remove(path); + } + } + } + + public static SfManifest open(FilesFacade filesFacade, String dir) { + String path = dir + "/" + FILE_NAME; + if (!filesFacade.exists(path)) { + return null; + } + if (filesFacade.length(path) != FILE_SIZE) { + // A wrong-sized manifest is creation debris: create() reaches the + // full FILE_SIZE via allocate() before writing the first record, + // so a mis-sized file proves the crash happened before any + // boundary was ever committed — nothing can depend on it yet + // (segment flags are stamped only after create() returns). Treat + // as absent so startup self-heals; genuine post-creation loss is + // still caught by the manifest-required flag check. + quarantineDebris(filesFacade, path, "wrong size " + filesFacade.length(path)); + return null; + } + int fd = filesFacade.openRW(path); + if (fd < 0) { + throw new MmapSegmentException("could not open SF manifest " + path); + } + long buffer = Unsafe.malloc(RECORD_SIZE, MemoryTag.NATIVE_DEFAULT); + try { + Record first = readRecord(filesFacade, fd, buffer, 0); + Record second = readRecord(filesFacade, fd, buffer, RECORD_SLOT_SIZE); + Record selected; + if (first == null) { + selected = second; + } else if (second == null || first.generation > second.generation) { + selected = first; + } else { + selected = second; + } + if (selected == null) { + // No valid record in either slot. create() makes the first + // record durable (write + fsync) before returning, and every + // later update() rewrites only ONE slot — so a torn update + // leaves the sibling record intact. Zero valid records + // therefore proves a creation crash, not boundary loss. + // Self-heal by treating the file as absent. If durable state + // DID depend on a manifest (flags stamped, i.e. double-slot + // bit rot), recovery still fails closed on the + // manifest-required flag check. + filesFacade.close(fd); + // Ownership released: quarantineDebris may throw (when both + // rename and remove fail) and the catch below must not close + // this fd again -- the OS may already have handed the number + // to another thread, and a double-close would silently kill + // an unrelated descriptor. + fd = -1; + quarantineDebris(filesFacade, path, "no valid CRC-protected record"); + return null; + } + return new SfManifest(filesFacade, path, fd, selected.generation, + selected.headBase, selected.activeBase); + } catch (Throwable t) { + if (fd != -1) { + filesFacade.close(fd); + } + throw t; + } finally { + Unsafe.free(buffer, RECORD_SIZE, MemoryTag.NATIVE_DEFAULT); + } + } + + public long activeBase() { + return activeBase; + } + + @Override + public synchronized void close() { + // Synchronized against update() so the scratch buffer can never be + // freed under a concurrent writer; update() checks `closed` inside + // the same monitor. + if (!closed) { + closed = true; + Unsafe.free(writeScratch, RECORD_SIZE, MemoryTag.NATIVE_DEFAULT); + filesFacade.close(fd); + } + } + + public long headBase() { + return headBase; + } + + /** + * Unlinks {@code dir}'s manifest file. Used when a slot is being reset to + * the "nothing durable" state (fresh-start cleanup, close-time drain, or + * recovery accepting a segment-less slot as empty). Returns {@code true} + * when the file is confirmed gone (removed, or never existed). + */ + static boolean removeFile(FilesFacade filesFacade, String dir) { + String path = dir + "/" + FILE_NAME; + return filesFacade.remove(path) || !filesFacade.exists(path); + } + + public synchronized void update(long newHeadBase, long newActiveBase) { + if (closed) { + throw new IllegalStateException("SF manifest is closed"); + } + // Committed boundaries only ever move forward: head advances on trim, + // active advances on rotation. Clamp instead of throwing because the + // two writers (producer rotation, manager trim) are serialized on the + // ring monitor but may each compute their argument from a snapshot + // the other has already moved past — e.g. rotation reads the sealed + // list while a trimmed-but-not-yet-removed head segment still sits in + // it. Regressing a durable boundary would let a later crash-recovery + // demand a segment file the trim path already unlinked (startup would + // fail on "missing head segment") or, worse, re-expose stale files + // below an already-committed head. + if (generation > 0) { + if (newHeadBase < headBase) { + newHeadBase = headBase; + } + if (newActiveBase < activeBase) { + newActiveBase = activeBase; + } + } + if (newHeadBase < 0 || newActiveBase < newHeadBase) { + throw new IllegalArgumentException("invalid SF manifest boundaries"); + } + if (generation > 0 && headBase == newHeadBase && activeBase == newActiveBase) { + return; + } + long nextGeneration = generation + 1; + Unsafe.getUnsafe().setMemory(writeScratch, RECORD_SIZE, (byte) 0); + Unsafe.getUnsafe().putInt(writeScratch, MAGIC); + Unsafe.getUnsafe().putInt(writeScratch + 4, VERSION); + Unsafe.getUnsafe().putLong(writeScratch + 8, nextGeneration); + Unsafe.getUnsafe().putLong(writeScratch + 16, newHeadBase); + Unsafe.getUnsafe().putLong(writeScratch + 24, newActiveBase); + int crc = Crc32c.update(Crc32c.INIT, writeScratch, CRC_OFFSET); + Unsafe.getUnsafe().putInt(writeScratch + CRC_OFFSET, crc); + long offset = (nextGeneration & 1L) * RECORD_SLOT_SIZE; + if (filesFacade.write(fd, writeScratch, RECORD_SIZE, offset) != RECORD_SIZE) { + throw new MmapSegmentException("short write updating SF manifest " + path); + } + if (filesFacade.fsync(fd) != 0) { + throw new MmapSegmentException("could not sync SF manifest " + path); + } + generation = nextGeneration; + headBase = newHeadBase; + activeBase = newActiveBase; + } + + /** + * Moves creation-crash debris aside so a subsequent exclusive create can + * succeed. Prefers rename (keeps the bytes for postmortem); falls back to + * remove; throws when neither works — leaving the debris in place would + * wedge every subsequent {@link #create}. + */ + private static void quarantineDebris(FilesFacade filesFacade, String path, String reason) { + LOG.warn("SF manifest {} is creation-crash debris ({}); quarantining and starting " + + "from the segment files", path, reason); + if (filesFacade.rename(path, path + ".corrupt") == 0) { + return; + } + if (filesFacade.remove(path)) { + return; + } + throw new MmapSegmentException("could not quarantine invalid SF manifest " + path); + } + + private static Record readRecord(FilesFacade filesFacade, int fd, long buffer, long offset) { + Unsafe.getUnsafe().setMemory(buffer, RECORD_SIZE, (byte) 0); + if (filesFacade.read(fd, buffer, RECORD_SIZE, offset) != RECORD_SIZE) { + return null; + } + if (Unsafe.getUnsafe().getInt(buffer) != MAGIC + || Unsafe.getUnsafe().getInt(buffer + 4) != VERSION) { + return null; + } + int expected = Unsafe.getUnsafe().getInt(buffer + CRC_OFFSET); + int actual = Crc32c.update(Crc32c.INIT, buffer, CRC_OFFSET); + if (expected != actual) { + return null; + } + long generation = Unsafe.getUnsafe().getLong(buffer + 8); + long headBase = Unsafe.getUnsafe().getLong(buffer + 16); + long activeBase = Unsafe.getUnsafe().getLong(buffer + 24); + if (generation <= 0 || headBase < 0 || activeBase < headBase) { + return null; + } + return new Record(generation, headBase, activeBase); + } + + private static final class Record { + private final long activeBase; + private final long generation; + private final long headBase; + + private Record(long generation, long headBase, long activeBase) { + this.generation = generation; + this.headBase = headBase; + this.activeBase = activeBase; + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SfOperationalException.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SfOperationalException.java new file mode 100644 index 00000000..308b99ee --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SfOperationalException.java @@ -0,0 +1,38 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client.sf.cursor; + +/** + * Operational SF setup failure that may clear on a later retry. Extending + * {@link IllegalStateException} preserves the existing caller-visible + * construction-failure contract while giving internal recovery code a typed + * signal that must not permanently quarantine an orphan slot. + */ +public final class SfOperationalException extends IllegalStateException { + + public SfOperationalException(String message) { + super(message); + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SfRecoveryException.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SfRecoveryException.java new file mode 100644 index 00000000..d3f266c3 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SfRecoveryException.java @@ -0,0 +1,44 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client.sf.cursor; + +/** + * Terminal SF recovery failure. The on-disk state proves that the durable + * segment chain is corrupt or incomplete and requires operator intervention. + * Operational filesystem failures continue to use {@link MmapSegmentException} + * so callers can retry them without quarantining otherwise recoverable data. + *

    + * One refinement is deliberately non-terminal for unattended callers: + * {@link SfSanitizedResidueException} marks a first-sight failure thrown + * AFTER recovery durably healed the chain, so a single retry validates + * clean. Catch sites that quarantine on this type must intercept the + * refinement first. + */ +public class SfRecoveryException extends MmapSegmentException { + + public SfRecoveryException(String message) { + super(message); + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SfSanitizedResidueException.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SfSanitizedResidueException.java new file mode 100644 index 00000000..8a58c5b2 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SfSanitizedResidueException.java @@ -0,0 +1,50 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client.sf.cursor; + +/** + * First-sight failure over a chain that recovery has already healed. + * Thrown from the fail-closed branch of the sealed-residue sanitize: frame + * accounting was proven complete (contiguity plus boundary checks), the + * proven-dead suffix residue was durably zeroed ({@code msync} + + * {@code fsync} completed — a sync failure throws + * {@link MmapSegmentException} instead and never reaches this type), and + * the throw exists solely to surface the incident on the startup that + * observed it. An immediate re-open re-runs the same proofs over the + * zeroed suffix and succeeds. + *

    + * Attended callers (producer startup, where an operator or supervisor + * restarts the process) should keep the parent's fail-closed semantics: + * the restart proves the chain clean. Unattended callers (the orphan + * drainer) may retry construction once instead of quarantining — dropping + * a {@code .failed} sentinel over a just-healed slot would strand its + * replayable backlog until an operator clears the sentinel by hand. + */ +public final class SfSanitizedResidueException extends SfRecoveryException { + + public SfSanitizedResidueException(String message) { + super(message); + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java index 0b8379de..17264c5a 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java @@ -29,6 +29,7 @@ import io.questdb.client.std.MemoryTag; import io.questdb.client.std.QuietCloseable; import io.questdb.client.std.Unsafe; +import org.jetbrains.annotations.TestOnly; import java.nio.charset.StandardCharsets; @@ -36,9 +37,9 @@ * Advisory exclusive lock for a single SF slot directory. *

    * One {@code .lock} file per slot, held via {@code flock}/{@code LockFileEx} - * for the entire lifetime of the engine that owns the slot. The lock is - * automatically released when the fd is closed — including on hard process - * exit, since the kernel cleans up file locks for terminated processes. + * for the entire lifetime of the engine that owns the slot. Normal teardown + * explicitly unlocks it before closing the fd; hard process exit remains a + * backstop because the kernel cleans up file locks for terminated processes. *

    * The holder's PID is written to a sibling {@code .lock.pid} file at * acquisition time. A failed acquisition reads it back so the error message @@ -56,10 +57,15 @@ */ public final class SlotLock implements QuietCloseable { + private static final int DEAD_FD_FOR_TESTING = 1_000_000_000; private static final String LOCK_FILE_NAME = ".lock"; private static final String LOCK_PID_FILE_NAME = ".lock.pid"; + private static final Object RELEASE_RETRY_LOCK = new Object(); + private static SlotLock releaseRetryHead; private final String slotDir; private int fd; + private boolean isReleaseRetryPending; + private SlotLock releaseRetryNext; private SlotLock(String slotDir, int fd) { this.slotDir = slotDir; @@ -71,25 +77,43 @@ private SlotLock(String slotDir, int fd) { * acquires an exclusive {@code flock} on it. On contention, reads the * existing PID payload and throws with a descriptive message. * - * @throws IllegalStateException on dir-create failure, file-open failure, - * or lock contention. + * @throws SfOperationalException on directory or lock-file setup failure + * @throws SlotLockContentionException on lock contention */ public static SlotLock acquire(String slotDir) { + return acquire(slotDir, false); + } + + /** + * Acquires a slot and optionally makes the slot entry durable in its + * parent directory before any segment file is created. + */ + public static SlotLock acquire(String slotDir, boolean syncParentDirectory) { if (slotDir == null || slotDir.isEmpty()) { throw new IllegalArgumentException("slotDir must not be empty"); } + // Construction cleanup may have retained locks after explicit unlock + // failures. Drive every pending owner before opening a new descriptor. + // Path text cannot identify a physical file portably (symlinks and + // Windows case aliases are counterexamples), while the pending list is + // cold, error-only state and normally empty. + retryPendingReleases(); if (!Files.exists(slotDir)) { int rc = Files.mkdir(slotDir, Files.DIR_MODE_DEFAULT); if (rc != 0) { - throw new IllegalStateException( + throw new SfOperationalException( "could not create slot dir: " + slotDir + " rc=" + rc); } } + if (syncParentDirectory && Files.fsyncParentDir(slotDir) != 0) { + throw new SfOperationalException( + "could not sync parent directory for SF slot: " + slotDir); + } String lockPath = slotDir + "/" + LOCK_FILE_NAME; String pidPath = slotDir + "/" + LOCK_PID_FILE_NAME; int fd = Files.openRW(lockPath); if (fd < 0) { - throw new IllegalStateException( + throw new SfOperationalException( "could not open slot lock file: " + lockPath); } boolean ok = false; @@ -97,7 +121,7 @@ public static SlotLock acquire(String slotDir) { int rc = Files.lock(fd); if (rc != 0) { String holder = readHolder(pidPath); - throw new IllegalStateException( + throw new SlotLockContentionException( "sf slot already in use by another process [slot=" + slotDir + ", holder=" + holder + "]"); } @@ -111,19 +135,126 @@ public static SlotLock acquire(String slotDir) { } } + /** + * Side-effect-light contention probe: reports the current holder of the + * slot flock without creating the slot dir or lock file and without + * paying a full engine build. Opens the existing {@code .lock} file + * (absent means nothing can hold a flock on it), try-locks it + * non-blocking, and releases immediately on success. + *

    + * Returns a non-null holder description (the {@code .lock.pid} payload, + * or {@code "unknown"}) when the flock is currently held by a live + * owner; {@code null} when the lock is free or the probe could not + * determine state (missing lock file, open failure). Callers must treat + * {@code null} as "proceed to a full acquire", which owns real error + * classification -- the probe never throws. + *

    + * Races are benign in both directions: a free probe can still lose the + * subsequent acquire to a concurrent owner (the caller handles that + * contention exactly as before), and a held probe that goes stale the + * moment the owner exits is simply re-observed on the caller's next + * cycle. The probe's momentary hold can make a concurrent acquirer see + * spurious contention -- the same class of race two real contenders + * already have. + */ + public static String probeHolder(String slotDir) { + if (slotDir == null || slotDir.isEmpty()) { + return null; + } + // Same pre-step as acquire(): a lock retained by THIS process after + // an unconfirmed unlock would otherwise read as a live holder for as + // long as the retry list carries it. + retryPendingReleases(); + String lockPath = slotDir + "/" + LOCK_FILE_NAME; + if (!Files.exists(lockPath)) { + return null; + } + int fd = Files.openRW(lockPath); + if (fd < 0) { + return null; + } + if (Files.lock(fd) != 0) { + String holder = readHolder(slotDir + "/" + LOCK_PID_FILE_NAME); + Files.close(fd); + return holder; + } + // The flock was free and is momentarily ours. Route the release + // through the standard close() so an unconfirmed unlock is retained + // on the retry list exactly like a normal owner's -- a probe must + // never leak a held flock. + new SlotLock(slotDir, fd).close(); + return null; + } + + /** + * Replaces the live descriptor with a known-dead value until the returned + * guard closes. Test-only: exercises release retry paths without exposing + * mutable descriptor state. + */ + @TestOnly + public synchronized ReleaseFailureForTesting injectReleaseFailureForTesting() { + if (fd < 0 || fd == DEAD_FD_FOR_TESTING) { + throw new IllegalStateException("slot lock is not held by a live descriptor"); + } + ReleaseFailureForTesting releaseFailure = new ReleaseFailureForTesting(fd); + fd = DEAD_FD_FOR_TESTING; + return releaseFailure; + } + + @TestOnly + public synchronized boolean isReleaseFailureInjectedForTesting() { + return fd == DEAD_FD_FOR_TESTING; + } + /** Slot dir this lock guards. */ public String slotDir() { return slotDir; } + /** + * Explicitly releases the flock and reports whether the release was + * confirmed. After a successful unlock the native primitive closes + * the descriptor once, best-effort, and this object forgets its numeric + * value. It never retries that close: POSIX leaves descriptor state + * unspecified after some close failures (notably {@code EINTR}), so a + * retry could close an unrelated descriptor that reused the same number. + * We do NOT remove the {@code .lock} file or {@code .lock.pid} sidecar; a + * stale PID is harmless because the next acquirer overwrites it. + *

    + * When the explicit unlock itself fails, the fd is retained so a later + * attempt can safely retry the non-consuming unlock operation. Idempotent + * once the unlock has succeeded. + *

    + * Owners that gate a "slot dir is reusable" signal on the release + * (e.g. {@code CursorSendEngine.finishClose} publishing + * {@code closeCompleted}) must call this and check the result rather + * than {@link #close()}, which is best-effort by contract. + * + * @return {@code true} if the lock was explicitly released (or was already + * released), {@code false} if the OS reported an unlock failure + */ + public synchronized boolean release() { + if (fd < 0) { + return true; + } + if (release0(fd) == 0) { + fd = -1; + return true; + } + return false; + } + @Override public void close() { - // Closing the fd releases the lock. We do NOT remove the .lock - // file or the .lock.pid sidecar — a stale PID is harmless (next - // acquirer overwrites .lock.pid on success). - if (fd >= 0) { - Files.close(fd); - fd = -1; + // QuietCloseable cannot report a failure, so retain this object on an + // allocation-free retry list when unlock is unconfirmed. Serialize the + // release attempt and publication: an acquire that starts after close + // returns must not miss the retained owner. An acquire already racing + // an in-progress close may still observe ordinary lock contention. + synchronized (RELEASE_RETRY_LOCK) { + if (!release()) { + retainForReleaseRetryLocked(); + } } } @@ -152,6 +283,50 @@ private static String readHolder(String pidPath) { } } + private static native int release0(int fd); + + private void retainForReleaseRetryLocked() { + if (!isReleaseRetryPending) { + isReleaseRetryPending = true; + releaseRetryNext = releaseRetryHead; + releaseRetryHead = this; + } + } + + private synchronized void restoreFdForTesting(int savedFd) { + if (fd != DEAD_FD_FOR_TESTING) { + throw new IllegalStateException("slot lock release failure is not injected"); + } + fd = savedFd; + } + + private static void retryPendingReleases() { + synchronized (RELEASE_RETRY_LOCK) { + SlotLock previous = null; + SlotLock lock = releaseRetryHead; + while (lock != null) { + SlotLock next = lock.releaseRetryNext; + // release() reports operational unlock failure as false. Do + // not catch Error or unexpected programming failures here: + // hiding them as apparent lock contention would misdiagnose + // the process and create a new retry contract for VM errors. + if (lock.release()) { + if (previous == null) { + releaseRetryHead = next; + } else { + previous.releaseRetryNext = next; + } + lock.isReleaseRetryPending = false; + lock.releaseRetryNext = null; + lock = next; + continue; + } + previous = lock; + lock = next; + } + } + } + private static void writePid(String pidPath) { long pid; try { @@ -181,4 +356,22 @@ private static void writePid(String pidPath) { Files.close(wfd); } } + + @TestOnly + public final class ReleaseFailureForTesting implements QuietCloseable { + private final int savedFd; + private boolean isRestored; + + private ReleaseFailureForTesting(int savedFd) { + this.savedFd = savedFd; + } + + @Override + public synchronized void close() { + if (!isRestored) { + restoreFdForTesting(savedFd); + isRestored = true; + } + } + } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLockContentionException.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLockContentionException.java new file mode 100644 index 00000000..6dc98f67 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLockContentionException.java @@ -0,0 +1,37 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client.sf.cursor; + +/** + * Signals that another sender or drainer currently owns an SF slot lock. + * This expected contention condition is distinct from operational failures + * that prevent the lock file from being opened or created. + */ +public final class SlotLockContentionException extends IllegalStateException { + + public SlotLockContentionException(String message) { + super(message); + } +} diff --git a/core/src/main/java/io/questdb/client/impl/ConfigSchema.java b/core/src/main/java/io/questdb/client/impl/ConfigSchema.java index a255e638..9cd18134 100644 --- a/core/src/main/java/io/questdb/client/impl/ConfigSchema.java +++ b/core/src/main/java/io/questdb/client/impl/ConfigSchema.java @@ -88,6 +88,7 @@ public final class ConfigSchema { str("sf_durability", Side.INGRESS); str("sf_max_bytes", Side.INGRESS); str("sf_max_total_bytes", Side.INGRESS); + str("sf_sync_interval_millis", Side.INGRESS); str("transaction", Side.INGRESS); // EGRESS -- the QwpQueryClient applies. Typed where there is a range or diff --git a/core/src/main/java/io/questdb/client/impl/PooledSender.java b/core/src/main/java/io/questdb/client/impl/PooledSender.java index 1379c76c..7b4e5f80 100644 --- a/core/src/main/java/io/questdb/client/impl/PooledSender.java +++ b/core/src/main/java/io/questdb/client/impl/PooledSender.java @@ -32,6 +32,7 @@ import io.questdb.client.std.Decimal64; import io.questdb.client.std.bytes.DirectByteSlice; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.TestOnly; import java.time.Instant; import java.time.temporal.ChronoUnit; @@ -380,6 +381,21 @@ long generation() { return generation; } + @TestOnly + public Sender getDelegateForTesting() { + return slot.delegate(); + } + + @TestOnly + public int getSlotIndexForTesting() { + return slot.slotIndex(); + } + + @TestOnly + public boolean hasSameSlotForTesting(PooledSender that) { + return slot == that.slot; + } + SenderSlot slot() { return slot; } diff --git a/core/src/main/java/io/questdb/client/impl/QueryClientPool.java b/core/src/main/java/io/questdb/client/impl/QueryClientPool.java index 259ff10c..75323a12 100644 --- a/core/src/main/java/io/questdb/client/impl/QueryClientPool.java +++ b/core/src/main/java/io/questdb/client/impl/QueryClientPool.java @@ -27,6 +27,8 @@ import io.questdb.client.QueryException; import io.questdb.client.cutlass.qwp.client.QwpQueryClient; import org.jetbrains.annotations.TestOnly; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.ArrayDeque; import java.util.ArrayList; @@ -56,10 +58,19 @@ public final class QueryClientPool implements AutoCloseable { // close() can never block the caller unbounded. Tunable per pool via // closeQueryTimeoutMillis(long). static final long DEFAULT_CLOSE_QUERY_TIMEOUT_MILLIS = 5_000; + // Hard cap on close()'s wait for a creation blocked in DNS, TCP, TLS, or + // WebSocket setup. A late creator retains the client and native-scratch + // cleanup obligation until it returns and observes the closed pool. + static final long MAX_CLOSE_CREATION_WAIT_MILLIS = 5_000; + private static final Logger LOG = LoggerFactory.getLogger(QueryClientPool.class); private final long acquireTimeoutMillis; private final ArrayList all; private final ArrayDeque available; private final String configurationString; + // Signals completion of internally owned creation lifecycles. Kept + // separate from workerReleased so an acquire waiter cannot consume the only + // wakeup intended for close(). + private final Condition creationFinished; // Test seam. Production connects via QwpQueryClient.connect(); white-box // tests in io.questdb.client.test.impl reach the package-private constructor // by reflection to inject a hook that throws a non-RuntimeException @@ -146,6 +157,7 @@ public QueryClientPool( this.maxLifetimeMillis = maxLifetimeMillis; this.all = new ArrayList<>(maxSize); this.available = new ArrayDeque<>(maxSize); + this.creationFinished = lock.newCondition(); this.workerReleased = lock.newCondition(); int built = 0; // Tracks a worker built by createUnlocked() but not yet added to `all`: @@ -231,16 +243,11 @@ public QueryWorker acquire() { // incremented, permanently shrinking pool capacity until // every acquire() times out. Restoring the reservation // for any throwable is safe. - lock.lock(); - inFlightCreations--; - workerReleased.signal(); - lock.unlock(); // createUnlocked() returns a fully connected client // (socket + native scratch + I/O thread), so if start() // threw afterwards we must close it here -- nothing else - // references it. createUnlocked() already self-cleans - // when connect() throws, leaving created == null, so - // this only fires on the start()-throws path. + // references it. Keep the creation reservation until + // that cleanup finishes so close() cannot return first. if (created != null) { try { created.shutdown(); @@ -249,33 +256,45 @@ public QueryWorker acquire() { // original creation failure rethrown below. } } + lock.lock(); + try { + inFlightCreations--; + creationFinished.signalAll(); + workerReleased.signal(); + } finally { + lock.unlock(); + } throw new QueryException((byte) 0, "failed to create query client: " + e.getMessage(), e); } lock.lock(); - inFlightCreations--; if (closed) { - // Pool was closed mid-creation -- tear the fresh worker - // down rather than leaking it, but OUTSIDE the lock: - // shutdown() joins the dispatch thread for up to - // SHUTDOWN_JOIN_MILLIS, and close()/release()/discard()/ - // cancelIfCurrent() all contend on this lock (whose - // contract is "held only briefly"). The accounting above - // already ran under the lock, and the worker never - // entered `all`, so close()'s snapshot loop cannot race - // this teardown. + // Keep inFlightCreations reserved while shutdown runs + // outside the lock. The worker never entered `all`, so + // this reservation is close()'s sole ownership record. lock.unlock(); try { created.shutdown(); } catch (Throwable ignored) { // Best-effort: an Error from teardown must not mask // the closed-pool signal. + } finally { + lock.lock(); + try { + inFlightCreations--; + creationFinished.signalAll(); + workerReleased.signalAll(); + } finally { + lock.unlock(); + } } throw new QueryException((byte) 0, "QuestDB handle is closed"); } all.add(created); // Stamp the first lease id for this freshly built worker. created.bumpGeneration(); + inFlightCreations--; + creationFinished.signalAll(); return created; } if (remainingNanos <= 0) { @@ -307,6 +326,35 @@ public void close() { } closed = true; workerReleased.signalAll(); + // A creator can block in DNS, TCP, TLS, WebSocket upgrade, or the + // SERVER_INFO handshake. Wait boundedly rather than turning an + // unset connect_timeout into an unbounded QuestDB.close(). Timing + // out does not abandon the client or its native scratch: the creator + // keeps the reservation and, once construction returns, observes + // closed and shuts the worker down before releasing ownership. + // Preserve interruption while applying the same finite budget. + final long creationWaitMillis = Math.max(0, + Math.min(acquireTimeoutMillis, MAX_CLOSE_CREATION_WAIT_MILLIS)); + final long creationWaitNanos = TimeUnit.MILLISECONDS.toNanos(creationWaitMillis); + final long creationWaitDeadlineNanos = System.nanoTime() + creationWaitNanos; + long creationRemainingNanos = creationWaitNanos; + boolean creationWaitInterrupted = false; + while (inFlightCreations > 0 && creationRemainingNanos > 0) { + try { + creationFinished.awaitNanos(creationRemainingNanos); + } catch (InterruptedException e) { + creationWaitInterrupted = true; + } + creationRemainingNanos = creationWaitDeadlineNanos - System.nanoTime(); + } + if (creationWaitInterrupted) { + Thread.currentThread().interrupt(); + } + if (inFlightCreations > 0) { + LOG.warn("QueryClientPool.close(): {} query client creation(s) still in flight after {}ms; " + + "each creator retains cleanup ownership until construction returns", + inFlightCreations, creationWaitMillis); + } snapshot = new ArrayList<>(all); } finally { lock.unlock(); @@ -480,6 +528,16 @@ void release(QueryWorker w, long gen) { } } + @TestOnly + public boolean hasCreationWaiterForTesting() { + lock.lock(); + try { + return lock.hasWaiters(creationFinished); + } finally { + lock.unlock(); + } + } + // White-box accessor for tests: reports the current in-flight creation count // under the pool lock. A non-zero value after a failed acquire() means the // slot reservation was never released -- the capacity-shrink bug this guards @@ -494,6 +552,11 @@ public int inFlightCreations() { } } + @TestOnly + public boolean isClosedForTesting() { + return closed; + } + private QueryWorker createUnlocked() { QwpQueryClient client = QwpQueryClient.fromConfig(configurationString); try { diff --git a/core/src/main/java/io/questdb/client/impl/QuestDBImpl.java b/core/src/main/java/io/questdb/client/impl/QuestDBImpl.java index e3da539b..75ee0a26 100644 --- a/core/src/main/java/io/questdb/client/impl/QuestDBImpl.java +++ b/core/src/main/java/io/questdb/client/impl/QuestDBImpl.java @@ -181,8 +181,20 @@ public Sender borrowSender() { return senderPool.borrow(); } + // synchronized so concurrent close() callers serialize THROUGH the bounded + // shutdown sequence, not merely through the `closed` flip. `closed` is set + // before the teardown chain runs, so a plain volatile guard (or a bare CAS) + // would let a second caller observe closed==true and return while the first + // is still closing published resources. The monitor makes the losing caller + // block until the winner finishes, then it enters, sees `closed` and returns + // a no-op. A pool creator that outlives its finite shutdown wait retains and + // eventually cleans its unpublished resources on the creator thread; this + // monitor does not wait past that budget. No deadlock: the teardown steps + // (markClosing/housekeeper.stop()/queryPool.close()/senderPool.close()) + // never call back into QuestDBImpl.close() on another thread, so nothing + // contends for this monitor from within the critical section. @Override - public void close() { + public synchronized void close() { if (closed) { return; } @@ -203,6 +215,16 @@ public void close() { closeQuietly(senderPool); } + @TestOnly + public QueryClientPool getQueryPoolForTesting() { + return queryPool; + } + + @TestOnly + public SenderPool getSenderPoolForTesting() { + return senderPool; + } + private static void closeQuietly(PoolHousekeeper housekeeper) { if (housekeeper == null) { return; diff --git a/core/src/main/java/io/questdb/client/impl/SenderPool.java b/core/src/main/java/io/questdb/client/impl/SenderPool.java index 1cc214b4..d49a24e1 100644 --- a/core/src/main/java/io/questdb/client/impl/SenderPool.java +++ b/core/src/main/java/io/questdb/client/impl/SenderPool.java @@ -31,6 +31,8 @@ import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; import io.questdb.client.cutlass.qwp.client.sf.cursor.BackgroundDrainerListener; import io.questdb.client.cutlass.qwp.client.sf.cursor.OrphanScanner; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SlotLock; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SlotLockContentionException; import io.questdb.client.std.Files; import io.questdb.client.std.IntList; import org.jetbrains.annotations.TestOnly; @@ -40,6 +42,7 @@ import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Iterator; +import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.ReentrantLock; @@ -85,8 +88,8 @@ public final class SenderPool implements AutoCloseable { // Per-slot wall-clock cap on a single startup-recovery drain. Kept BELOW the // PoolHousekeeper stop/join budget (PoolHousekeeper.STOP_TIMEOUT_MILLIS) so a // recovery drain still in flight when close() arrives cannot outlive the - // housekeeper join -- the residual-budget bound that, together with the - // early markClosing() signal, keeps close() prompt (C1 fix). + // driver join -- the residual-budget bound that, together with the early + // markClosing() signal, keeps close() prompt (C1 fix). // // This caps only the DRAIN. The recovery build that precedes it is bounded // separately: recovery delegates force initial_connect_mode=OFF (see @@ -96,6 +99,27 @@ public final class SenderPool implements AutoCloseable { // -- the residual window documented on recoverOneSlotStep -- because the // transport has no application-level connect timeout to clamp it. private static final long RECOVERY_DRAIN_BUDGET_MILLIS = 1_000; + // The scan parks ONE candidate slot and moves on (C2) on the slot's Nth + // (3rd) consecutive recoverOneSlotStep failure: two retries in place, + // park on the third. The retry-in-place policy + // assumes a failure is server-wide and will repeat for every remaining + // slot, but a slot-specific persistent condition (e.g. a corrupt slot dir + // that fails the recovery build every time) would otherwise pin the cursor + // forever: the driver would livelock retrying that one slot each second + // while every higher-index slot's durable orphan data starves. Small + // enough to bound how long one slot can block the scan; large enough that + // the dominant server-wide transient failure keeps its retry-the-same- + // candidate behavior for the first attempts. + private static final int RECOVERY_MAX_SLOT_FAILURE_STREAK = 3; + // A direct SenderPool has no PoolHousekeeper to retry a transient startup + // recovery failure. Its private driver waits this long between failed + // attempts so an unavailable server does not cause a hot retry loop. + private static final long RECOVERY_RETRY_INTERVAL_MILLIS = 1_000; + // Hard cap on close()'s wait for a creation that is blocked in DNS, TCP, + // TLS, or WebSocket setup. The creator retains ownership after this budget: + // once construction returns, borrow() observes closed and tears down the + // delegate before releasing its reservation (including an SF slot). + static final long MAX_CLOSE_CREATION_WAIT_MILLIS = 5_000; // Hard cap on close()'s outstanding-lease wait. The acquire timeout is a // BORROW policy -- Long.MAX_VALUE legitimately means "block until a slot // frees" -- and must never unbound SHUTDOWN: without this cap a forgotten @@ -106,7 +130,16 @@ public final class SenderPool implements AutoCloseable { private final long acquireTimeoutMillis; private final ArrayList all; private final ArrayDeque available; + // Test-only constructor seam. Runs immediately before a possibly-started + // recovery driver is joined after constructor failure. Null in production; + // lifecycle tests use it to release a deliberately held driver and prove + // delegate cleanup happens only after the join. + private final Runnable beforeFailedStartupRecoveryJoinHook; private final String configurationString; + // Signals completion of internally owned creation lifecycles. Kept + // separate from slotReleased so a capacity waiter cannot consume the only + // wakeup intended for close(). + private final Condition creationFinished; // User-supplied ingest callbacks, shared across every pooled Sender this // pool builds. Null -> each sender keeps its loud-not-silent default. private final SenderConnectionListener connectionListener; @@ -119,15 +152,19 @@ public final class SenderPool implements AutoCloseable { // RuntimeException Throwable (e.g. an -ea AssertionError) mid-prewarm, // exercising the Error-safe delegate cleanup loop. private final IntFunction senderFactory; + // Test seam: runs immediately after a delegate factory returns, before + // listener registration and SenderSlot construction. Null in production; + // error-safety tests inject a preallocated throwable at this ownership gap. + private final Runnable postFactoryHook; // Factory for startup-recovery delegates. Distinct from senderFactory so a // recoverer can force a non-blocking initial connect (initial_connect_mode= - // OFF) regardless of user config: a recovery build runs on the - // PoolHousekeeper thread and must NOT inherit SYNC (auto-enabled by any - // reconnect_* knob), which would retry the connect for the whole reconnect - // budget inside build() -- far past PoolHousekeeper.STOP_TIMEOUT_MILLIS, so - // a close() landing during that build would make housekeeper.stop()'s join - // time out and leave the recoverer holding the slot flock after close() - // returned (M1). Mirrors senderFactory's test seam: an injected factory + // OFF) regardless of user config: a recovery build runs on a private direct + // driver or the PoolHousekeeper thread and must NOT inherit SYNC + // (auto-enabled by any reconnect_* knob), which would retry the connect for + // the whole reconnect budget inside build() -- far past + // PoolHousekeeper.STOP_TIMEOUT_MILLIS, so a close() landing during that + // build could make its driver join time out and leave the recoverer holding + // the slot flock after close() returned (M1). Mirrors senderFactory's test seam: an injected factory // (non-null) drives BOTH paths so white-box recovery tests keep control. private final IntFunction recoverySenderFactory; private final ReentrantLock lock = new ReentrantLock(); @@ -148,6 +185,55 @@ public final class SenderPool implements AutoCloseable { private final Condition slotReleased; // True iff the configuration enables store-and-forward (sf_dir set). private final boolean storeAndForward; + // Direct pools run one inline recovery attempt for backwards-compatible + // startup behavior, then this daemon continues any backlog left by budget + // exhaustion or a transient failure. Deferred pools use PoolHousekeeper and + // leave this null. + private final Object startupRecoverySignal = new Object(); + private final Thread startupRecoveryThread; + // Revived direct-pool recovery driver: spawned by recoverRetiredSlotAt() + // when a retired slot's late flock release re-arms an already-latched + // scan on a pool whose original private driver has exited. Deferred + // pools re-arm through the PoolHousekeeper tick instead and never spawn + // one. Joined by stopStartupRecoveryDriver() on close. Volatile: written + // under `lock`, read by close() without it. + private volatile Thread revivedStartupRecoveryThread; + // Ownership token for the private (direct-pool) recovery driver duty. + // Guarded by `lock`. TRUE while some thread owns runStartupRecoveryLoop; + // the loop drops it transactionally on exit (re-checking the latch under + // the lock first -- see releaseRecoveryDriverOwnership), and + // reviveDirectRecoveryDriverIfNeeded() spawns a successor only when it + // is FALSE. Replaces a Thread.isAlive() heuristic whose read raced the + // driver's final latch check: a driver observed "alive" could already + // have committed to exit, leaving an un-latched scan ownerless until the + // next release event or a restart. + private boolean recoveryDriverRunning; + // Driver policy captured for the re-arm path: deferred pools are driven + // by PoolHousekeeper (never spawn private drivers); direct pools own + // their (revivable) private driver, built via recoveryThreadFactory. + private final boolean deferStartupRecovery; + private final ThreadFactory recoveryThreadFactory; + // Test-only constructor seam for the failed-retry operation. Production + // uses waitForStartupRecoveryRetry(), which performs the one-second signal + // wait; lifecycle tests inject an event barrier without wall-clock checks. + private final Runnable startupRecoveryWaiter; + // Test seam: runs after the direct recovery driver's bounded join returns. + // Null in production; lifecycle tests prove the joined thread is quiescent. + private volatile Runnable afterStartupRecoveryJoinHook; + // Test seam: runs immediately before a capacity-starved borrow enters its + // condition wait, while it still holds the pool lock. Null in production; + // concurrency tests use a latch here to prove that several borrowers have + // all reached the wait path before recovering retired capacity. + private volatile Runnable beforeBorrowWaitHook; + // Test seam: runs immediately before the direct recovery driver's bounded + // join. Null in production; lifecycle tests coordinate close() with a + // deliberately held driver. + private volatile Runnable beforeStartupRecoveryJoinHook; + // Test seam: runs after a capacity-starved borrow's condition wait has + // exhausted its positive timeout, before the loop's terminal pass. Null in + // production; regression tests release a retired slot here to prove that + // the terminal pass re-probes returned capacity before throwing. + private volatile Runnable borrowWaitExpiredHook; // Slots removed from `all` whose delegate is still releasing its flock. // They keep reserving capacity (and their slotInUse mark) until the // flock drops, so the cap check and the slot allocator stay consistent @@ -171,34 +257,91 @@ public final class SenderPool implements AutoCloseable { // down on another thread. Guarded by lock. private int pendingLeaseTeardowns; // Slots whose delegate close() returned with the SF flock still held - // (the I/O thread refused to stop). Permanently consumed: the index is + // because an I/O or manager worker did not stop. Consumed while retired: // never freed and never reused, so no borrow ever hands out a still- // locked slot dir. Counted in the cap check so the lost capacity is - // accounted for. Guarded by lock; only ever ticks for SF slots. + // accounted for. NOT necessarily permanent: engine cleanup may be pending + // on a worker/I/O-thread exit path, so reprobeRetiredSlots() re-checks + // retiredSlots and returns any index whose flock has since dropped. + // Guarded by lock; only ever ticks for SF slots. private int leakedSlots; + // Deterministic white-box complexity counter. Counts delegate release + // probes performed by direct callbacks and fallback scans. Guarded by lock. + private long retiredSlotProbeCount; + // The retired slots behind the leakedSlots count: runtime reclaim paths + // (discardBroken/reapIdle via reclaimSlot) and the in-range startup- + // recovery pass (recoverOneSlotStep, which retains the recoverer slot for + // exactly this purpose). Re-probed by reprobeRetiredSlots() so a late + // flock release (deferred engine cleanup on a worker exit path) restores + // the pool's capacity instead of ratcheting it down until process exit. + // Out-of-range startup recoverers are NEVER added: they carry no + // leakedSlots tick and their index has no slotInUse entry to free. + // Pre-sized to maxSize (every entry keeps a distinct in-range slot index + // reserved, so size can never exceed maxSize): add() never grows the + // backing array, so a retire (leakedSlots++ then add, under lock) cannot + // fail on allocation and strand a counted-but-untracked slot that + // reprobeRetiredSlots() could never recover. Guarded by lock. + private final ArrayList retiredSlots; // SF slots currently held by the in-range startup-recovery pass // (recoverOneSlotStep): each is reserved under `lock` for the // duration of its drain and counted in the borrow() cap check so a // concurrent borrow can neither over-allocate past maxSize nor target a - // dir being recovered. Only ever non-zero on the deferred (housekeeper- - // driven) recovery path, where recovery overlaps borrow()/return; on the - // inline construction path the pool is still single-threaded. Guarded by - // lock; only ever ticks for SF slots. + // dir being recovered. The inline constructor drive is single-threaded, + // but the continuing direct-pool driver and deferred housekeeper driver can + // overlap borrow()/return after publication. Guarded by lock; only ever + // ticks for SF slots. private int recoveringSlots; // Resumable startup-recovery scan cursor. Advanced only by the single - // recovery driver -- the inline constructor loop (single-threaded, - // unpublished) or the PoolHousekeeper thread (the sole deferred driver) -- + // recovery driver -- the inline constructor loop followed by its private + // direct-pool thread, or the PoolHousekeeper thread (the sole deferred + // driver) -- // so the cursor itself needs no lock; the per-slot reservation it performs // (slotInUse/recoveringSlots) is still taken under `lock` because borrow() // races it. recoveryInRangeNext is the next in-range index in [0, maxSize) // for pass 1; recoveryOutOfRange / recoveryOutOfRangeNext are the lazily // built pass-2 work list (same-base slots at index >= maxSize) and its - // cursor; recoveryComplete latches true when the whole scan finishes or is - // aborted, making runStartupRecoveryStep()/...ToCompletion() idempotent. + // cursor; recoveryComplete latches true only when the whole scan finishes. + // A transient build failure or drain timeout leaves the current candidate + // pending so a later tick or explicit drive can retry it on the same pool. + // A slot-SPECIFIC failure -- flock contention with another live owner, or + // RECOVERY_MAX_SLOT_FAILURE_STREAK consecutive failures on one slot -- is + // instead "parked": the cursor advances past it so it cannot starve the + // remaining slots, recoveryDeferredThisCycle records the park, and once + // both passes finish with parked slots outstanding the cursors rewind for + // another cycle on a later tick instead of latching recoveryComplete. + // A RETIRED index whose dir still holds data is deferred the same way on + // every walk (see the reserved-skip branch in recoverOneSlotStep) so the + // latch can never strand a retired slot's data while the pool lives. + // If the retire lands only AFTER the scan latched (a runtime + // discardBroken/reapIdle reclaim), the late flock release re-arms the + // scan instead: recoverRetiredSlotAt() rewinds the cursors and clears + // recoveryComplete -- volatile, so the un-latch (written under `lock` by + // release callbacks and reprobes) is visible to the unlocked driver + // gates -- and, for direct pools whose private driver already exited, + // revives the driver. Deferred pools re-arm through the housekeeper tick. + // recoveryFailStreak/-Slot track consecutive failures on one candidate; + // recoveryWarnedSlots dedups the per-slot WARNs so an indefinitely + // retried slot logs once per failure episode, not once per retry. All of + // this state is owned by the single recovery driver like the cursors. private int recoveryInRangeNext; private IntList recoveryOutOfRange; private int recoveryOutOfRangeNext; - private boolean recoveryComplete; + private volatile boolean recoveryComplete; + // Set by recoverRetiredSlotAt() whenever a released retired slot's dir is + // still a candidate orphan, and consumed (under `lock`) by the scan's + // end-of-cycle latch decision. Closes the mid-cycle window: a retire AND + // its release can both land while the scan is alive but after the cursor + // already passed that index (it was LIVE when walked -- no retired- + // candidate deferral -- and recoveryComplete was still false at release, + // so the post-latch re-arm does not apply). Without this flag such a + // cycle could end with zero deferrals and latch past the freed dir's + // stranded data. Producer and consumer both run under `lock`, so no + // set can slip between the driver's check and its latch write. + private volatile boolean recoveryRearmRequested; + private int recoveryDeferredThisCycle; + private int recoveryFailStreak; + private int recoveryFailStreakSlot = -1; + private final IntList recoveryWarnedSlots = new IntList(); public SenderPool( String configurationString, @@ -209,7 +352,7 @@ public SenderPool( long maxLifetimeMillis ) { this(configurationString, minSize, maxSize, acquireTimeoutMillis, - idleTimeoutMillis, maxLifetimeMillis, null, false, null, null, null); + idleTimeoutMillis, maxLifetimeMillis, null, false, null, null, null, null, null, null, null); } // Test-only constructor exposing the senderFactory seam: production builds @@ -238,8 +381,8 @@ public SenderPool( // reachable-but-not-acking server; the owner (QuestDBImpl) then drives // recovery one slot per tick on the PoolHousekeeper thread via // runStartupRecoveryStep(). White-box SF tests call this directly; the - // in-range recovery pass is concurrency-safe against borrow()/return on the - // deferred path -- see recoverOneSlotStep(). + // in-range recovery pass is concurrency-safe against borrow()/return after + // pool publication -- see recoverOneSlotStep(). @TestOnly public SenderPool( String configurationString, @@ -253,7 +396,43 @@ public SenderPool( ) { this(configurationString, minSize, maxSize, acquireTimeoutMillis, idleTimeoutMillis, maxLifetimeMillis, senderFactory, - deferStartupRecovery, null, null, null); + deferStartupRecovery, null, null, null, null, null, null, null); + } + + // Test-only constructor adding a deterministic fault hook for the ownership + // gap after a delegate factory returns. + @TestOnly + public SenderPool( + String configurationString, + int minSize, + int maxSize, + long acquireTimeoutMillis, + long idleTimeoutMillis, + long maxLifetimeMillis, + IntFunction senderFactory, + boolean deferStartupRecovery, + Runnable postFactoryHook + ) { + this(configurationString, minSize, maxSize, acquireTimeoutMillis, + idleTimeoutMillis, maxLifetimeMillis, senderFactory, + deferStartupRecovery, null, null, null, postFactoryHook, null, null, null); + } + + @TestOnly + public static SenderPool createWithRecoveryControlsForTesting( + String configurationString, + int minSize, + int maxSize, + long acquireTimeoutMillis, + IntFunction senderFactory, + ThreadFactory recoveryThreadFactory, + Runnable recoveryWaiter, + Runnable beforeFailedRecoveryJoinHook + ) { + return new SenderPool(configurationString, minSize, maxSize, acquireTimeoutMillis, + Long.MAX_VALUE, Long.MAX_VALUE, senderFactory, false, + null, null, null, null, recoveryThreadFactory, recoveryWaiter, + beforeFailedRecoveryJoinHook); } // Full constructor adding the user-supplied ingest callbacks (error @@ -273,6 +452,29 @@ public SenderPool( SenderErrorHandler errorHandler, SenderConnectionListener connectionListener, BackgroundDrainerListener drainerListener + ) { + this(configurationString, minSize, maxSize, acquireTimeoutMillis, + idleTimeoutMillis, maxLifetimeMillis, senderFactory, + deferStartupRecovery, errorHandler, connectionListener, + drainerListener, null, null, null, null); + } + + private SenderPool( + String configurationString, + int minSize, + int maxSize, + long acquireTimeoutMillis, + long idleTimeoutMillis, + long maxLifetimeMillis, + IntFunction senderFactory, + boolean deferStartupRecovery, + SenderErrorHandler errorHandler, + SenderConnectionListener connectionListener, + BackgroundDrainerListener drainerListener, + Runnable postFactoryHook, + ThreadFactory recoveryThreadFactory, + Runnable recoveryWaiter, + Runnable beforeFailedRecoveryJoinHook ) { if (minSize < 0 || maxSize < 1 || minSize > maxSize) { throw new IllegalArgumentException("invalid pool sizing: min=" + minSize + ", max=" + maxSize); @@ -291,8 +493,17 @@ public SenderPool( this.acquireTimeoutMillis = acquireTimeoutMillis; this.idleTimeoutMillis = idleTimeoutMillis; this.maxLifetimeMillis = maxLifetimeMillis; + this.postFactoryHook = postFactoryHook; + this.beforeFailedStartupRecoveryJoinHook = beforeFailedRecoveryJoinHook; + this.deferStartupRecovery = deferStartupRecovery; + this.recoveryThreadFactory = recoveryThreadFactory; + this.startupRecoveryWaiter = recoveryWaiter != null + ? recoveryWaiter + : this::waitForStartupRecoveryRetry; this.all = new ArrayList<>(maxSize); this.available = new ArrayDeque<>(maxSize); + this.retiredSlots = new ArrayList<>(maxSize); + this.creationFinished = lock.newCondition(); this.slotReleased = lock.newCondition(); // Probe the config once, up front: this validates it eagerly (so a // bad config fails at construction even when minSize == 0) and tells @@ -324,17 +535,7 @@ public SenderPool( // propagate without running the cleanup below, leaking every // already-built delegate's flock + mmap'd ring + I/O thread and // resurrecting "sf slot already in use" on the next attempt. - for (int i = 0; i < built; i++) { - try { - all.get(i).delegate().close(); - } catch (Throwable ignored) { - // Best-effort cleanup: a delegate close() can throw an - // Error (e.g. an -ea AssertionError) as well as a - // RuntimeException. Swallow either so we still close the - // remaining pre-warmed slots and rethrow the original - // construction failure below. - } - } + closePrewarmedDelegates(built); throw e; } // Prewarm succeeded. Recover any unacked data a previous run left in @@ -343,18 +544,46 @@ public SenderPool( // (deferStartupRecovery=true) so QuestDB.build() never blocks on a slow // or reachable-but-not-acking server; direct constructions run it inline // here, while still single-threaded and unpublished. - if (!deferStartupRecovery) { - runStartupRecoveryToCompletion(); + Thread recoveryThread = null; + try { + if (!deferStartupRecovery) { + runStartupRecoveryToCompletion(); + if (storeAndForward && !recoveryComplete) { + ThreadFactory threadFactory = recoveryThreadFactory != null + ? recoveryThreadFactory + : SenderPool::createStartupRecoveryThread; + recoveryThread = threadFactory.newThread(this::runStartupRecoveryLoop); + recoveryThread.setDaemon(true); + // Ownership precedes start(): the loop's transactional + // exit is the only place the token is dropped. + recoveryDriverRunning = true; + } + } + this.startupRecoveryThread = recoveryThread; + if (recoveryThread != null) { + recoveryThread.start(); + } + } catch (Throwable e) { + // Thread allocation, configuration, or start can throw after + // prewarm transferred delegate ownership to this constructor. The + // pool cannot be returned, so stop a possibly-started driver before + // tearing down delegates it could otherwise still recover against. + // Preserve the construction throwable; cleanup is best-effort. + markClosing(); + stopFailedStartupRecoveryDriver(recoveryThread); + closePrewarmedDelegates(built); + throw e; } } /** - * Drives startup SF recovery to completion in a single call, bounded by one - * shared {@code acquireTimeoutMillis} wall-clock budget (and each individual - * drain by {@link #RECOVERY_DRAIN_BUDGET_MILLIS}). Used by the inline - * construction path -- single-threaded and unpublished -- and by manual / - * test drives. No-op when SF is off, the pool is shutting down, or recovery - * has already finished. Idempotent. + * Drives startup SF recovery toward completion in a single call, bounded by + * one shared {@code acquireTimeoutMillis} wall-clock budget (and each + * individual drain by {@link #RECOVERY_DRAIN_BUDGET_MILLIS}). Used by the + * inline construction path -- single-threaded and unpublished -- and by + * manual / test drives. Budget exhaustion or a transient slot failure leaves + * the cursor pending for the pool's continuing driver. No-op when SF is off, + * the pool is shutting down, or recovery has already finished. Idempotent. */ void runStartupRecoveryToCompletion() { if (!storeAndForward) { @@ -366,41 +595,110 @@ void runStartupRecoveryToCompletion() { // accepted for a single borrow, so we reuse it as the total budget; once // spent, the remaining slots wait for a later attempt (data stays // durable on disk). - final long deadlineNanos = - System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(acquireTimeoutMillis); + final long budgetNanos = TimeUnit.MILLISECONDS.toNanos(acquireTimeoutMillis); + final long startNanos = System.nanoTime(); while (!closed && !recoveryComplete) { - long remainingMillis = TimeUnit.NANOSECONDS.toMillis(deadlineNanos - System.nanoTime()); - if (remainingMillis <= 0) { + // Subtract nanoTime samples before comparing them with the duration. + // This avoids an overflowed absolute deadline when + // acquireTimeoutMillis is Long.MAX_VALUE and keeps the intended + // duration comparison explicit. + long elapsedNanos = System.nanoTime() - startNanos; + if (elapsedNanos >= budgetNanos) { LOG.warn("startup SF recovery: {}ms budget exhausted; " - + "skipping remaining slots", acquireTimeoutMillis); - recoveryComplete = true; + + "deferring remaining slots", acquireTimeoutMillis); return; } + long remainingNanos = budgetNanos - elapsedNanos; + long remainingMillis = TimeUnit.NANOSECONDS.toMillis(remainingNanos); + if (remainingMillis == 0) { + // drain() accepts milliseconds; preserve a positive sub-ms + // remainder instead of treating truncation as exhaustion. + remainingMillis = 1; + } if (!recoverOneSlotStep(Math.min(remainingMillis, RECOVERY_DRAIN_BUDGET_MILLIS))) { return; } } } + private void runStartupRecoveryLoop() { + while (true) { + if (closed || Thread.currentThread().isInterrupted()) { + // Unconditional exits: still drop ownership so a later re-arm + // on a live pool can spawn a successor (the revive gate + // ignores a closing pool anyway). + releaseRecoveryDriverOwnership(false); + return; + } + if (recoveryComplete) { + // Transactional exit: re-check the latch under `lock` so an + // un-latch (recoverRetiredSlotAt's re-arm) cannot slip + // between this thread's decision to exit and its ownership + // drop -- the lost-wakeup race a Thread.isAlive() probe had. + if (releaseRecoveryDriverOwnership(true)) { + return; + } + // An un-latch landed first: this thread stays the driver. + continue; + } + boolean hasImmediateWork; + try { + hasImmediateWork = runStartupRecoveryStep(); + } catch (Throwable ignored) { + // Match PoolHousekeeper: startup recovery is best-effort and a + // delegate Error must not permanently kill its only driver. + hasImmediateWork = false; + } + if (!hasImmediateWork && !closed && !recoveryComplete) { + startupRecoveryWaiter.run(); + } + } + } + + /** + * Drops this thread's private-driver ownership token, transactionally + * against the re-arm path. With {@code recheckLatch}, an un-latch that + * raced this driver's exit decision is detected under {@code lock} and + * ownership is KEPT ({@code false} is returned; the caller keeps + * driving). The producer ({@link #recoverRetiredSlotAt}) clears the + * latch and reads {@code recoveryDriverRunning} under the same lock, so + * exactly one of the two parties always owns the follow-up: either this + * driver observes the cleared latch, or the producer observes the + * dropped token and spawns a successor. No interleaving leaves an + * un-latched scan ownerless. + */ + private boolean releaseRecoveryDriverOwnership(boolean recheckLatch) { + lock.lock(); + try { + if (recheckLatch && !closed && !recoveryComplete) { + return false; + } + recoveryDriverRunning = false; + return true; + } finally { + lock.unlock(); + } + } + /** * Recovers at most ONE stranded managed slot and reports whether more remain. - * Driven by the {@link PoolHousekeeper} one slot per step, back-to-back, on - * the reap-loop thread: each step performs at most one drain bounded by - * {@link #RECOVERY_DRAIN_BUDGET_MILLIS} -- kept below the housekeeper - * stop/join budget -- on a delegate whose initial connect is forced OFF + * The private direct driver or {@link PoolHousekeeper} drives steps + * back-to-back: each step performs at most one drain bounded by + * {@link #RECOVERY_DRAIN_BUDGET_MILLIS} -- kept below the driver stop/join + * budget -- on a delegate whose initial connect is forced OFF * ({@link #defaultRecoverySender}) so the build makes at most one connect - * attempt. The housekeeper re-checks {@code stop} between steps while - * {@link QuestDBImpl#close()} raises the {@code closed} shutdown signal (via - * {@link #markClosing()}) BEFORE stopping it, so a {@code close()} landing - * mid-recovery normally only has to wait out a single bounded drain. The one + * attempt. Each owner raises the {@code closed} shutdown signal before + * joining its driver, so a {@code close()} landing mid-recovery normally + * only has to wait out a single bounded drain. The one * exception is an in-flight connect to a black-holed host, which blocks on * the OS connect timeout -- see the residual-window note on * {@link #recoverOneSlotStep}. * No-op (returns {@code false}) when SF is off, the pool is shutting down, or * recovery has already finished. * - * @return {@code true} if recovery has more work (call again), {@code false} - * when recovery is complete or the pool is shutting down + * @return {@code true} if recovery has more work immediately; {@code false} + * when recovery is complete, the pool is shutting down, or a transient + * failure deferred the current candidate until a later tick */ boolean runStartupRecoveryStep() { if (!storeAndForward || closed || recoveryComplete) { @@ -432,23 +730,38 @@ boolean runStartupRecoveryStep() { *

    * The in-range pass reserves each slot index under {@code lock} for the * duration of its recovery AND counts it in the borrow() capacity check (via - * {@code recoveringSlots}), so a concurrent borrow on the deferred path can - * neither target the dir being recovered nor over-allocate past + * {@code recoveringSlots}), so a concurrent borrow after pool publication + * can neither target the dir being recovered nor over-allocate past * {@code maxSize}. Prewarmed/borrowed slots (already live, holding their * flock) are skipped, as are empty slots (a cheap directory probe); only a - * slot that actually holds stranded data spends the step's single drain. The + * slot that actually holds stranded data spends the step's single drain. A + * RETIRED index (a wedged close() kept its flock; see {@link #reclaimSlot} + * and the retire branch below) is likewise never drained in place, but + * while its dir still holds data it counts as a deferral, so the scan keeps + * cycling instead of latching {@code recoveryComplete} past stranded data + * that only a restart or a lucky borrow of that index would ever deliver. The * out-of-range pass needs no reservation: those indices have no * {@code slotInUse} entry and are never allocated by borrow(). *

    * Best-effort throughout: a build/close Error or a slow drain is logged and - * never propagates, since the data stays durable on disk for a later attempt; - * the first build failure or drain timeout latches {@code recoveryComplete} - * (the failure will very likely repeat for every remaining slot). + * never propagates, since the data stays durable on disk for a later attempt. + * A build failure or drain timeout stops the current drive but leaves its + * candidate pending so a later driver attempt can retry after a transient + * condition clears; it does not poison recovery for the life of the pool. + * A slot-SPECIFIC condition -- flock contention with another live owner, or + * {@link #RECOVERY_MAX_SLOT_FAILURE_STREAK} consecutive failures on the + * same candidate -- instead parks that slot: the cursor advances past it so + * one wedged slot cannot starve the remaining slots' recovery, and once + * both passes finish with parked slots outstanding the scan rewinds for + * another cycle on a later tick (nothing is abandoned while the pool + * lives). Per-slot failure WARNs are deduplicated across retries via + * {@link #warnSlotOnce}. *

    - * Boundedness / residual window. Recovery is driven on the - * PoolHousekeeper thread, and {@code close()} relies on a step finishing - * within {@code PoolHousekeeper.STOP_TIMEOUT_MILLIS}. A step is build + - * drain + close. The drain is capped by {@link #RECOVERY_DRAIN_BUDGET_MILLIS} + * Boundedness / residual window. A continuing recovery step runs on + * either the direct pool's private driver or the PoolHousekeeper thread, and + * {@code close()} relies on a step finishing within + * {@code PoolHousekeeper.STOP_TIMEOUT_MILLIS}. A step is build + drain + + * close. The drain is capped by {@link #RECOVERY_DRAIN_BUDGET_MILLIS} * and the build forces {@code initial_connect_mode=OFF} (see * {@link #defaultRecoverySender}), so it makes at most ONE connect attempt * instead of a SYNC reconnect-budget retry loop. That removes the @@ -456,7 +769,7 @@ boolean runStartupRecoveryStep() { * One residual window remains and is NOT closed here: a single in-flight * connect to a black-holed/firewalled host blocks on the OS connect timeout * (the transport exposes no application-level connect timeout to clamp it). - * If {@code close()} lands during that one connect the housekeeper join can + * If {@code close()} lands during that one connect, its driver join can * still time out and the detached build releases the slot flock shortly * after {@code close()} returns. No data is lost (the slot stays durable on * disk); the exposure is a brief "sf slot already in use" window on an @@ -470,7 +783,7 @@ private boolean recoverOneSlotStep(long stepBudgetMillis) { recoveryComplete = true; return false; } - final boolean[] flockHeld = new boolean[1]; + final SenderSlot[] retained = new SenderSlot[1]; // Pass 1: in-range managed slots [0, maxSize). Skip live and empty slots // cheaply; spend the step on the first slot that actually holds data. @@ -479,26 +792,49 @@ private boolean recoverOneSlotStep(long stepBudgetMillis) { return false; } int i = recoveryInRangeNext; - // Reserve this index unless prewarm (or a concurrent borrow, on the - // deferred path) already holds it live. Count the reservation in + String slotPath = sfDir + "/" + slotBaseId + "-" + i; + // Reserve this index unless prewarm (or a concurrent borrow after + // publication) already holds it live. Count the reservation in // recoveringSlots so the borrow() cap check cannot over-allocate // while this slot is held for recovery. boolean reserved; + boolean retired = false; lock.lock(); try { reserved = slotInUse[i]; if (!reserved) { slotInUse[i] = true; recoveringSlots++; + } else { + retired = isRetiredSlotIndex(i); } } finally { lock.unlock(); } if (reserved) { + // A reserved index is normally LIVE (prewarm or a concurrent + // borrow owns it) and is none of recovery's business. A RETIRED + // index is different: its flock is held by this pool's own + // wedged former delegate, and any unacked data in its dir is + // exactly as stranded as a CONTENDED slot's. Without counting + // it as a deferral the scan would latch recoveryComplete and + // abandon that data until a restart or a lucky borrow of this + // exact index -- which steady low load may never produce. Count + // it -- the same rule as a CONTENDED park -- so the end-of-scan + // rewind keeps the cycle alive; once the deferred engine + // cleanup releases the flock, reprobeRetiredSlots()/the release + // callback frees the index and a later cycle reserves and + // drains it right here. The isCandidateOrphan probe (a few + // syscalls, outside the lock) keeps an already-clean retired + // dir from cycling the scan forever, and racing a concurrent + // recover/borrow can only over-count -- costing one extra + // cheap rewound walk, never a missed candidate. + if (retired && OrphanScanner.isCandidateOrphan(slotPath)) { + recoveryDeferredThisCycle++; + } recoveryInRangeNext++; continue; } - String slotPath = sfDir + "/" + slotBaseId + "-" + i; if (!OrphanScanner.isCandidateOrphan(slotPath)) { // No stranded data: release the reservation and keep scanning; // an empty slot must not cost a whole step. @@ -513,44 +849,70 @@ private boolean recoverOneSlotStep(long stepBudgetMillis) { recoveryInRangeNext++; continue; } - // A real candidate -> spend the step on it. Advance the cursor first - // so a resume never reprocesses this index. - recoveryInRangeNext++; - boolean stopScan = drainCandidateSlotForRecovery(i, slotPath, stepBudgetMillis, flockHeld); + // A real candidate -> spend the step on it. Advance the cursor only + // after success so a transient build/drain failure remains retryable. + RecoveryDrainOutcome outcome = drainCandidateSlotForRecovery(i, slotPath, stepBudgetMillis, retained); lock.lock(); try { // Release the recovery reservation accounting; from here either // leakedSlots (retire) or the freed index carries the cap math. recoveringSlots--; - if (flockHeld[0]) { - // close() bailed early with the I/O thread still running and - // the flock still held. Retire the slot permanently (mirror + if (retained[0] != null) { + // close() retained the flock because an I/O or manager + // worker did not stop. Retire the slot (mirror // discardBroken/reapIdle): keep slotInUse[i] set and count it // in leakedSlots so the borrow() cap math accounts for the // lost capacity and no later borrow ever reuses the - // still-locked dir. + // still-locked dir. Keep the recoverer in retiredSlots so + // reprobeRetiredSlots() restores the capacity once the + // deferred engine cleanup releases the flock — without it + // the retirement would be permanent even after the release + // (fatal at maxSize=1: every later borrow would time out). leakedSlots++; - LOG.warn("startup SF recovery: slot {} retired permanently: delegate close() returned with " - + "the flock still held (I/O thread refused to stop); pool capacity reduced by 1, " - + "now {} of {} usable [leakedSlots={}]", + addRetiredSlot(retained[0]); + LOG.warn("startup SF recovery: slot {} retired: delegate close() returned with " + + "the flock still held (I/O or manager worker did not stop); pool capacity reduced by 1, " + + "now {} of {} usable [leakedSlots={}]; the slot is re-probed and recovered " + + "if the worker releases the flock later", i, maxSize - leakedSlots, maxSize, leakedSlots); } else { slotInUse[i] = false; - // On the deferred path a borrow may be waiting on capacity - // this recovery held; the freed index can now admit a - // creation. + // On a post-publication drive, a borrow may be waiting on + // capacity this recovery held; the freed index can now admit + // a creation. slotReleased.signal(); } } finally { lock.unlock(); } - if (stopScan) { - // A build failure or drain timeout that will very likely repeat - // for every remaining slot -- abort the scan; the data stays - // durable on disk for a later attempt. Do not start pass 2. - recoveryComplete = true; + if (outcome == RecoveryDrainOutcome.CONTENDED) { + // Slot-specific by construction: another LIVE owner holds this + // slot's flock and may do so indefinitely. Park the slot -- + // advance past it; the end-of-scan cycle rewind re-probes it + // later -- so it cannot starve the remaining slots, and keep + // scanning within this step (no drain was spent on it). + recoveryDeferredThisCycle++; + recoveryInRangeNext++; + continue; + } + if (outcome == RecoveryDrainOutcome.FAILED) { + if (parkAfterFailure(i)) { + // The same candidate failed RECOVERY_MAX_SLOT_FAILURE_STREAK + // consecutive times: stop presuming the failure is + // server-wide. Park the slot and move on; the end-of-scan + // cycle rewind retries it later, so nothing is abandoned. + recoveryDeferredThisCycle++; + recoveryInRangeNext++; + return true; + } + // Presumed transient/server-wide: stop this drive without + // advancing the cursor. The same live pool retries this + // candidate after the transient condition is removed instead + // of paying a likely-identical failure per remaining slot. return false; } + clearFailStreak(i); + recoveryInRangeNext++; return true; } @@ -571,75 +933,192 @@ private boolean recoverOneSlotStep(long stepBudgetMillis) { if (closed) { return false; } - int idx = recoveryOutOfRange.getQuick(recoveryOutOfRangeNext++); + int idx = recoveryOutOfRange.getQuick(recoveryOutOfRangeNext); String slotPath = sfDir + "/" + slotBaseId + "-" + idx; if (!OrphanScanner.isCandidateOrphan(slotPath)) { + recoveryOutOfRangeNext++; continue; } - boolean stopScan = drainCandidateSlotForRecovery(idx, slotPath, stepBudgetMillis, flockHeld); - if (flockHeld[0]) { + RecoveryDrainOutcome outcome = drainCandidateSlotForRecovery(idx, slotPath, stepBudgetMillis, retained); + if (retained[0] != null) { // Out of the pool's [0, maxSize) capacity range: there is no // slotInUse entry to retire and no future borrow targets this - // dir, so a still-held flock only leaks this recoverer's I/O - // thread (a best-effort teardown loss, logged). Crucially we do + // dir, so a still-held flock only leaks this recoverer's + // worker-reachable resources (a best-effort teardown loss, + // logged). Crucially we do // NOT touch leakedSlots -- that would wrongly shrink the - // in-range pool capacity. + // in-range pool capacity -- and we do NOT add to retiredSlots: + // there is no capacity to recover, and freeSlotIndex(idx) + // would index past the slotInUse array (sized maxSize). LOG.warn("startup SF recovery: out-of-range slot {} closed with the flock still held " - + "(I/O thread refused to stop); its data is durable on disk for a later attempt", + + "(I/O or manager worker did not stop); its data is durable on disk for a later attempt", slotPath); } - if (stopScan) { - recoveryComplete = true; + if (outcome == RecoveryDrainOutcome.CONTENDED) { + // Same parking rule as the in-range pass: a live flock holder + // (e.g. a sibling drainer that adopted this out-of-range slot) + // is slot-specific and possibly long-lived; the end-of-scan + // cycle rewind re-probes it after the holder lets go. No + // capacity bookkeeping is involved out of range. + recoveryDeferredThisCycle++; + recoveryOutOfRangeNext++; + continue; + } + if (outcome == RecoveryDrainOutcome.FAILED) { + if (parkAfterFailure(idx)) { + recoveryDeferredThisCycle++; + recoveryOutOfRangeNext++; + return true; + } + // Keep the out-of-range cursor on this candidate so a later + // tick retries it after the presumed-transient condition + // clears; no capacity bookkeeping is involved. return false; } + clearFailStreak(idx); + recoveryOutOfRangeNext++; return true; } - recoveryComplete = true; + if (recoveryDeferredThisCycle > 0) { + // At least one slot was parked this cycle (contended, persistently + // failing, or retired with data still on disk). Its data stays durable on disk, so + // instead of latching recoveryComplete -- which would abandon it + // until a restart or a lucky borrow of that index -- rewind the + // scan and let the driver's retry cadence run another cycle. + // Recovered, live and empty slots are re-skipped cheaply; parked + // slots get a fresh (bounded) attempt. Per-slot WARNs stay + // deduplicated across cycles via recoveryWarnedSlots. + recoveryDeferredThisCycle = 0; + recoveryInRangeNext = 0; + recoveryOutOfRangeNext = 0; + return false; + } + // Latch decision under `lock`: a mid-cycle flock release can free a + // still-candidate dir AFTER this cycle's cursor passed its index (it + // was live/retired at walk time -- no deferral) and BEFORE the latch + // (so the post-latch re-arm in recoverRetiredSlotAt does not apply + // either). recoverRetiredSlotAt raises recoveryRearmRequested under + // the same lock, so consuming it here mutually excludes the race: no + // release can slip between this check and the latch write. + lock.lock(); + try { + if (recoveryRearmRequested) { + recoveryRearmRequested = false; + recoveryDeferredThisCycle = 0; + recoveryInRangeNext = 0; + recoveryOutOfRangeNext = 0; + return false; + } + recoveryComplete = true; + } finally { + lock.unlock(); + } return false; } + private void closePrewarmedDelegates(int built) { + for (int i = 0; i < built; i++) { + try { + all.get(i).delegate().close(); + } catch (Throwable ignored) { + // Best-effort cleanup: one delegate close failure must not + // strand later prewarmed delegates or replace the original + // construction throwable. + } + } + } + + private static Thread createStartupRecoveryThread(Runnable runnable) { + return new Thread(runnable, "questdb-sender-pool-recovery"); + } + /** * Drains one candidate orphan slot dir within {@code remainingMillis}, * best-effort and never throwing. Builds a recoverer on {@code slotIndex} * (whose {@link #defaultSender} derives the dir {@code -slotIndex}), * drains its unacked data, and closes the delegate. Shared by both recovery * passes -- the in-range pass and the out-of-range pass -- which differ only - * in their slot bookkeeping, handled by the caller via {@code flockHeld}. + * in their slot bookkeeping, handled by the caller via {@code retainedOut}. * - * @param flockHeld single-element out-param set to {@code true} iff a - * recoverer was built and its {@code close()} returned with - * the flock still held (the I/O thread refused to stop) - * @return {@code true} if a build/drain failure occurred that will very - * likely repeat for every remaining slot, so the caller should stop scanning + * @param retainedOut single-element out-param set to the recoverer iff one + * was built and its {@code close()} returned with the + * flock still held because a worker did not stop; the + * in-range caller keeps it in {@link #retiredSlots} so a + * late flock release can be re-probed. {@code null} when + * the flock was released (or no recoverer was built). + * @return {@link RecoveryDrainOutcome#CONTENDED} when the slot flock is + * held by another live owner (slot-specific: park it and keep scanning); + * {@link RecoveryDrainOutcome#FAILED} on any other build/drain failure + * (presumed server-wide, likely to repeat for every remaining slot); + * {@link RecoveryDrainOutcome#DRAINED} when the candidate was drained or + * was no longer a candidate */ - private boolean drainCandidateSlotForRecovery(int slotIndex, String slotPath, - long remainingMillis, boolean[] flockHeld) { - flockHeld[0] = false; + private RecoveryDrainOutcome drainCandidateSlotForRecovery(int slotIndex, String slotPath, + long remainingMillis, SenderSlot[] retainedOut) { + retainedOut[0] = null; // Hoisted so the flock check after the try can consult it: // createRecoverer() takes the slot flock on -slotIndex, and - // delegate().close() can early-return with the I/O thread still running - // (flock still held). + // delegate().close() can retain it when an I/O or manager worker does + // not stop. SenderSlot recoverer = null; - boolean stopScan = false; + RecoveryDrainOutcome outcome = RecoveryDrainOutcome.DRAINED; try { if (!OrphanScanner.isCandidateOrphan(slotPath)) { - return false; + return RecoveryDrainOutcome.DRAINED; + } + // O(1) contention pre-probe (flock only). A slot parked as + // CONTENDED is re-probed on every retry cycle for as long as its + // live owner runs -- potentially that owner's whole lifetime -- + // and the full recovery build below (config re-parse, builder + // graph, parent-dir fsync barriers in periodic durability, owned + // SegmentManager allocation) would exist only to reach + // SlotLock.acquire and throw. Ask the flock directly first so + // the steady-state cost of a held slot is a few syscalls per + // cycle, not a build. Races are benign in both directions: a + // free probe can still lose the acquire inside the build (the + // contention catch below parks exactly as before), and a held + // probe that goes stale is re-observed on the next cycle. An + // indeterminate probe (null) falls through to the build, which + // owns real error classification. + String probedHolder = SlotLock.probeHolder(slotPath); + if (probedHolder != null) { + if (warnSlotOnce(slotIndex)) { + LOG.warn("startup SF recovery: slot {} is held by another live owner (holder={}); " + + "parking it and continuing with the remaining slots", + slotPath, probedHolder); + } + return RecoveryDrainOutcome.CONTENDED; } try { // Recovery delegate: forced OFF-mode initial connect (see // createRecoverer / defaultRecoverySender), so this build does // at most ONE connect attempt -- it never inherits the SYNC - // reconnect-budget retry loop that would block this - // (PoolHousekeeper) thread for minutes (M1). + // reconnect-budget retry loop that would block this recovery + // driver for minutes (M1). recoverer = createRecoverer(slotIndex); + } catch (SlotLockContentionException contention) { + // The slot flock is held by another LIVE owner. That condition + // is scoped to THIS slot dir by construction -- it says nothing + // about the server or the remaining slots -- and it can persist + // for the owner's whole lifetime, so retrying it in place would + // starve every remaining slot (C2). Report it distinctly so the + // scan parks this slot and continues. + if (warnSlotOnce(slotIndex)) { + LOG.warn("startup SF recovery: slot {} is held by another live owner ({}); " + + "parking it and continuing with the remaining slots", + slotPath, contention.toString()); + } + return RecoveryDrainOutcome.CONTENDED; } catch (Throwable buildErr) { // A build/connect failure (e.g. server unreachable) will very // likely repeat for every remaining slot, so stop here rather // than pay a connect timeout per slot. - LOG.warn("startup SF recovery: could not open slot {} ({}); " - + "skipping remaining slots", slotPath, buildErr.toString()); - return true; + if (warnSlotOnce(slotIndex)) { + LOG.warn("startup SF recovery: could not open slot {} ({}); " + + "deferring this and remaining slots", slotPath, buildErr.toString()); + } + return RecoveryDrainOutcome.FAILED; } try { // Cap the drain at the remaining shared budget and short-circuit @@ -647,14 +1126,19 @@ private boolean drainCandidateSlotForRecovery(int slotIndex, String slotPath, // will very likely do the same for every remaining slot -- the // same reasoning as the build-failure case above. if (!recoverer.delegate().drain(remainingMillis)) { - LOG.warn("startup SF recovery: drain did not ack slot {} " - + "within {}ms; skipping remaining slots", - slotPath, remainingMillis); - stopScan = true; + if (warnSlotOnce(slotIndex)) { + LOG.warn("startup SF recovery: drain did not ack slot {} " + + "within {}ms; deferring this and remaining slots", + slotPath, remainingMillis); + } + outcome = RecoveryDrainOutcome.FAILED; } } catch (Throwable drainErr) { - LOG.warn("startup SF recovery: drain failed for slot {} ({})", - slotPath, drainErr.toString()); + if (warnSlotOnce(slotIndex)) { + LOG.warn("startup SF recovery: drain failed for slot {} ({}); deferring it", + slotPath, drainErr.toString()); + } + outcome = RecoveryDrainOutcome.FAILED; } finally { try { recoverer.delegate().close(); @@ -664,13 +1148,69 @@ private boolean drainCandidateSlotForRecovery(int slotIndex, String slotPath, } } } catch (Throwable scanErr) { - LOG.warn("startup SF recovery: scan failed for slot {} ({})", - slotPath, scanErr.toString()); + if (warnSlotOnce(slotIndex)) { + LOG.warn("startup SF recovery: scan failed for slot {} ({}); deferring it", + slotPath, scanErr.toString()); + } + outcome = RecoveryDrainOutcome.FAILED; + } + if (recoverer != null && !flockReleased(recoverer)) { + retainedOut[0] = recoverer; } - if (recoverer != null) { - flockHeld[0] = !flockReleased(recoverer); + if (outcome == RecoveryDrainOutcome.DRAINED && recoveryWarnedSlots.contains(slotIndex)) { + recoveryWarnedSlots.remove(slotIndex); + LOG.info("startup SF recovery: slot {} recovered after earlier deferrals", slotPath); + } + return outcome; + } + + /** + * Clears the consecutive-failure streak after a successful drain so the + * next candidate starts from a clean count. Driver-thread state (see the + * scan cursor comment); no lock needed. + */ + private void clearFailStreak(int slotIndex) { + if (recoveryFailStreakSlot == slotIndex) { + recoveryFailStreakSlot = -1; + recoveryFailStreak = 0; + } + } + + /** + * Records one FAILED drain attempt on a candidate slot and reports whether + * the scan should park it: {@code true} once the SAME slot has failed + * {@link #RECOVERY_MAX_SLOT_FAILURE_STREAK} consecutive times, meaning the + * failure is evidently slot-specific rather than server-wide and must not + * pin the cursor any longer (C2). Driver-thread state; no lock needed. + */ + private boolean parkAfterFailure(int slotIndex) { + if (recoveryFailStreakSlot != slotIndex) { + recoveryFailStreakSlot = slotIndex; + recoveryFailStreak = 0; + } + if (++recoveryFailStreak < RECOVERY_MAX_SLOT_FAILURE_STREAK) { + return false; } - return stopScan; + recoveryFailStreakSlot = -1; + recoveryFailStreak = 0; + return true; + } + + /** + * Dedups per-slot recovery WARNs: a contended or persistently failing slot + * is retried indefinitely (its data stays durable on disk), and warning on + * every retry produced an unbounded one-WARN-per-second stream for the life + * of the process (C2). Returns {@code true} only the first time a slot + * fails since its last successful drain, so each failure episode logs + * exactly once (a follow-up failure with a DIFFERENT cause is deliberately + * folded into the same episode). Driver-thread state; no lock needed. + */ + private boolean warnSlotOnce(int slotIndex) { + if (recoveryWarnedSlots.contains(slotIndex)) { + return false; + } + recoveryWarnedSlots.add(slotIndex); + return true; } public PooledSender borrow() { @@ -715,33 +1255,26 @@ public PooledSender borrow() { // idempotent, so undoing the reservation for any // throwable is safe. lock.lock(); - inFlightCreations--; - freeSlotIndex(slotIndex); - slotReleased.signal(); - lock.unlock(); + try { + inFlightCreations--; + freeSlotIndex(slotIndex); + creationFinished.signalAll(); + slotReleased.signal(); + } finally { + lock.unlock(); + } throw e; } lock.lock(); - inFlightCreations--; if (closed) { - // Pool was closed mid-creation -- destroy the new connection - // rather than leaking it. Other waiters have been signaled - // by close() already. The delegate is closed OUTSIDE the - // lock (mirroring retireLease): its close() can block for - // seconds (bounded ack drain, drainer-pool wind-down) or - // longer (unbounded I/O-thread latch await behind an - // OS-level connect), which held here would stall close(), - // giveBack/retireLease and reapIdle behind the pool lock. - // Accounting first, under the lock: for an SF slot the - // index reservation moves from inFlightCreations to - // closingSlots until the close below releases the flock, - // and pendingLeaseTeardowns keeps the out-of-lock close - // visible to close()'s outstanding-teardown wait. + // Pool was closed mid-creation. Keep inFlightCreations + // reserved while the delegate is closed outside the + // lock: close() waits on that counter, so the creation + // remains internally owned until its teardown completes. boolean reserved = created.slotIndex() >= 0; if (reserved) { closingSlots++; } - pendingLeaseTeardowns++; lock.unlock(); try { created.delegate().close(); @@ -749,31 +1282,56 @@ public PooledSender borrow() { // Best-effort: an Error (e.g. -ea AssertionError) // from teardown must not mask the closed-pool signal. } finally { - // Re-lock to reclaim the SF slot index and signal a - // close() waiting on this teardown. MUST run even if - // the delegate close threw, otherwise the slot stays - // reserved forever and close() waits out its full - // budget on a teardown that already happened. lock.lock(); - pendingLeaseTeardowns--; - if (reserved) { - reclaimSlot(created, " after closed-mid-creation teardown"); + try { + if (reserved) { + reclaimSlot(created, " after closed-mid-creation teardown"); + } + inFlightCreations--; + creationFinished.signalAll(); + slotReleased.signalAll(); + } finally { + lock.unlock(); } - slotReleased.signalAll(); - lock.unlock(); } throw new LineSenderException("QuestDB handle is closed"); } all.add(created); created.bumpGeneration(); + inFlightCreations--; + creationFinished.signalAll(); return new PooledSender(created, created.generation()); } + // Capacity-starved: re-probe retired slots BEFORE the terminal + // timeout check — a deferred engine cleanup may have released a + // flock since the retire, and the freed index can admit a + // creation right now. The delegate normally signals this pool + // after deferred release, while this probe also covers delegates + // that do not expose that notification and release/listener races. + // Ordering matters twice over: a + // zero-timeout (try-once) borrow must get its one probe before + // throwing, and a borrower whose awaitNanos budget just expired + // must get a final probe on its wake-up pass instead of timing + // out on capacity that has already come back. + if (reprobeRetiredSlots()) { + continue; + } if (remainingNanos <= 0) { throw new LineSenderException( "timed out waiting for a Sender from the pool after " + acquireTimeoutMillis + "ms"); } try { + Runnable beforeWaitHook = beforeBorrowWaitHook; + if (beforeWaitHook != null) { + beforeWaitHook.run(); + } remainingNanos = slotReleased.awaitNanos(remainingNanos); + if (remainingNanos <= 0) { + Runnable hook = borrowWaitExpiredHook; + if (hook != null) { + hook.run(); + } + } } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new LineSenderException("interrupted while waiting for a Sender from the pool"); @@ -786,12 +1344,127 @@ public PooledSender borrow() { } } + @TestOnly + public Sender buildRecoverySenderForTesting(int slotIndex) { + return defaultRecoverySender(slotIndex); + } + + @TestOnly + public Sender buildSenderForTesting(int slotIndex) { + return defaultSender(slotIndex); + } + + @TestOnly + public void discardBrokenForTesting(PooledSender sender) { + discardBroken(sender); + } + + @TestOnly + public int getInFlightCreationsForTesting() { + lock.lock(); + try { + return inFlightCreations; + } finally { + lock.unlock(); + } + } + + @TestOnly + public int getRetiredSlotCountForTesting() { + lock.lock(); + try { + return retiredSlots.size(); + } finally { + lock.unlock(); + } + } + + @TestOnly + public long getRetiredSlotProbeCountForTesting() { + lock.lock(); + try { + return retiredSlotProbeCount; + } finally { + lock.unlock(); + } + } + + @TestOnly + public Thread getStartupRecoveryThreadForTesting() { + return startupRecoveryThread; + } + + @TestOnly + public boolean hasCreationWaiterForTesting() { + lock.lock(); + try { + return lock.hasWaiters(creationFinished); + } finally { + lock.unlock(); + } + } + + @TestOnly + public boolean isCloseStartedForTesting() { + lock.lock(); + try { + return closeStarted; + } finally { + lock.unlock(); + } + } + + @TestOnly + public boolean isClosedForTesting() { + return closed; + } + + @TestOnly + public boolean isRecoveryCompleteForTesting() { + return recoveryComplete; + } + + @TestOnly + public boolean isSlotInUseForTesting(int slotIndex) { + lock.lock(); + try { + return slotInUse[slotIndex]; + } finally { + lock.unlock(); + } + } + + @TestOnly + public void markClosingForTesting() { + markClosing(); + } + + @TestOnly + public void runStartupRecoveryToCompletionForTesting() { + runStartupRecoveryToCompletion(); + } + + @TestOnly + public boolean runStartupRecoveryStepForTesting() { + return runStartupRecoveryStep(); + } + + @TestOnly + public void setBeforeBorrowWaitHook(Runnable hook) { + this.beforeBorrowWaitHook = hook; + } + + @TestOnly + public void setBorrowWaitExpiredHook(Runnable hook) { + this.borrowWaitExpiredHook = hook; + } + /** * Raises the shutdown signal early -- without tearing down live delegates -- - * so an in-flight startup-recovery step driven on the {@link PoolHousekeeper} - * thread stops promptly between slots. {@link QuestDBImpl#close()} calls this - * BEFORE stopping the housekeeper, so the housekeeper join cannot time out - * waiting on a fresh slot's recovery drain. Idle-delegate teardown and the + * so an in-flight startup-recovery step stops promptly between slots. Direct + * {@link #close()} calls this before joining its private driver; + * {@link QuestDBImpl#close()} calls it before stopping the housekeeper. + * Idle-delegate teardown and the * outstanding-lease wait still happen in {@link #close()} (guarded by * {@code closeStarted}, so this early signal never short-circuits them); * borrowed delegates returned after this signal are torn down on the @@ -800,6 +1473,9 @@ public PooledSender borrow() { */ void markClosing() { closed = true; + synchronized (startupRecoverySignal) { + startupRecoverySignal.notifyAll(); + } } /** @@ -809,6 +1485,9 @@ void markClosing() { * free their native memory under its feet -- a use-after-free / SEGV, not * an exception (C1). Instead: *

      + *
    1. waits boundedly for internally owned creations; a late creator keeps + * ownership and performs its closed-mid-creation teardown asynchronously; + * then
      2. *
    2. waits boundedly (up to {@code acquireTimeoutMillis}, hard-capped at * {@link #MAX_CLOSE_LEASE_WAIT_MILLIS}) for outstanding leases to come home -- {@link #giveBack} and {@link #discardBroken} * observe {@code closed} and tear each delegate down on the returning @@ -823,6 +1502,11 @@ void markClosing() { */ @Override public void close() { + // Direct pools own their continuing recovery driver. Stop it before + // snapshotting/closing delegates; deferred pools have a null thread and + // QuestDBImpl stops their external PoolHousekeeper before calling here. + markClosing(); + stopStartupRecoveryDriver(); SenderSlot[] idleSnapshot; lock.lock(); try { @@ -835,6 +1519,35 @@ public void close() { closed = true; // Wake parked borrowers so they observe the shutdown and throw. slotReleased.signalAll(); + // A creator can block in DNS, TCP, TLS, or WebSocket setup. Wait + // boundedly rather than turning an unset connect_timeout into an + // unbounded QuestDB.close(). Timing out does not abandon ownership: + // the reservation and any SF slot stay assigned to the creator, + // which observes closed and tears down a late result before releasing + // them. Preserve the caller's interrupt status while still applying + // the same finite wait budget. + final long creationWaitMillis = Math.max(0, + Math.min(acquireTimeoutMillis, MAX_CLOSE_CREATION_WAIT_MILLIS)); + final long creationWaitNanos = TimeUnit.MILLISECONDS.toNanos(creationWaitMillis); + final long creationWaitDeadlineNanos = System.nanoTime() + creationWaitNanos; + long creationRemainingNanos = creationWaitNanos; + boolean creationWaitInterrupted = false; + while (inFlightCreations > 0 && creationRemainingNanos > 0) { + try { + creationFinished.awaitNanos(creationRemainingNanos); + } catch (InterruptedException e) { + creationWaitInterrupted = true; + } + creationRemainingNanos = creationWaitDeadlineNanos - System.nanoTime(); + } + if (creationWaitInterrupted) { + Thread.currentThread().interrupt(); + } + if (inFlightCreations > 0) { + LOG.warn("SenderPool.close(): {} sender creation(s) still in flight after {}ms; " + + "each creator retains cleanup ownership and releases its SF slot when construction returns", + inFlightCreations, creationWaitMillis); + } // Bounded graceful wait for outstanding leases. A slot is borrowed // iff it is in `all` but not in `available`; retireLease's // delegate-close section (running outside the lock on a returning @@ -934,6 +1647,83 @@ public void giveBack(PooledSender ps) { retireLease(ps, " during pool shutdown"); } + @TestOnly + public void setRetiredSlotProbeCountForTesting(long count) { + lock.lock(); + try { + retiredSlotProbeCount = count; + } finally { + lock.unlock(); + } + } + + @TestOnly + public void setStartupRecoveryJoinHooksForTesting(Runnable beforeJoinHook, Runnable afterJoinHook) { + this.beforeStartupRecoveryJoinHook = beforeJoinHook; + this.afterStartupRecoveryJoinHook = afterJoinHook; + } + + private void stopFailedStartupRecoveryDriver(Thread recoveryThread) { + if (recoveryThread == null || recoveryThread == Thread.currentThread()) { + return; + } + if (beforeFailedStartupRecoveryJoinHook != null) { + beforeFailedStartupRecoveryJoinHook.run(); + } + boolean interrupted = false; + while (recoveryThread.isAlive()) { + try { + recoveryThread.join(); + } catch (InterruptedException e) { + interrupted = true; + } + } + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + + private void stopStartupRecoveryDriver() { + if (startupRecoveryThread != null && startupRecoveryThread != Thread.currentThread()) { + if (beforeStartupRecoveryJoinHook != null) { + beforeStartupRecoveryJoinHook.run(); + } + try { + startupRecoveryThread.join(PoolHousekeeper.STOP_TIMEOUT_MILLIS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + if (afterStartupRecoveryJoinHook != null) { + afterStartupRecoveryJoinHook.run(); + } + } + // A revived driver (late-release re-arm on a direct pool; see + // reviveDirectRecoveryDriverIfNeeded) is joined with the same bound: + // `closed` is already raised, so its loop exits on the next waiter + // wake-up, within RECOVERY_RETRY_INTERVAL_MILLIS < STOP_TIMEOUT. + Thread revived = revivedStartupRecoveryThread; + if (revived != null && revived != Thread.currentThread()) { + try { + revived.join(PoolHousekeeper.STOP_TIMEOUT_MILLIS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + } + + private void waitForStartupRecoveryRetry() { + synchronized (startupRecoverySignal) { + if (closed || recoveryComplete) { + return; + } + try { + startupRecoverySignal.wait(RECOVERY_RETRY_INTERVAL_MILLIS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + } + /** * Retires one lease on the calling (borrower's) thread: validates the * lease generation under the lock, removes the slot from {@code all}, @@ -1002,7 +1792,9 @@ private void retireLease(PooledSender ps, String context) { pendingLeaseTeardowns--; if (reserved) { // Free the index only when the flock was released; a slot - // left locked is retired permanently. + // left locked is retired into retiredSlots, recoverable + // by reprobeRetiredSlots() if the deferred cleanup drops + // the flock later. reclaimSlot(s, context); } slotReleased.signalAll(); @@ -1028,6 +1820,11 @@ public void reapIdle() { if (closed) { return; } + // Housekeeper tick doubles as the retired-slot recovery driver: + // a slot retired because its worker did not stop is re-probed + // here and returns to the free set once the deferred cleanup + // finally released its flock. + reprobeRetiredSlots(); Iterator it = available.iterator(); while (it.hasNext() && all.size() > minSize) { SenderSlot s = it.next(); @@ -1067,7 +1864,7 @@ public void reapIdle() { } // Return reserved SF slot indices to the free set -- but only for // slots whose delegate confirmed the flock was released. A slot - // left locked (I/O thread refused to stop) is retired permanently. + // left locked because a worker did not stop is retired permanently. if (storeAndForward) { lock.lock(); try { @@ -1106,12 +1903,15 @@ public int totalSize() { } /** - * Snapshot of the number of SF slots permanently retired because a - * delegate {@code close()} returned with the slot flock still held (the - * I/O thread refused to stop). Each leaked slot permanently lowers the - * pool's effective capacity ({@code maxSize - leakedSlotCount()}). A - * non-zero, growing value explains a pool that has started timing out - * every {@code borrow()}. For metrics and tests. + * Snapshot of the number of SF slots currently retired because a + * delegate {@code close()} returned with the slot flock still held after + * an I/O or manager worker did not stop. Each leaked slot lowers the + * pool's effective capacity ({@code maxSize - leakedSlotCount()}) while + * retired. Retired slots are re-probed (housekeeper tick and + * capacity-starved borrows) and recovered once the delegate's deferred + * cleanup releases the flock, so the count can go back down; a non-zero, + * persistent value means a worker is still wedged and explains a pool + * that has started timing out {@code borrow()}. For metrics and tests. */ public int leakedSlotCount() { lock.lock(); @@ -1122,8 +1922,39 @@ public int leakedSlotCount() { } } + private SenderSlot createSlot(IntFunction factory, int slotIndex) { + Sender delegate = factory.apply(slotIndex); + try { + if (postFactoryHook != null) { + postFactoryHook.run(); + } + SenderSlot slot = new SenderSlot(delegate, this, slotIndex); + if (delegate instanceof QwpWebSocketSender) { + ((QwpWebSocketSender) delegate).setSlotLockReleaseListener( + () -> recoverReleasedSlot(slot)); + } + return slot; + } catch (Throwable failure) { + if (delegate instanceof QwpWebSocketSender) { + try { + ((QwpWebSocketSender) delegate).setSlotLockReleaseListener(null); + } catch (Throwable deregistrationFailure) { + addSuppressed(failure, deregistrationFailure); + } + } + if (delegate != null) { + try { + delegate.close(); + } catch (Throwable closeFailure) { + addSuppressed(failure, closeFailure); + } + } + throw failure; + } + } + private SenderSlot createUnlocked(int slotIndex) { - return new SenderSlot(senderFactory.apply(slotIndex), this, slotIndex); + return createSlot(senderFactory, slotIndex); } /** @@ -1134,7 +1965,18 @@ private SenderSlot createUnlocked(int slotIndex) { * {@link #drainCandidateSlotForRecovery}. */ private SenderSlot createRecoverer(int slotIndex) { - return new SenderSlot(recoverySenderFactory.apply(slotIndex), this, slotIndex); + return createSlot(recoverySenderFactory, slotIndex); + } + + private static void addSuppressed(Throwable failure, Throwable cleanupFailure) { + if (failure != cleanupFailure) { + try { + failure.addSuppressed(cleanupFailure); + } catch (Throwable ignored) { + // Preserve the original construction failure even if recording + // the secondary cleanup failure cannot allocate. + } + } } private Sender defaultSender(int slotIndex) { @@ -1144,9 +1986,10 @@ private Sender defaultSender(int slotIndex) { /** * Same managed-slot delegate as {@link #defaultSender}, but with the * initial connect forced to {@link Sender.InitialConnectMode#OFF}. Used - * only for startup recovery, which runs on the PoolHousekeeper thread: OFF - * makes the build do at most ONE connect attempt instead of retrying for - * the whole reconnect budget (SYNC, auto-enabled by any reconnect_* knob), + * only for startup recovery, which runs on a private direct driver or the + * PoolHousekeeper thread: OFF makes the build do at most ONE connect attempt + * instead of retrying for the whole reconnect budget (SYNC, auto-enabled by + * any reconnect_* knob), * keeping a recovery step bounded below * {@code PoolHousekeeper.STOP_TIMEOUT_MILLIS}. See M1 / the residual-window * note on {@link #recoverOneSlotStep}. @@ -1215,8 +2058,8 @@ private Sender buildManagedSlotSender(int slotIndex, boolean forRecovery) { // its OWN slot (the one recoverOneSlotStep is processing); it must // never start a BackgroundDrainerPool for sibling/foreign orphans. // If it did, the delegate's close() -- called from - // drainCandidateSlotForRecovery() on the PoolHousekeeper thread, - // BEFORE its cursorEngine.close() releases the slot flock -- would + // drainCandidateSlotForRecovery() on the recovery driver, BEFORE + // its cursorEngine.close() releases the slot flock -- would // block in BackgroundDrainerPool.close() for up to // GRACEFUL_DRAIN_MILLIS + STOP_GRACE_MILLIS (3s) against a // reachable-but-not-acking server. That overruns a recovery step's @@ -1271,7 +2114,7 @@ private void freeSlotIndex(int idx) { * non-QWP delegate never holds an SF flock, so it is always treated as * released. A {@link QwpWebSocketSender} reports it via * {@link QwpWebSocketSender#isSlotLockReleased()} -- false means close() - * bailed early with the I/O thread still running and the flock still held. + * retained the flock because an I/O or manager worker did not stop. */ private static boolean flockReleased(SenderSlot s) { Sender d = s.delegate(); @@ -1281,11 +2124,17 @@ private static boolean flockReleased(SenderSlot s) { /** * Reclaims one SF slot after its delegate's {@code close()} has been * attempted. When the flock was released the index returns to the free - * set; when {@code close()} returned with the flock still held (the I/O - * thread refused to stop) the slot is retired permanently -- - * {@code leakedSlots++} and {@code slotInUse[idx]} stays set -- so the cap - * math accounts for the lost capacity and no later borrow ever reuses the - * still-locked dir. Either way {@code closingSlots} is decremented. + * set; when {@code close()} returned with the flock still held because an + * I/O or manager worker did not stop, the slot is retired -- + * {@code leakedSlots++}, {@code slotInUse[idx]} stays set, and the sender + * joins {@code retiredSlots} -- so the cap math accounts for the lost + * capacity and no borrow reuses the still-locked dir unless + * {@link #reprobeRetiredSlots} later observes the deferred cleanup's + * release and recovers the index. Either way {@code closingSlots} is + * decremented. A later release that frees a dir still holding unacked + * data also re-arms the startup recovery scan (see + * {@link #recoverRetiredSlotAt}) so the data is drained in-process + * instead of waiting for a restart or a lucky borrow of that index. *

      * Caller must hold {@code lock} and is responsible for signalling waiters * (only the free path admits a new creation). Shared by @@ -1303,9 +2152,202 @@ private boolean reclaimSlot(SenderSlot s, String context) { return true; } leakedSlots++; - LOG.warn("SF slot {} retired permanently{}: delegate close() returned with the flock still held " + - "(I/O thread refused to stop); pool capacity reduced by 1, now {} of {} usable [leakedSlots={}]", + addRetiredSlot(s); + LOG.warn("SF slot {} retired{}: delegate close() returned with the flock still held " + + "(I/O or manager worker did not stop); pool capacity reduced by 1, now {} of {} usable " + + "[leakedSlots={}]; the slot is re-probed and recovered if the worker releases the flock later", s.slotIndex(), context, maxSize - leakedSlots, maxSize, leakedSlots); return false; } + + private boolean recoverReleasedSlot(SenderSlot s) { + lock.lock(); + try { + int retiredIndex = s.retiredIndex(); + if (retiredIndex < 0 + || retiredIndex >= retiredSlots.size() + || retiredSlots.get(retiredIndex) != s) { + // The callback raced retirement, or is stale/duplicate. The + // retirement path probes before insertion, and periodic scans + // remain as a fallback, so no capacity can be lost here. + return false; + } + retiredSlotProbeCount++; + if (!flockReleased(s)) { + return false; + } + recoverRetiredSlotAt(retiredIndex); + slotReleased.signalAll(); + return true; + } finally { + lock.unlock(); + } + } + + /** + * Re-probes every retired slot (see {@link #reclaimSlot}) and returns to + * the free set any whose delegate now reports the flock released — the + * deferred engine cleanup (manager-worker or I/O-thread exit path) has + * run since the retire. Restores {@code leakedSlots} capacity and signals + * waiters so a parked borrow can admit a creation immediately. + *

      + * Caller must hold {@code lock}. The probe is cheap on the delegate side + * ({@code isSlotLockReleased()} reads volatiles, and only re-arms the + * shared flock-release retry in the rare orphaned-retry state), so + * holding the pool lock across it cannot stall behind delegate teardown. + * + * @return {@code true} if at least one slot's capacity was recovered + */ + private boolean reprobeRetiredSlots() { + boolean recovered = false; + for (int i = retiredSlots.size() - 1; i >= 0; i--) { + SenderSlot s = retiredSlots.get(i); + retiredSlotProbeCount++; + if (flockReleased(s)) { + recoverRetiredSlotAt(i); + recovered = true; + } + } + if (recovered) { + slotReleased.signalAll(); + } + return recovered; + } + + private void addRetiredSlot(SenderSlot s) { + s.retiredIndex(retiredSlots.size()); + retiredSlots.add(s); + } + + /** + * Whether {@code idx} is currently held by a RETIRED slot (see + * {@link #reclaimSlot} and the in-range recovery retire branch) rather + * than a live one. Lets the recovery scan tell "reserved because + * borrowed/prewarmed" apart from "reserved because a wedged close() left + * the flock held" for its deferral accounting: only the latter's dir can + * hold stranded data no borrow is coming for. Caller must hold + * {@code lock}; retiredSlots is bounded by maxSize, so the walk is cheap. + */ + private boolean isRetiredSlotIndex(int idx) { + for (int i = 0, n = retiredSlots.size(); i < n; i++) { + if (retiredSlots.get(i).slotIndex() == idx) { + return true; + } + } + return false; + } + + private void recoverRetiredSlotAt(int retiredIndex) { + SenderSlot s = retiredSlots.get(retiredIndex); + int last = retiredSlots.size() - 1; + if (retiredIndex < last) { + SenderSlot moved = retiredSlots.get(last); + retiredSlots.set(retiredIndex, moved); + moved.retiredIndex(retiredIndex); + } + retiredSlots.remove(last); + s.retiredIndex(-1); + leakedSlots--; + freeSlotIndex(s.slotIndex()); + LOG.info("SF slot {} recovered: deferred cleanup released the flock after retirement; " + + "pool capacity restored, now {} of {} usable [leakedSlots={}]", + s.slotIndex(), maxSize - leakedSlots, maxSize, leakedSlots); + // The freed dir may still hold unacked durable data: a runtime retire + // (discardBroken/reapIdle) can land AFTER the startup scan latched + // recoveryComplete, and close() never drains an unowned dir, so + // without re-arming the scan that data would wait for a restart or a + // lucky borrow of exactly this index. When the latch is already set, + // rewind the cursors FIRST, then clear the volatile latch (the write + // order publishes the rewound cursors to the next driver's unlocked + // gate read), then revive the private driver when this is a direct + // pool whose original driver already exited; recoveryComplete==true + // here also guarantees no driver is mid-step, so touching the + // driver-owned cursors is safe. When the scan is still ALIVE, only + // raise recoveryRearmRequested: the cursor may already have passed + // this index while it was live/retired, and the flag makes the + // end-of-cycle latch decision rewind instead -- the cursors stay + // strictly driver-owned. The isCandidateOrphan probe is a bounded + // directory scan; holding the pool lock across it mirrors + // reprobeRetiredSlots()' delegate probes. The pass-2 work list is + // kept and only its cursor rewound, exactly like the end-of-scan + // cycle rewind. + if (sfDir != null + && OrphanScanner.isCandidateOrphan(sfDir + "/" + slotBaseId + "-" + s.slotIndex())) { + if (recoveryComplete) { + recoveryInRangeNext = 0; + recoveryOutOfRangeNext = 0; + recoveryDeferredThisCycle = 0; + recoveryRearmRequested = false; + recoveryComplete = false; + LOG.info("SF slot {} released with stranded data still on disk; re-arming the " + + "startup recovery scan to drain it", s.slotIndex()); + reviveDirectRecoveryDriverIfNeeded(); + } else { + recoveryRearmRequested = true; + LOG.info("SF slot {} released with stranded data while the recovery scan is " + + "mid-cycle; flagging a rewind so this cycle cannot latch past it", + s.slotIndex()); + } + } + } + + /** + * Re-arms a driver for an un-latched scan (see {@link #recoverRetiredSlotAt}). + * Deferred pools need nothing: {@code PoolHousekeeper} calls + * {@link #runStartupRecoveryStep()} on every tick and its gate re-opens + * with the cleared latch. A direct pool's private driver exits once the + * scan completes, so when no thread currently owns the driver duty + * ({@code recoveryDriverRunning} false), spawn a successor running the + * same loop -- it drains the backlog, re-latches, and exits; close() + * joins it via {@link #stopStartupRecoveryDriver}. Ownership is a + * lock-guarded token, not a {@code Thread.isAlive()} probe: the loop + * drops it transactionally ({@link #releaseRecoveryDriverOwnership} + * re-checks the latch under this same lock first), so a driver that is + * mid-exit either observes this caller's un-latch and keeps driving, or + * has already dropped the token and a successor is spawned here -- no + * interleaving leaves the un-latched scan ownerless. Caller must hold + * {@code lock}; {@code Thread.start()} is cheap enough to run under it. + * Best-effort: a spawn failure is logged and never propagates into the + * release callback that triggered the re-arm (the data stays durable + * for a restart). + */ + private void reviveDirectRecoveryDriverIfNeeded() { + if (deferStartupRecovery || closed) { + return; + } + if (recoveryDriverRunning) { + // A live driver's exit re-checks the latch under this lock, so + // it is guaranteed to observe the un-latch that brought us here. + return; + } + try { + ThreadFactory factory = recoveryThreadFactory != null + ? recoveryThreadFactory + : SenderPool::createStartupRecoveryThread; + Thread t = factory.newThread(this::runStartupRecoveryLoop); + t.setDaemon(true); + recoveryDriverRunning = true; + revivedStartupRecoveryThread = t; + t.start(); + } catch (Throwable e) { + recoveryDriverRunning = false; + LOG.warn("could not revive the startup recovery driver ({}); the released slot's " + + "data stays durable on disk for a later attempt or restart", e.toString()); + } + } + + // Outcome of one drainCandidateSlotForRecovery attempt, letting the scan + // tell a slot-specific contention (park the slot and keep scanning) apart + // from a failure presumed server-wide (retry the same candidate, bounded + // by RECOVERY_MAX_SLOT_FAILURE_STREAK). + private enum RecoveryDrainOutcome { + // createRecoverer() lost the slot flock to another live owner -- a + // condition scoped to that slot dir by construction that can outlive + // any retry interval. + CONTENDED, + // The candidate was drained, or was no longer a candidate. + DRAINED, + // Any other build/drain failure, presumed transient and server-wide. + FAILED + } } diff --git a/core/src/main/java/io/questdb/client/impl/SenderSlot.java b/core/src/main/java/io/questdb/client/impl/SenderSlot.java index 19c93671..5d9ed6ff 100644 --- a/core/src/main/java/io/questdb/client/impl/SenderSlot.java +++ b/core/src/main/java/io/questdb/client/impl/SenderSlot.java @@ -48,6 +48,11 @@ final class SenderSlot { private final Sender delegate; private final SenderPool pool; private final int slotIndex; + // Index in SenderPool.retiredSlots, or -1 while this slot is not retired. + // Guarded by the pool lock. The pool maintains this index across swap + // removals so the delegate's release callback can recover this exact slot + // without scanning every other retired slot. + private int retiredIndex = -1; // Monotonic lease id. Mutated only under the SenderPool lock (bumped in // borrow() when the slot is handed out and in giveBack()/discardBroken() // when it is returned). A PooledSender wrapper captures it live for its @@ -112,6 +117,14 @@ SenderPool pool() { return pool; } + int retiredIndex() { + return retiredIndex; + } + + void retiredIndex(int retiredIndex) { + this.retiredIndex = retiredIndex; + } + int slotIndex() { return slotIndex; } diff --git a/core/src/main/java/io/questdb/client/network/JavaTlsClientSocket.java b/core/src/main/java/io/questdb/client/network/JavaTlsClientSocket.java index c1b1eec7..f4dc94df 100644 --- a/core/src/main/java/io/questdb/client/network/JavaTlsClientSocket.java +++ b/core/src/main/java/io/questdb/client/network/JavaTlsClientSocket.java @@ -31,6 +31,7 @@ import io.questdb.client.std.MemoryTag; import io.questdb.client.std.Unsafe; import io.questdb.client.std.Vect; +import org.jetbrains.annotations.TestOnly; import org.slf4j.Logger; import javax.net.ssl.SSLContext; @@ -149,6 +150,14 @@ public void close() { } } + @Override + public void closeTraffic() { + // A concurrent TLS send may still use the SSLEngine and direct buffers. + // Shut down only the delegate traffic path here; close() releases the + // retained fd and frees TLS state after the owning worker has stopped. + delegate.closeTraffic(); + } + @Override public int getFd() { return delegate.getFd(); @@ -166,6 +175,26 @@ public void of(int fd) { state = STATE_PLAINTEXT; } + @TestOnly + public void setPlaintextStateForTesting() { + sslEngine = null; + state = STATE_PLAINTEXT; + } + + @TestOnly + public void setTlsStateForTesting(SSLEngine engine) { + if (state != STATE_PLAINTEXT) { + throw new IllegalStateException("socket must be in plaintext state"); + } + sslEngine = engine; + state = STATE_TLS; + } + + @TestOnly + public TlsStateForTesting snapshotTlsStateForTesting() { + return new TlsStateForTesting(this); + } + @Override public int recv(long bufferPtr, int bufferLen) { assert sslEngine != null; @@ -643,4 +672,35 @@ private int writeToSocket(int bytesToSend) { LIMIT_FIELD_OFFSET = Unsafe.getUnsafe().objectFieldOffset(limitField); CAPACITY_FIELD_OFFSET = Unsafe.getUnsafe().objectFieldOffset(capacityField); } + + @TestOnly + public static final class TlsStateForTesting { + private final ByteBuffer callerOutputBuffer; + private final SSLEngine sslEngine; + private final int state; + private final ByteBuffer unwrapInputBuffer; + private final ByteBuffer unwrapOutputBuffer; + private final ByteBuffer wrapInputBuffer; + private final ByteBuffer wrapOutputBuffer; + + private TlsStateForTesting(JavaTlsClientSocket socket) { + callerOutputBuffer = socket.callerOutputBuffer; + sslEngine = socket.sslEngine; + state = socket.state; + unwrapInputBuffer = socket.unwrapInputBuffer; + unwrapOutputBuffer = socket.unwrapOutputBuffer; + wrapInputBuffer = socket.wrapInputBuffer; + wrapOutputBuffer = socket.wrapOutputBuffer; + } + + public boolean hasSameStateForTesting(TlsStateForTesting that) { + return callerOutputBuffer == that.callerOutputBuffer + && sslEngine == that.sslEngine + && state == that.state + && unwrapInputBuffer == that.unwrapInputBuffer + && unwrapOutputBuffer == that.unwrapOutputBuffer + && wrapInputBuffer == that.wrapInputBuffer + && wrapOutputBuffer == that.wrapOutputBuffer; + } + } } diff --git a/core/src/main/java/io/questdb/client/network/Net.java b/core/src/main/java/io/questdb/client/network/Net.java index f649d330..b9e669d0 100644 --- a/core/src/main/java/io/questdb/client/network/Net.java +++ b/core/src/main/java/io/questdb/client/network/Net.java @@ -151,6 +151,8 @@ public static void init() { public static native int setTcpNoDelay(int fd, boolean noDelay); + public static native int shutdown(int fd); + public static long sockaddr(int ipv4address, int port) { SOCK_ADDR_COUNTER.incrementAndGet(); return sockaddr0(ipv4address, port); diff --git a/core/src/main/java/io/questdb/client/network/NetworkFacade.java b/core/src/main/java/io/questdb/client/network/NetworkFacade.java index d23824a5..4c66f0df 100644 --- a/core/src/main/java/io/questdb/client/network/NetworkFacade.java +++ b/core/src/main/java/io/questdb/client/network/NetworkFacade.java @@ -78,6 +78,15 @@ public interface NetworkFacade { int setTcpNoDelay(int fd, boolean noDelay); + /** + * Shuts down traffic on a descriptor owned by this facade without releasing + * the descriptor. Custom facades must override this method rather than let + * native code operate on a synthetic or remapped descriptor. + */ + default int shutdown(int fd) { + throw new UnsupportedOperationException("traffic shutdown is not supported by this network facade"); + } + long sockaddr(int address, int port); int socketTcp(boolean blocking); diff --git a/core/src/main/java/io/questdb/client/network/NetworkFacadeImpl.java b/core/src/main/java/io/questdb/client/network/NetworkFacadeImpl.java index 64ea0dc7..1d1d7e20 100644 --- a/core/src/main/java/io/questdb/client/network/NetworkFacadeImpl.java +++ b/core/src/main/java/io/questdb/client/network/NetworkFacadeImpl.java @@ -132,6 +132,11 @@ public int setTcpNoDelay(int fd, boolean noDelay) { return Net.setTcpNoDelay(fd, noDelay); } + @Override + public int shutdown(int fd) { + return Net.shutdown(fd); + } + @Override public long sockaddr(int address, int port) { return Net.sockaddr(address, port); diff --git a/core/src/main/java/io/questdb/client/network/PlainSocket.java b/core/src/main/java/io/questdb/client/network/PlainSocket.java index 555affd2..cc175f51 100644 --- a/core/src/main/java/io/questdb/client/network/PlainSocket.java +++ b/core/src/main/java/io/questdb/client/network/PlainSocket.java @@ -29,7 +29,7 @@ public class PlainSocket implements Socket { private final Logger log; private final NetworkFacade nf; - private int fd = -1; + private volatile int fd = -1; public PlainSocket(NetworkFacade nf, Logger log) { this.nf = nf; @@ -37,13 +37,22 @@ public PlainSocket(NetworkFacade nf, Logger log) { } @Override - public void close() { + public synchronized void close() { if (fd != -1) { nf.close(fd, log); fd = -1; } } + @Override + public synchronized void closeTraffic() { + if (fd != -1 && nf.shutdown(fd) != 0) { + throw new IllegalStateException( + "could not shut down socket traffic [fd=" + fd + ", errno=" + nf.errno() + ']' + ); + } + } + @Override public int getFd() { return fd; diff --git a/core/src/main/java/io/questdb/client/network/Socket.java b/core/src/main/java/io/questdb/client/network/Socket.java index 0cdce517..6fceb30b 100644 --- a/core/src/main/java/io/questdb/client/network/Socket.java +++ b/core/src/main/java/io/questdb/client/network/Socket.java @@ -37,6 +37,17 @@ public interface Socket extends QuietCloseable { int WRITE_FLAG = 1; + /** + * Closes only the network traffic path so a concurrent blocking send or + * receive returns. Implementations must retain any I/O buffers until the + * owning worker has stopped and {@link #close()} performs full cleanup. + * The compatibility default fails without touching implementation-owned + * resources; custom sockets must override this method to opt in. + */ + default void closeTraffic() { + throw new UnsupportedOperationException("traffic shutdown is not supported by this socket"); + } + /** * @return file descriptor associated with the socket. */ diff --git a/core/src/main/java/io/questdb/client/std/DefaultFilesFacade.java b/core/src/main/java/io/questdb/client/std/DefaultFilesFacade.java index dca93e84..194d0972 100644 --- a/core/src/main/java/io/questdb/client/std/DefaultFilesFacade.java +++ b/core/src/main/java/io/questdb/client/std/DefaultFilesFacade.java @@ -86,6 +86,11 @@ public int fsync(int fd) { return Files.fsync(fd); } + @Override + public int fsyncDir(String dir) { + return Files.fsyncDir(dir); + } + @Override public long length(int fd) { return Files.length(fd); @@ -106,6 +111,21 @@ public int mkdir(String path, int mode) { return Files.mkdir(path, mode); } + @Override + public long mmap(int fd, long len, long offset, int flags, int memoryTag) { + return Files.mmap(fd, len, offset, flags, memoryTag); + } + + @Override + public int msync(long addr, long len, boolean async) { + return Files.msync(addr, len, async); + } + + @Override + public void munmap(long address, long len, int memoryTag) { + Files.munmap(address, len, memoryTag); + } + @Override public int openCleanRW(String path) { return Files.openCleanRW(path); @@ -126,6 +146,16 @@ public int openRW(long pathPtr) { return Files.openRW(pathPtr); } + @Override + public int openRWExclusive(String path) { + return Files.openRWExclusive(path); + } + + @Override + public int openRWExclusive(long pathPtr) { + return Files.openRWExclusive(pathPtr); + } + @Override public long length(long pathPtr) { return Files.length(pathPtr); diff --git a/core/src/main/java/io/questdb/client/std/Files.java b/core/src/main/java/io/questdb/client/std/Files.java index 02756c37..522dddc3 100644 --- a/core/src/main/java/io/questdb/client/std/Files.java +++ b/core/src/main/java/io/questdb/client/std/Files.java @@ -24,6 +24,7 @@ package io.questdb.client.std; +import java.io.File; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; @@ -154,6 +155,24 @@ public static int openRW(long pathPtr) { return openRW0(pathPtr); } + /** + * Atomically creates {@code path} for read-write access. Fails with -1 + * when the path already exists; existing bytes are never truncated. + */ + public static int openRWExclusive(String path) { + long ptr = pathPtr(path); + try { + return openRWExclusive0(ptr); + } finally { + freePathPtr(ptr); + } + } + + /** Native-path variant of {@link #openRWExclusive(String)}. */ + public static int openRWExclusive(long pathPtr) { + return openRWExclusive0(pathPtr); + } + /** * Opens {@code path} for append-only writes, creating it (mode 0644) if * absent. Every {@link #append(int, long, long)} writes at end-of-file @@ -408,6 +427,29 @@ public static String utf8ToString(long nameZ) { */ public static native int fsync(int fd); + /** + * Forces directory-entry updates under {@code dir} to durable storage. + * Returns 0 on success and non-zero on failure. Callers use this after + * unlinking files whose absence must survive a host crash. + */ + public static int fsyncDir(String dir) { + long ptr = pathPtr(dir); + try { + return fsyncDir0(ptr); + } finally { + freePathPtr(ptr); + } + } + + /** + * Forces the directory entry naming {@code path} to durable storage by + * syncing its absolute parent directory. + */ + public static int fsyncParentDir(String path) { + File parent = new File(path).getAbsoluteFile().getParentFile(); + return parent == null ? 0 : fsyncDir(parent.getPath()); + } + /** * Truncates the file to exactly {@code size} bytes via {@code ftruncate}. * Returns {@code true} on success. Does NOT reserve disk space — the @@ -531,6 +573,53 @@ public static void munmap(long address, long len, int memoryTag) { */ public static native int msync(long addr, long len, boolean async); + // Guards the optional mlock/munlock natives: a packaged native library + // built before these symbols existed throws UnsatisfiedLinkError on first + // use. Flip to unavailable and report refusal (-1) from then on, so + // callers degrade to their unpinned tier instead of crashing. + private static volatile boolean mlockLinked = true; + + /** + * Best-effort page pin over {@code [addr, addr+len)} of an mmap'd region. + * Whole pages containing any part of the range are locked; {@code addr} + * should be page-aligned for portability. Returns 0 on success, -1 when + * the platform refuses (RLIMIT_MEMLOCK, missing privilege) or the loaded + * native library predates the symbol. Callers must treat refusal as a + * soft downgrade, never an error. + */ + public static int mlock(long addr, long len) { + if (!mlockLinked) { + return -1; + } + try { + return mlock0(addr, len); + } catch (UnsatisfiedLinkError err) { + mlockLinked = false; + return -1; + } + } + + /** + * Releases a pin taken by {@link #mlock(long, long)}. Best-effort: a + * refusal is ignorable ({@code munmap} implicitly drops any remaining + * locks on the range). + */ + public static int munlock(long addr, long len) { + if (!mlockLinked) { + return -1; + } + try { + return munlock0(addr, len); + } catch (UnsatisfiedLinkError err) { + mlockLinked = false; + return -1; + } + } + + private static native int mlock0(long addr, long len); + + private static native int munlock0(long addr, long len); + /** * Returns a native pointer to the current entry's null-terminated name * (UTF-8). Pointer is valid only until the next {@link #findNext(long)} @@ -555,10 +644,14 @@ public static void munmap(long address, long len, int memoryTag) { static native int close0(int fd); + static native int fsyncDir0(long lpszName); + static native int openRO0(long lpszName); static native int openRW0(long lpszName); + static native int openRWExclusive0(long lpszName); + static native int openAppend0(long lpszName); static native int openCleanRW0(long lpszName); diff --git a/core/src/main/java/io/questdb/client/std/FilesFacade.java b/core/src/main/java/io/questdb/client/std/FilesFacade.java index 1b408cf4..921684a0 100644 --- a/core/src/main/java/io/questdb/client/std/FilesFacade.java +++ b/core/src/main/java/io/questdb/client/std/FilesFacade.java @@ -85,21 +85,22 @@ public interface FilesFacade { int fsync(int fd); + default int fsyncDir(String dir) { + return Files.fsyncDir(dir); + } + + /** + * Returns the current byte length of the file referenced by open descriptor + * {@code fd}, or a negative value when the descriptor cannot be statted. + */ long length(int fd); /** - * Stat length of the file at {@code path}, in bytes. Default delegates to - * {@link Files#length(String)}. - * - *

      Test injection point: {@code MmapSegment.openExisting} maps the file to - * this length and scans straight out of the mapping, so a wrapping facade - * that returns a value larger than the real file makes the mapping - * extend past end-of-file. A read of a page beyond real EOF raises SIGBUS on - * every filesystem (which HotSpot translates to a catchable - * {@code InternalError} at an {@code Unsafe} intrinsic site) — the same fault - * a genuinely unbacked/sparse page raises on ZFS, but reproduced - * deterministically on ext4/xfs too. That is what lets recovery's mmap-fault - * guard be regression-tested on any CI runner rather than only on ZFS. + * Stat length of the file at {@code path}, in bytes. + * {@link DefaultFilesFacade} delegates to {@link Files#length(String)}. + * Code that already owns an open descriptor should prefer + * {@link #length(int)} so path replacement cannot make the stat refer to a + * different file. */ long length(String path); @@ -107,6 +108,36 @@ public interface FilesFacade { int mkdir(String path, int mode); + /** + * Best-effort page pin over {@code [addr, addr+len)} of an mmap'd region. + * Returns 0 when the range is locked, non-zero when the platform refuses + * (RLIMIT_MEMLOCK, missing privilege, or a native library without the + * symbol). Callers must treat refusal as a soft downgrade, never an error. + */ + default int mlock(long addr, long len) { + return Files.mlock(addr, len); + } + + default long mmap(int fd, long len, long offset, int flags, int memoryTag) { + return Files.mmap(fd, len, offset, flags, memoryTag); + } + + default int msync(long addr, long len, boolean async) { + return Files.msync(addr, len, async); + } + + /** + * Releases a pin taken by {@link #mlock(long, long)}. Best-effort; + * refusals are ignorable ({@code munmap} implicitly unlocks). + */ + default int munlock(long addr, long len) { + return Files.munlock(addr, len); + } + + default void munmap(long address, long len, int memoryTag) { + Files.munmap(address, len, memoryTag); + } + int openCleanRW(String path); /** @@ -122,12 +153,27 @@ public interface FilesFacade { /** Variant of {@link #openRW(String)} taking a pre-encoded native UTF-8 path pointer. */ int openRW(long pathPtr); + default int openRWExclusive(String path) { + return Files.openRWExclusive(path); + } + + default int openRWExclusive(long pathPtr) { + return Files.openRWExclusive(pathPtr); + } + /** * Variant of {@code length(String)} taking a pre-encoded native UTF-8 path * pointer; same allocation-elision rationale as {@link #openRW(long)}. */ long length(long pathPtr); + /** + * Reads up to {@code len} bytes from the absolute file {@code offset} into + * native memory at {@code addr}, without changing the descriptor position. + * A positive result may be shorter than requested and callers that require + * the whole range must retry. For a non-zero request, {@code 0} means EOF + * or no progress and a negative result indicates an operating-system error. + */ long read(int fd, long addr, long len, long offset); boolean remove(String path); diff --git a/core/src/main/java/io/questdb/client/std/ObjList.java b/core/src/main/java/io/questdb/client/std/ObjList.java index 9e0e7b29..34d989ae 100644 --- a/core/src/main/java/io/questdb/client/std/ObjList.java +++ b/core/src/main/java/io/questdb/client/std/ObjList.java @@ -149,7 +149,7 @@ public void remove(int from, int to) { System.arraycopy(buffer, to + 1, buffer, from, move); } pos = Math.max(0, pos - (to - from + 1)); - Arrays.fill(buffer, pos, buffer.length - 1, null); + Arrays.fill(buffer, pos, buffer.length, null); } public void remove(int index) { diff --git a/core/src/test/java/io/questdb/client/test/SourceHygieneTest.java b/core/src/test/java/io/questdb/client/test/SourceHygieneTest.java new file mode 100644 index 00000000..9351e95d --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/SourceHygieneTest.java @@ -0,0 +1,88 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test; + +import org.junit.Test; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertTrue; + +/** + * Tripwire against mutation-tooling edits leaking into shipped sources. + *

      + * Commit {@code 88d6b792} accidentally captured a concurrent mutation-testing + * edit that neutralized {@code SegmentRing.requestSyncBeforeRotation} to + * {@code return false; // MUTANT: gate neutralized} -- injected into a shared + * worktree between test validation and {@code git add}. The full suite stayed + * green and only human re-reading caught it ({@code 71cfbe2e}). Mutation + * tools mark their edits precisely so they can be found; this test makes that + * marker a build failure instead of a code-review lottery ticket. + */ +public class SourceHygieneTest { + + private static final String MARKER = "MUTANT"; + + @Test + public void testNoMutationToolMarkersInMainSources() throws IOException { + // Surefire runs with the module directory (core/) as cwd. + Path root = Paths.get("src", "main", "java"); + if (!Files.isDirectory(root)) { + root = Paths.get("core", "src", "main", "java"); + } + assertTrue("main source root not found from " + Paths.get("").toAbsolutePath() + + " -- fix the path resolution rather than skipping the tripwire", + Files.isDirectory(root)); + + final List offenders = new ArrayList<>(); + Files.walkFileTree(root, new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + if (file.toString().endsWith(".java")) { + List lines = Files.readAllLines(file, StandardCharsets.UTF_8); + for (int i = 0; i < lines.size(); i++) { + if (lines.get(i).contains(MARKER)) { + offenders.add(file + ":" + (i + 1) + " " + lines.get(i).trim()); + } + } + } + return FileVisitResult.CONTINUE; + } + }); + assertTrue("mutation-tool markers must never reach main sources " + + "(a neutralized durability gate shipped exactly this way in 88d6b792); " + + "offending lines:\n" + String.join("\n", offenders), + offenders.isEmpty()); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/CloseDrainTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/CloseDrainTest.java index 9f1500f2..e4fb86c5 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/CloseDrainTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/CloseDrainTest.java @@ -35,8 +35,10 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.file.Paths; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; /** * Regression tests for the close() drain semantics. @@ -49,6 +51,83 @@ */ public class CloseDrainTest { + @Test(timeout = 30_000L) + public void testCloseBlocksAcrossAllReplicaWindowUntilPromotion() throws Exception { + DelayingAckHandler handler = new DelayingAckHandler(0); + try (TestWebSocketServer server = new TestWebSocketServer(handler, false, "PRIMARY")) { + server.setRejectWithRole("REPLICA"); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + String cfg = "ws::addr=localhost:" + server.getPort() + + ";initial_connect_retry=async" + + ";reconnect_initial_backoff_millis=20" + + ";reconnect_max_backoff_millis=100" + + ";close_flush_timeout_millis=10000;"; + QwpWebSocketSender sender = (QwpWebSocketSender) Sender.fromConfig(cfg); + CountDownLatch closeDrainWaiting = new CountDownLatch(1); + CountDownLatch releaseCloseDrain = new CountDownLatch(1); + AtomicReference closeFailure = new AtomicReference<>(); + AtomicReference hookFailure = new AtomicReference<>(); + sender.setCloseDrainWaitingHook(() -> { + closeDrainWaiting.countDown(); + try { + if (!releaseCloseDrain.await(10, TimeUnit.SECONDS)) { + throw new AssertionError("promotion did not release the close-drain witness"); + } + } catch (Throwable t) { + hookFailure.set(t); + } + }); + sender.table("foo").longColumn("v", 1L).atNow(); + sender.flush(); + + Thread closer = new Thread(() -> { + try { + sender.close(); + } catch (Throwable t) { + closeFailure.set(t); + } + }, "all-replica-close-drain"); + try { + closer.start(); + Assert.assertTrue("server never produced the all-replica role rejection", + server.awaitRoleReject(5, TimeUnit.SECONDS)); + Assert.assertTrue("close never observed its real unacknowledged drain target", + closeDrainWaiting.await(5, TimeUnit.SECONDS)); + + Assert.assertTrue("pre-promotion witness must include a role rejection", + server.roleRejectCount() >= 1); + Assert.assertEquals("pre-promotion data must remain unacknowledged", + -1L, sender.getAckedFsn()); + Assert.assertTrue("close must remain pending for the whole all-replica window", + closer.isAlive()); + Assert.assertEquals("promotion must not have delivered or replayed the frame yet", + 0L, handler.nextSeq.get()); + + // The close-drain barrier has held continuously since it observed + // targetFsn > ackedFsn. Promote first, then release the barrier: + // completion therefore cannot precede the deterministic recovery event. + server.setAdvertisedRole("PRIMARY"); + server.setRejectWithRole(null); + releaseCloseDrain.countDown(); + closer.join(10_000L); + + Assert.assertFalse("close did not complete after promotion", closer.isAlive()); + Assert.assertNull("close-drain witness failed", hookFailure.get()); + Assert.assertNull("close failed after promotion", closeFailure.get()); + Assert.assertEquals("the unacknowledged frame must be delivered exactly once", + 1L, handler.nextSeq.get()); + } finally { + server.setAdvertisedRole("PRIMARY"); + server.setRejectWithRole(null); + releaseCloseDrain.countDown(); + closer.join(10_000L); + sender.close(); + } + } + } + @Test public void testCloseBlocksUntilAckArrives() throws Exception { // Server delays every ACK by 800ms. With the default @@ -77,6 +156,56 @@ public void testCloseBlocksUntilAckArrives() throws Exception { } } + @Test + public void testCloseStartedHookRunsAfterClosedStateTransition() throws Exception { + QwpWebSocketSender sender = QwpWebSocketSender.createForTesting("localhost", 1); + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + AtomicReference closerFailure = new AtomicReference<>(); + AtomicReference hookFailure = new AtomicReference<>(); + sender.setCloseStartedHook(() -> { + try { + try { + sender.table("must_reject_after_close_started"); + Assert.fail("close-started hook ran before closed=true was published"); + } catch (LineSenderException expected) { + // Required lifecycle boundary: public operations already reject. + } + entered.countDown(); + if (!release.await(10, TimeUnit.SECONDS)) { + throw new AssertionError("close-started hook release timed out"); + } + } catch (Throwable t) { + hookFailure.set(t); + entered.countDown(); + } + }); + + Assert.assertEquals("installing the hook must not emit a pre-invocation witness", + 1L, entered.getCount()); + Thread closer = new Thread(() -> { + try { + sender.close(); + } catch (Throwable t) { + closerFailure.set(t); + } + }, "close-started-hook-test"); + try { + closer.start(); + Assert.assertTrue("close() never reached its internal lifecycle witness", + entered.await(10, TimeUnit.SECONDS)); + Assert.assertTrue("close() completed while its internal witness was held", + closer.isAlive()); + } finally { + release.countDown(); + closer.join(10_000L); + sender.close(); + } + Assert.assertFalse("close thread did not finish", closer.isAlive()); + Assert.assertNull("close-started hook failed", hookFailure.get()); + Assert.assertNull("close() failed", closerFailure.get()); + } + @Test public void testCloseFastWhenTimeoutIsZero() throws Exception { // Same delayed-ACK server, but with close_flush_timeout_millis=0 diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/InitialConnectAsyncTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/InitialConnectAsyncTest.java index 173374ec..5f04ca68 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/InitialConnectAsyncTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/InitialConnectAsyncTest.java @@ -27,6 +27,7 @@ import io.questdb.client.Sender; import io.questdb.client.SenderError; import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; +import io.questdb.client.cutlass.qwp.websocket.WebSocketCloseCode; import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; import org.jetbrains.annotations.NotNull; import org.junit.Assert; @@ -43,8 +44,10 @@ import java.nio.charset.StandardCharsets; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.LongSupplier; /** * Behavior of {@code initial_connect_retry=async}: the producer-thread @@ -205,6 +208,78 @@ public void testAsyncDeliversBufferedRowsWhenServerArrivesLate() throws Exceptio } } + @Test + public void testAsyncMetricsDistinguishInitialSendFromReconnectReplay() throws Exception { + ReplayMetricsHandler handler = new ReplayMetricsHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + String cfg = "ws::addr=localhost:" + server.getPort() + + sfDirOpt() + ";initial_connect_retry=async" + + ";reconnect_initial_backoff_millis=20" + + ";reconnect_max_backoff_millis=200" + + ";close_flush_timeout_millis=2000;"; + Sender sender = Sender.fromConfig(cfg); + try { + QwpWebSocketSender wss = (QwpWebSocketSender) sender; + handler.bind(wss, server); + + // Publish before the server starts accepting. The frame has + // never been on any wire, so its eventual first delivery must + // increase sent, but not replayed. + sender.table("foo").longColumn("v", 1L).atNow(); + sender.flush(); + awaitAtLeastOneConnectAttempt(wss); + Assert.assertEquals("the server must not handshake before start", 0, server.handshakeCount()); + + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + Assert.assertTrue("the initial frame must be ACKed", + handler.awaitInitialAck(5, TimeUnit.SECONDS)); + awaitMetricAtLeast("initial ACK", wss::getTotalAcks, 1L); + Assert.assertEquals("exactly one frame must be sent initially", + 1L, wss.getTotalFramesSent()); + long replayedAfterInitialDelivery = wss.getTotalFramesReplayed(); + + // The second frame reaches the established connection, but the + // fixture closes that connection without ACKing it. It must then + // arrive again on a genuinely new connection and count once as + // replayed. + sender.table("foo").longColumn("v", 2L).atNow(); + sender.flush(); + Assert.assertTrue("the fixture must close after the unacked frame", + handler.awaitDisconnect(5, TimeUnit.SECONDS)); + Assert.assertTrue("the reconnect must reach the gated server", + server.awaitRoleReject(5, TimeUnit.SECONDS)); + + // Publish another frame while reconnect is blocked at a role + // reject, then reopen the gate. This frame has never been sent + // and must not inflate the replay counter after reconnect. + sender.table("foo").longColumn("v", 3L).atNow(); + sender.flush(); + server.setRejectWithRole(null); + + Assert.assertTrue("the unacked frame must be replayed", + handler.awaitReplay(5, TimeUnit.SECONDS)); + Assert.assertTrue("the outage-queued frame must be delivered", + handler.awaitOutageQueuedDelivery(5, TimeUnit.SECONDS)); + awaitMetricAtLeast("replayed frame", wss::getTotalFramesReplayed, 1L); + awaitMetricAtLeast("post-reconnect ACKs", wss::getTotalAcks, 3L); + + Assert.assertTrue("replay must arrive on a new WebSocket connection", + handler.wasReplayOnNewConnection()); + Assert.assertTrue("the server must observe a reconnect", + server.handshakeCount() >= 2); + Assert.assertEquals("initial delivery is not a replay", + 0L, replayedAfterInitialDelivery); + Assert.assertEquals("three first sends plus one genuine resend", + 4L, wss.getTotalFramesSent()); + Assert.assertEquals("only the genuine resend is replayed", + 1L, wss.getTotalFramesReplayed()); + } finally { + closeQuietly(sender); + } + } + } + @Test public void testAsyncReturnsImmediatelyWithNoServer() { // No server. With async mode, fromConfig must return fast — the @@ -345,6 +420,17 @@ private static void awaitAtLeastOneConnectAttempt(QwpWebSocketSender wss) { * before the budget expired would satisfy it even if the loop then * exited. */ + private static void awaitMetricAtLeast(String label, LongSupplier metric, long expected) { + long deadlineNanos = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (metric.getAsLong() < expected) { + if (System.nanoTime() > deadlineNanos) { + throw new AssertionError(label + " did not reach " + expected + + " within 5s; current=" + metric.getAsLong()); + } + io.questdb.client.std.Compat.onSpinWait(); + } + } + private static void awaitReconnectAttemptsAdvance(QwpWebSocketSender wss) { long snapshot = wss.getTotalReconnectAttempts(); long deadlineNanos = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); @@ -476,6 +562,83 @@ public void onError(@NotNull SenderError err) { } } + private static class ReplayMetricsHandler implements TestWebSocketServer.WebSocketServerHandler { + private final CountDownLatch disconnect = new CountDownLatch(1); + private final CountDownLatch initialAck = new CountDownLatch(1); + private final CountDownLatch outageQueuedDelivery = new CountDownLatch(1); + private final AtomicLong received = new AtomicLong(); + private final CountDownLatch replay = new CountDownLatch(1); + private final AtomicBoolean replayOnNewConnection = new AtomicBoolean(); + private TestWebSocketServer.ClientHandler initialClient; + private volatile TestWebSocketServer server; + private volatile QwpWebSocketSender sender; + + boolean awaitDisconnect(long timeout, TimeUnit unit) throws InterruptedException { + return disconnect.await(timeout, unit); + } + + boolean awaitInitialAck(long timeout, TimeUnit unit) throws InterruptedException { + return initialAck.await(timeout, unit); + } + + boolean awaitOutageQueuedDelivery(long timeout, TimeUnit unit) throws InterruptedException { + return outageQueuedDelivery.await(timeout, unit); + } + + boolean awaitReplay(long timeout, TimeUnit unit) throws InterruptedException { + return replay.await(timeout, unit); + } + + void bind(QwpWebSocketSender sender, TestWebSocketServer server) { + this.sender = sender; + this.server = server; + } + + @Override + public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + long ordinal = received.incrementAndGet(); + try { + if (ordinal == 1L) { + initialClient = client; + client.sendBinary(AckHandler.buildAck(0L)); + initialAck.countDown(); + } else if (ordinal == 2L) { + // Receipt alone can race the sender's post-send metric + // increments. Wait until sendBinary returned before closing, + // proving this frame completed a send and must be replayed. + awaitSentFrames(2L); + server.setRejectWithRole("REPLICA"); + client.sendClose(WebSocketCloseCode.GOING_AWAY, "test reconnect"); + disconnect.countDown(); + } else if (ordinal == 3L) { + replayOnNewConnection.set(client != initialClient); + client.sendBinary(AckHandler.buildAck(0L)); + replay.countDown(); + } else if (ordinal == 4L) { + client.sendBinary(AckHandler.buildAck(1L)); + outageQueuedDelivery.countDown(); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + boolean wasReplayOnNewConnection() { + return replayOnNewConnection.get(); + } + + private void awaitSentFrames(long expected) { + long deadlineNanos = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (sender.getTotalFramesSent() < expected) { + if (System.nanoTime() > deadlineNanos) { + throw new AssertionError("sent-frame metric did not reach " + expected + + " before disconnect; current=" + sender.getTotalFramesSent()); + } + io.questdb.client.std.Compat.onSpinWait(); + } + } + } + /** * Raw-socket fixture: every accepted connection responds with HTTP * 401 Unauthorized and closes. Used to drive the async-init diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderCursorEngineAttachmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderCursorEngineAttachmentTest.java new file mode 100644 index 00000000..9184fd3c --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderCursorEngineAttachmentTest.java @@ -0,0 +1,191 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegmentException; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; + +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpWebSocketSenderCursorEngineAttachmentTest { + + private static final long SEGMENT_SIZE = 64 * 1024L; + + @Rule + public final TemporaryFolder sfDir = TemporaryFolder.builder().assureDeletion().build(); + + @Test + public void testNullDoesNotDetachAnAttachedEngine() throws Exception { + assertMemoryLeak(() -> { + CursorSendEngine engine = new CursorSendEngine(null, SEGMENT_SIZE); + QwpWebSocketSender sender = QwpWebSocketSender.createForTesting("localhost", 1); + try { + sender.setCursorEngine(engine, true); + assertSecondAttachmentRejected(sender, null, false); + sender.close(); + Assert.assertTrue("sender must retain ownership after rejected detach", + engine.isCloseCompleted()); + } finally { + sender.close(); + engine.close(); + } + }); + } + + @Test + public void testOwnedEngineCannotBeReplacedAndItsSlotIsReleased() throws Exception { + assertMemoryLeak(() -> { + File slotDir = new File(sfDir.getRoot(), "owned-slot"); + CursorSendEngine first = null; + CursorSendEngine replacement = null; + CursorSendEngine reacquired = null; + QwpWebSocketSender sender = null; + boolean replacementRejected = false; + try { + first = new CursorSendEngine(slotDir.getAbsolutePath(), SEGMENT_SIZE); + replacement = new CursorSendEngine(null, SEGMENT_SIZE); + sender = QwpWebSocketSender.createForTesting("localhost", 1); + sender.setCursorEngine(first, true); + try { + sender.setCursorEngine(replacement, true); + } catch (LineSenderException e) { + replacementRejected = true; + Assert.assertTrue(e.getMessage().contains("already attached")); + } + + sender.close(); + sender = null; + try { + reacquired = new CursorSendEngine(slotDir.getAbsolutePath(), SEGMENT_SIZE); + } catch (IllegalStateException e) { + throw new AssertionError("sender close did not release the first owned " + + "engine's slot after a second attachment attempt", e); + } + Assert.assertTrue("second attachment must be rejected", replacementRejected); + } finally { + if (sender != null) { + sender.close(); + } + if (reacquired != null) { + reacquired.close(); + } + if (replacement != null) { + replacement.close(); + } + if (first != null) { + first.close(); + } + } + }); + } + + @Test + public void testSameSharedEngineCannotTransferOwnership() throws Exception { + assertMemoryLeak(() -> { + CursorSendEngine engine = new CursorSendEngine(null, SEGMENT_SIZE); + QwpWebSocketSender sender = QwpWebSocketSender.createForTesting("localhost", 1); + try { + sender.setCursorEngine(engine, false); + assertSecondAttachmentRejected(sender, engine, true); + sender.close(); + Assert.assertFalse("rejected attachment must not transfer ownership", + engine.isCloseCompleted()); + } finally { + sender.close(); + engine.close(); + } + }); + } + + @Test + public void testLatchedDurabilityFailureSurfacesThroughSenderFlushAndAwait() throws Exception { + assertMemoryLeak(() -> { + // The ring gate (SegmentRing.appendOrFsn) guards every publish + // path, but an EMPTY flush()/flushAndGetSequence() and a + // timeoutMillis <= 0 awaitAckedFsn() poll publish nothing: the + // sender-level cursorEngine.checkDurability() call sites are + // their ONLY durability act. Without them an empty flush + // silently succeeds against an unsyncable slot and a poll loop + // spins on "not yet" forever instead of surfacing the latched + // barrier failure. Deleting any of the three sender call sites + // previously survived the whole suite -- this pins the + // user-visible layer (the engine seam has its own pin in + // CursorSendEngineTest). + CursorSendEngine engine = new CursorSendEngine(null, SEGMENT_SIZE); + QwpWebSocketSender sender = QwpWebSocketSender.createForTesting("localhost", 1); + try { + sender.setCursorEngine(engine, false); + MmapSegmentException failure = new MmapSegmentException("injected data-sync failure"); + engine.getRingForTesting().recordDurabilityFailureForTesting(failure); + for (int i = 0; i < 2; i++) { // repeatable until cleared, not one-shot + try { + sender.flush(); + Assert.fail("empty flush() must surface the latched durability failure, call #" + i); + } catch (MmapSegmentException expected) { + Assert.assertSame("the latched instance itself must surface", failure, expected); + } + try { + sender.flushAndGetSequence(); + Assert.fail("empty flushAndGetSequence() must surface the latched durability " + + "failure, call #" + i); + } catch (MmapSegmentException expected) { + Assert.assertSame(failure, expected); + } + try { + sender.awaitAckedFsn(0L, 0L); + Assert.fail("awaitAckedFsn(fsn, 0) must throw instead of polling 'not yet', " + + "call #" + i); + } catch (MmapSegmentException expected) { + Assert.assertSame(failure, expected); + } + } + } finally { + sender.close(); + engine.close(); + } + }); + } + + private static void assertSecondAttachmentRejected( + QwpWebSocketSender sender, + CursorSendEngine engine, + boolean takeOwnership + ) { + try { + sender.setCursorEngine(engine, takeOwnership); + Assert.fail("expected second cursor engine attachment to be rejected"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("already attached")); + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SlotLockReleasedContractTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SlotLockReleasedContractTest.java index 85522e9d..cc501a8c 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SlotLockReleasedContractTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SlotLockReleasedContractTest.java @@ -24,9 +24,22 @@ package io.questdb.client.test.cutlass.qwp.client; +import io.questdb.client.DefaultHttpClientConfiguration; import io.questdb.client.Sender; +import io.questdb.client.SenderConnectionEvent; +import io.questdb.client.SenderError; +import io.questdb.client.cutlass.http.client.WebSocketClient; import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; +import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderConnectionDispatcher; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderErrorDispatcher; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderProgressDispatcher; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SlotLock; +import io.questdb.client.network.PlainSocketFactory; +import io.questdb.client.std.Files; import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; import io.questdb.client.test.tools.TestUtils; import org.junit.Assert; @@ -36,8 +49,12 @@ import java.lang.reflect.Field; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import java.nio.file.Paths; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.LockSupport; /** @@ -46,9 +63,13 @@ *

        *
      • Happy path — a clean {@code close()} that winds the I/O thread * down reports {@code isSlotLockReleased() == true}.
        • - *
      • Leak path — a {@code close()} that early-returns because the I/O - * thread refused to stop ({@code ioThreadStopped == false}) reports + *
      • Leak path — a {@code close()} that retains the lock because an + * I/O or manager worker did not stop reports * {@code isSlotLockReleased() == false}.
        • + *
      • Recovery path — the getter is monotonic but not frozen: it + * re-probes the retained engine, so once the deferred cleanup (worker + * exit path or a retried close) releases the flock, the getter flips to + * {@code true} and a pool that retired the slot can recover it.
        • *
      * The pool's {@code flockReleased(s)} treats {@code false} as "flock still held, * retire the slot permanently". Before this test that contract was driven only @@ -178,6 +199,380 @@ public void testSlotLockNotReleasedWhenIoThreadRefusesToStop() throws Exception }); } + /** + * Manager-worker handoff path: engine close retains the slot while a shared + * manager is still inside a service pass. The sender must expose that + * retained flock to SenderPool. Repeated sender close calls remain no-ops; + * the manager pass owns deferred cleanup and releases the flock when it + * finishes, so the sender can expose recovery without a direct close retry. + */ + @Test(timeout = 30_000L) + public void testSlotLockNotReleasedUntilManagerWorkerQuiesces() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String tmpDir = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-slot-lock-manager-" + System.nanoTime()).toString(); + Assert.assertEquals(0, Files.mkdir(tmpDir, Files.DIR_MODE_DEFAULT)); + String slot = tmpDir + "/slot"; + long segSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 32L; + SegmentManager manager = new SegmentManager(segSize, TimeUnit.SECONDS.toNanos(60)); + CountDownLatch cleanupFinished = new CountDownLatch(1); + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(); + AtomicReference hookErr = new AtomicReference<>(); + CursorSendEngine engine = null; + QwpWebSocketSender wss = null; + boolean managerClosed = false; + try { + manager.setAfterRingCleanupHook(cleanupFinished::countDown); + manager.setBeforeInstallSyncHook(() -> { + if (!fired.compareAndSet(false, true)) return; + workerBlocked.countDown(); + try { + if (!releaseWorker.await(20, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, + new AssertionError("timed out waiting for test to release worker")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + manager.start(); + engine = new CursorSendEngine(slot, segSize, manager); + Assert.assertTrue("worker never reached install hook", + workerBlocked.await(5, TimeUnit.SECONDS)); + + wss = QwpWebSocketSender.createForTesting("localhost", 1); + wss.setCursorEngine(engine, true); + manager.setWorkerJoinTimeoutMillis(50L); + wss.close(); + + Assert.assertFalse("sender reported a retained flock as released", + wss.isSlotLockReleased()); + Assert.assertFalse("engine close must remain incomplete while worker is in service", + engine.isCloseCompleted()); + try { + SlotLock probe = SlotLock.acquire(slot); + probe.close(); + Assert.fail("slot became acquirable while manager worker was still in service"); + } catch (Exception expected) { + // good — the incomplete close retained the flock. + } + + // Sender.close() is idempotent: a second call must not retry + // or change ownership while the manager worker is still live. + wss.close(); + Assert.assertFalse("repeated close changed retained-flock reporting", + wss.isSlotLockReleased()); + Assert.assertFalse("repeated close unexpectedly retried engine cleanup", + engine.isCloseCompleted()); + + releaseWorker.countDown(); + Assert.assertTrue("shared-manager pass did not finish deferred cleanup", + cleanupFinished.await(5, TimeUnit.SECONDS)); + Assert.assertTrue("deferred cleanup did not complete after the ring pass", + engine.isCloseCompleted()); + // Recovery contract: the sender re-probes its retained engine, + // so the completed cleanup (and released flock) MUST become + // visible — this is what lets SenderPool recover a slot it + // had already retired as leaked. + Assert.assertTrue("sender must expose the late flock release to the pool", + wss.isSlotLockReleased()); + try (SlotLock probe = SlotLock.acquire(slot)) { + Assert.assertNotNull("slot must be acquirable after deferred cleanup", probe); + } + + manager.close(); + managerClosed = true; + if (hookErr.get() != null) { + throw new AssertionError("install hook failed", hookErr.get()); + } + } finally { + manager.setAfterRingCleanupHook(null); + manager.setBeforeInstallSyncHook(null); + releaseWorker.countDown(); + if (wss != null) { + try { + wss.close(); + } catch (Throwable ignored) { + } + } + if (!managerClosed) { + manager.close(); + } + rmDirRecursive(tmpDir); + Files.remove(tmpDir); + } + }); + } + + /** + * Delegated-I/O-close recovery path: when {@code close()} bails early + * because the I/O thread refuses to stop, the engine close is delegated + * to that thread's exit path ({@code delegateEngineClose()} returned + * true) and the sender must retain the engine for re-probing — + * {@code isSlotLockReleased()} stays false while the thread lives, then + * flips true the moment the thread's exit path completes the engine + * close and releases the flock. Pre-fix, this branch discarded the + * engine reference, so the late release was permanently invisible to a + * pool that had retired the slot. The existing forged I/O-refusal test + * can never reach this branch: its sabotaged loop throws from + * {@code delegateEngineClose()} itself (nulled latch), before the + * retained-engine assignment. + *

      + * The wedge is the same deterministic one CursorWebSocketSendLoop's C5 + * test uses: the I/O thread sits in a blocking "connect" (a + * ReconnectFactory immune to unpark, never interrupted by loop.close()), + * and the closer thread's pre-set interrupt flag makes + * {@code shutdownLatch.await()} throw immediately — the real production + * path to {@code ioThreadStopped = false}. + */ + @Test(timeout = 30_000L) + public void testDelegatedIoThreadEngineCloseFlipsSlotLockReleased() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String tmpDir = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-slot-lock-delegated-" + System.nanoTime()).toString(); + Assert.assertEquals(0, Files.mkdir(tmpDir, Files.DIR_MODE_DEFAULT)); + String slot = tmpDir + "/slot"; + long segSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 32L; + final CountDownLatch enteredConnect = new CountDownLatch(1); + final CountDownLatch releaseConnect = new CountDownLatch(1); + final AtomicReference ioThreadRef = new AtomicReference<>(); + final StubWebSocketClient stubClient = new StubWebSocketClient(); + // Healthy owned manager: once the wedged I/O thread is released, + // its exit-path engine.close() completes normally and releases + // the flock — the flip this test pins. + CursorSendEngine engine = new CursorSendEngine(slot, segSize); + CursorWebSocketSendLoop loop = null; + QwpWebSocketSender wss = null; + try { + // Stand-in for a blocking native connect(2): entered by the + // loop's I/O thread, immune to unpark, never interrupted by + // loop.close(). + CursorWebSocketSendLoop.ReconnectFactory stuckConnect = () -> { + ioThreadRef.set(Thread.currentThread()); + enteredConnect.countDown(); + releaseConnect.await(); + return stubClient; + }; + loop = new CursorWebSocketSendLoop( + null /* async-initial-connect: the I/O thread drives the connect */, + engine, 0L, 1_000L, + stuckConnect, + 5_000L, 100L, 5_000L, false); + loop.start(); + Assert.assertTrue("I/O thread never reached the connect factory", + enteredConnect.await(5, TimeUnit.SECONDS)); + + wss = QwpWebSocketSender.createForTesting("localhost", 1); + wss.setCursorEngine(engine, true); + wss.setCursorSendLoopForTesting(loop); + + // Drive the real early-bail close() on a thread whose pending + // interrupt lands in loop.close()'s shutdownLatch.await(). + AtomicReference closeFailure = new AtomicReference<>(); + QwpWebSocketSender wssRef = wss; + Thread closer = new Thread(() -> { + Thread.currentThread().interrupt(); + try { + wssRef.close(); + } catch (Throwable t) { + closeFailure.set(t); + } + }, "delegated-close-closer"); + closer.setDaemon(true); + closer.start(); + closer.join(TimeUnit.SECONDS.toMillis(10)); + Assert.assertFalse("closer thread did not finish", closer.isAlive()); + Assert.assertNotNull("close() must surface the failed I/O-thread stop", + closeFailure.get()); + + // The I/O thread is still wedged: the flock is retained and + // must be reported retained. + Assert.assertFalse( + "isSlotLockReleased() must be false while the delegated engine " + + "close is pending on the wedged I/O thread", + wss.isSlotLockReleased()); + Assert.assertFalse("engine close must not have run yet", + engine.isCloseCompleted()); + try { + SlotLock probe = SlotLock.acquire(slot); + probe.close(); + Assert.fail("slot became acquirable while the delegated engine close " + + "was still pending"); + } catch (Exception expected) { + // good — the flock is genuinely held. + } + + // Un-wedge the connect. The I/O thread exits; its exit path + // runs the delegated engine.close(), which releases the flock. + releaseConnect.countDown(); + Thread ioThread = ioThreadRef.get(); + Assert.assertNotNull(ioThread); + ioThread.join(TimeUnit.SECONDS.toMillis(10)); + Assert.assertFalse("I/O thread did not exit after the connect returned", + ioThread.isAlive()); + + // The recovery contract under test: the getter re-probes the + // retained engine, so the late release MUST become visible — + // this is what lets SenderPool recover the retired slot. + Assert.assertTrue("delegated engine close must have completed on the " + + "I/O thread's exit path", + engine.isCloseCompleted()); + Assert.assertTrue( + "isSlotLockReleased() must flip true once the delegated engine " + + "close released the flock — otherwise the pool retires the " + + "slot's capacity until process exit", + wss.isSlotLockReleased()); + try (SlotLock probe = SlotLock.acquire(slot)) { + Assert.assertNotNull("slot must be acquirable after the delegated close", probe); + } + } finally { + releaseConnect.countDown(); + // Reap the loop's bookkeeping now that the I/O thread is gone + // (close() threw mid-teardown, so ioThread was left set). + Thread.interrupted(); + if (loop != null) { + try { + loop.close(); + } catch (Throwable ignored) { + } + } + if (engine != null && !engine.isCloseCompleted()) { + try { + engine.close(); + } catch (Throwable ignored) { + } + } + // The early-return close() deliberately leaked the resources + // the (then-running) I/O thread might touch; free the same set + // the post-guard tail would have freed. + if (wss != null) { + freeFieldQuietly(wss, "buffer0"); + freeFieldQuietly(wss, "buffer1"); + freeFieldQuietly(wss, "client"); + freeFieldQuietly(wss, "errorDispatcher"); + freeFieldQuietly(wss, "progressDispatcher"); + freeFieldQuietly(wss, "connectionDispatcher"); + } + stubClient.close(); + rmDirRecursive(tmpDir); + Files.remove(tmpDir); + } + }); + } + + @Test(timeout = 30_000L) + public void testFailedIoStopReclaimsSenderResourcesAfterWorkerExit() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String tmpDir = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-slot-lock-full-cleanup-" + System.nanoTime()).toString(); + Assert.assertEquals(0, Files.mkdir(tmpDir, Files.DIR_MODE_DEFAULT)); + String slot = tmpDir + "/slot"; + long segSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 32L; + CountDownLatch enteredConnect = new CountDownLatch(1); + CountDownLatch releaseConnect = new CountDownLatch(1); + AtomicReference ioThreadRef = new AtomicReference<>(); + StubWebSocketClient loopClient = new StubWebSocketClient(); + StubWebSocketClient senderClient = new StubWebSocketClient(); + CursorSendEngine engine = new CursorSendEngine(slot, segSize); + CursorWebSocketSendLoop loop = null; + try { + CursorWebSocketSendLoop.ReconnectFactory stuckConnect = () -> { + ioThreadRef.set(Thread.currentThread()); + enteredConnect.countDown(); + releaseConnect.await(); + return loopClient; + }; + loop = new CursorWebSocketSendLoop( + null, engine, 0L, 1_000L, stuckConnect, + 5_000L, 100L, 5_000L, false); + loop.start(); + Assert.assertTrue("I/O thread never reached the connect factory", + enteredConnect.await(5, TimeUnit.SECONDS)); + + QwpWebSocketSender sender = QwpWebSocketSender.createForTesting("localhost", 1); + sender.setConnectionListener(event -> { + }); + sender.setErrorHandler(error -> { + }); + sender.setProgressHandler(ackedFsn -> { + }); + sender.setClientForTesting(senderClient); + sender.setCursorEngine(engine, true); + sender.setCursorSendLoopForTesting(loop); + + SenderConnectionDispatcher connectionDispatcher = sender.getConnectionDispatcherForTesting(); + SenderErrorDispatcher errorDispatcher = sender.getErrorDispatcherForTesting(); + SenderProgressDispatcher progressDispatcher = sender.getProgressDispatcherForTesting(); + Assert.assertTrue(connectionDispatcher.offer(new SenderConnectionEvent( + SenderConnectionEvent.Kind.DISCONNECTED, + null, SenderConnectionEvent.NO_PORT, + null, SenderConnectionEvent.NO_PORT, + SenderConnectionEvent.NO_ATTEMPT_NUMBER, + SenderConnectionEvent.NO_ROUND_NUMBER, + null, 0L))); + Assert.assertTrue(errorDispatcher.offer(new SenderError( + SenderError.Category.UNKNOWN, SenderError.Policy.RETRIABLE, + SenderError.NO_STATUS_BYTE, null, SenderError.NO_MESSAGE_SEQUENCE, + -1L, -1L, null, 0L))); + Assert.assertTrue(progressDispatcher.offer(0L)); + Thread connectionDispatcherThread = connectionDispatcher.getWorkerThreadForTesting(); + Thread errorDispatcherThread = errorDispatcher.getWorkerThreadForTesting(); + Thread progressDispatcherThread = progressDispatcher.getWorkerThreadForTesting(); + Assert.assertNotNull(connectionDispatcherThread); + Assert.assertNotNull(errorDispatcherThread); + Assert.assertNotNull(progressDispatcherThread); + + AtomicReference closeFailure = new AtomicReference<>(); + Thread closer = new Thread(() -> { + Thread.currentThread().interrupt(); + try { + sender.close(); + } catch (Throwable t) { + closeFailure.set(t); + } + }, "full-cleanup-closer"); + closer.start(); + closer.join(TimeUnit.SECONDS.toMillis(10)); + Assert.assertFalse("closer thread did not finish", closer.isAlive()); + Assert.assertNotNull("close() must surface the failed I/O-thread stop", + closeFailure.get()); + + releaseConnect.countDown(); + Thread ioThread = ioThreadRef.get(); + Assert.assertNotNull(ioThread); + ioThread.join(TimeUnit.SECONDS.toMillis(10)); + Assert.assertFalse("I/O thread did not exit after release", ioThread.isAlive()); + Assert.assertTrue("complete sender cleanup callback did not finish", + sender.isCloseCleanupComplete()); + Assert.assertTrue("loop-owned WebSocket client was not closed", loopClient.isClosed()); + Assert.assertTrue("sender-owned WebSocket client was not closed", senderClient.isClosed()); + Assert.assertFalse("connection dispatcher worker was not reclaimed", + connectionDispatcherThread.isAlive()); + Assert.assertFalse("error dispatcher worker was not reclaimed", + errorDispatcherThread.isAlive()); + Assert.assertFalse("progress dispatcher worker was not reclaimed", + progressDispatcherThread.isAlive()); + Assert.assertTrue("engine cleanup did not complete", engine.isCloseCompleted()); + try (SlotLock probe = SlotLock.acquire(slot)) { + Assert.assertNotNull(probe); + } + } finally { + releaseConnect.countDown(); + Thread.interrupted(); + if (loop != null) { + try { + loop.close(); + } catch (Throwable ignored) { + } + } + rmDirRecursive(tmpDir); + Files.remove(tmpDir); + } + }); + } + // ------------------------------------------------------------------ utils private static void freeFieldQuietly(Object target, String name) { @@ -192,6 +587,28 @@ private static void freeFieldQuietly(Object target, String name) { } } + private static void rmDirRecursive(String dir) { + if (!Files.exists(dir)) return; + long find = Files.findFirst(dir); + if (find <= 0) return; + try { + int rc = 1; + while (rc > 0) { + String name = Files.utf8ToString(Files.findName(find)); + if (name != null && !".".equals(name) && !"..".equals(name)) { + String child = dir + "/" + name; + if (!Files.remove(child)) { + rmDirRecursive(child); + Files.remove(child); + } + } + rc = Files.findNext(find); + } + } finally { + Files.findClose(find); + } + } + private static T readField(Object target, String name, Class type) throws Exception { Class cls = target.getClass(); while (cls != null) { @@ -221,6 +638,39 @@ private static void setField(Object target, String name, Object value) throws Ex throw new NoSuchFieldException(name); } + /** + * Minimal concrete {@link WebSocketClient} — never performs I/O. Handed + * to the loop by the stuck-connect factory; the loop's exit path closes + * it (close is idempotent via the superclass). + */ + private static final class StubWebSocketClient extends WebSocketClient { + private final AtomicBoolean isClosed = new AtomicBoolean(); + + StubWebSocketClient() { + super(DefaultHttpClientConfiguration.INSTANCE, PlainSocketFactory.INSTANCE); + } + + @Override + public void close() { + isClosed.set(true); + super.close(); + } + + @Override + protected void ioWait(int timeout, int op) { + throw new UnsupportedOperationException("stub: no socket"); + } + + boolean isClosed() { + return isClosed.get(); + } + + @Override + protected void setupIoWait() { + // no-op + } + } + /** ACKs every binary frame with a running sequence so flush/close drain cleanly. */ private static final class AckAllHandler implements TestWebSocketServer.WebSocketServerHandler { private final AtomicLong nextSeq = new AtomicLong(); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/SfFromConfigTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/SfFromConfigTest.java index 0c27d698..1644d561 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/SfFromConfigTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/SfFromConfigTest.java @@ -26,6 +26,7 @@ import io.questdb.client.Sender; import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; import io.questdb.client.std.Files; import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; import io.questdb.client.test.tools.TestUtils; @@ -165,6 +166,46 @@ public void testSfMaxTotalBytesAcceptsLargeValue() throws Exception { }); } + @Test + public void testPeriodicDurabilityDefaultAndExplicitInterval() throws Exception { + TestUtils.assertMemoryLeak(() -> { + AckHandler handler = new AckHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + String base = "ws::addr=localhost:" + server.getPort() + + ";sf_dir=" + sfDir + ";sf_durability=periodic;"; + try (Sender sender = Sender.fromConfig(base)) { + QwpWebSocketSender ws = (QwpWebSocketSender) sender; + Assert.assertEquals(5_000_000_000L, + ws.getCursorEngineForTesting().getSyncIntervalNanosForTesting()); + sender.table("foo").longColumn("v", 1L).atNow(); + sender.flush(); + } + + try (Sender sender = Sender.fromConfig(base + + "sender_id=explicit;sf_sync_interval_millis=123;")) { + QwpWebSocketSender ws = (QwpWebSocketSender) sender; + Assert.assertEquals(123_000_000L, + ws.getCursorEngineForTesting().getSyncIntervalNanosForTesting()); + } + } + }); + } + + @Test + public void testPeriodicDurabilityRequiresDirectory() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (Sender ignored = Sender.fromConfig( + "ws::addr=localhost:1;sf_durability=periodic;")) { + Assert.fail("expected periodic mode without sf_dir to fail"); + } catch (LineSenderException expected) { + Assert.assertTrue(expected.getMessage(), expected.getMessage().contains("requires sf_dir")); + } + }); + } + @Test public void testSfDurabilityAppendNotYetSupported() throws Exception { TestUtils.assertMemoryLeak(() -> { @@ -221,6 +262,20 @@ public void testSfDurabilityOnTcpRejected() throws Exception { }); } + @Test + public void testSfSyncIntervalRequiresPeriodicMode() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String config = "ws::addr=localhost:1;sf_dir=" + sfDir + + ";sf_sync_interval_millis=5000;"; + try (Sender ignored = Sender.fromConfig(config)) { + Assert.fail("expected interval without periodic mode to fail"); + } catch (LineSenderException expected) { + Assert.assertTrue(expected.getMessage(), + expected.getMessage().contains("requires sf_durability=periodic")); + } + }); + } + @Test public void testSfMaxBytesAcceptsSizeSuffixes() throws Exception { TestUtils.assertMemoryLeak(() -> { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/AckWatermarkTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/AckWatermarkTest.java index 001f62ce..848305b2 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/AckWatermarkTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/AckWatermarkTest.java @@ -26,6 +26,9 @@ import io.questdb.client.cutlass.qwp.client.sf.cursor.AckWatermark; import io.questdb.client.std.Files; +import io.questdb.client.std.FilesFacade; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; import io.questdb.client.test.tools.TestUtils; import org.junit.After; import org.junit.Before; @@ -36,6 +39,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; public class AckWatermarkTest { @@ -76,6 +80,61 @@ public void testCrossSessionPersistence() throws Exception { }); } + @Test + public void testFacadeMmapFaultFailsOpenAndClosesFd() throws Exception { + // The watermark mapping must be reachable from an injected facade so + // tests can fault-inject mmap. On a rejected mapping, open() must + // fail cleanly and release the fd through the same facade. + TestUtils.assertMemoryLeak(() -> { + MappingFilesFacade ff = new MappingFilesFacade(true); + assertNull("open must fail when the injected facade rejects mmap", + AckWatermark.open(ff, slotDir)); + assertEquals("facade must receive the watermark mmap call", 1, ff.mmapCalls); + assertEquals("failed open must close the watermark fd via the facade", + 1, ff.watermarkFdCloseCalls); + assertEquals("a rejected mapping must not be munmapped", 0, ff.munmapCalls); + }); + } + + @Test + public void testFallsBackFromTornPlausibleHighValue() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (AckWatermark watermark = AckWatermark.open(slotDir)) { + assertNotNull(watermark); + watermark.write(255L); + watermark.write(256L); + watermark.sync(); + } + + // Model a torn little-endian 255 -> 256 transition: the high byte + // reached storage while the low byte retained 0xff, yielding the + // false FSN 511. A recovered ring with publishedFsn=599 cannot + // reject that value using its segment ceiling. + long publishedFsn = 599L; + long corruptFsn = 511L; + assertTrue(corruptFsn <= publishedFsn); + String path = slotDir + "/" + AckWatermark.FILE_NAME; + int fd = Files.openRW(path); + assertTrue(fd >= 0); + long corruptByte = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT); + try { + Unsafe.getUnsafe().putByte(corruptByte, (byte) 0xff); + long fsnOffset = AckWatermark.FILE_SIZE == 16 ? 8L : 16L; + assertEquals(1L, Files.write(fd, corruptByte, 1, fsnOffset)); + assertEquals(0, Files.fsync(fd)); + } finally { + Files.close(fd); + Unsafe.free(corruptByte, 1, MemoryTag.NATIVE_DEFAULT); + } + + try (AckWatermark watermark = AckWatermark.open(slotDir)) { + assertNotNull(watermark); + assertEquals("torn latest record must fall back to the older watermark", + 255L, watermark.read()); + } + }); + } + @Test public void testFreshFileReadsAsInvalid() throws Exception { // open() creates the file zero-filled, so magic is 0 and read() @@ -102,6 +161,53 @@ public void testNegativeFsnRoundTrips() throws Exception { }); } + @Test + public void testOpenAndCloseRouteMappingThroughFacade() throws Exception { + // The lifetime mapping and its release must go through the injected + // FilesFacade, matching MmapSegment, so test facades can observe them. + TestUtils.assertMemoryLeak(() -> { + MappingFilesFacade ff = new MappingFilesFacade(false); + try (AckWatermark w = AckWatermark.open(ff, slotDir)) { + assertNotNull(w); + assertEquals("open must mmap through the injected facade", 1, ff.mmapCalls); + w.write(7L); + assertEquals(7L, w.read()); + } + assertEquals("close must munmap through the injected facade", 1, ff.munmapCalls); + assertEquals("close must release the watermark fd via the facade", + 1, ff.watermarkFdCloseCalls); + }); + } + + @Test + public void testPhysicalReleaseIsIdempotentAcrossTestingSeamAndClose() throws Exception { + TestUtils.assertMemoryLeak(() -> { + AckWatermark watermark = AckWatermark.open(slotDir); + assertNotNull(watermark); + try { + assertTrue("first test release must relinquish physical storage", + watermark.releaseStorageButKeepWritableForTesting()); + assertFalse("repeated test release must not touch physical storage again", + watermark.releaseStorageButKeepWritableForTesting()); + watermark.close(); + assertFalse("close after test release must keep physical cleanup idempotent", + watermark.releaseStorageButKeepWritableForTesting()); + } finally { + watermark.close(); + } + + AckWatermark normallyClosed = AckWatermark.open(slotDir); + assertNotNull(normallyClosed); + try { + normallyClosed.close(); + assertFalse("ordinary close must record physical relinquishment", + normallyClosed.releaseStorageButKeepWritableForTesting()); + } finally { + normallyClosed.close(); + } + }); + } + @Test public void testRemoveOrphanDeletesFile() throws Exception { TestUtils.assertMemoryLeak(() -> { @@ -133,6 +239,38 @@ public void testRepeatedWriteUpdatesValue() throws Exception { }); } + @Test + public void testSingleSectorTearLeavesPriorRecordRecoverable() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (AckWatermark watermark = AckWatermark.open(slotDir)) { + assertNotNull(watermark); + watermark.write(100L); + watermark.write(200L); + watermark.sync(); + } + + String path = slotDir + "/" + AckWatermark.FILE_NAME; + int fd = Files.openRW(path); + assertTrue(fd >= 0); + int tornBytes = (int) Math.min(512L, Files.length(path)); + long tornSector = Unsafe.malloc(tornBytes, MemoryTag.NATIVE_DEFAULT); + try { + Unsafe.getUnsafe().setMemory(tornSector, tornBytes, (byte) 0xA5); + assertEquals(tornBytes, Files.write(fd, tornSector, tornBytes, 0)); + assertEquals(0, Files.fsync(fd)); + } finally { + Files.close(fd); + Unsafe.free(tornSector, tornBytes, MemoryTag.NATIVE_DEFAULT); + } + + try (AckWatermark watermark = AckWatermark.open(slotDir)) { + assertNotNull(watermark); + assertEquals("one aligned 512-byte tear must leave the prior record valid", + 100L, watermark.read()); + } + }); + } + @Test public void testStaleFileWithWrongSizeIsResetOnOpen() throws Exception { // A leftover file with an unexpected size (corruption, partial @@ -171,4 +309,68 @@ public void testWriteReadInSameSession() throws Exception { } }); } + + /** + * Delegating facade that counts mmap/munmap traffic and watermark-fd + * closes, optionally rejecting the mapping to exercise the open() + * failure path. + */ + private static final class MappingFilesFacade implements FilesFacade { + private final boolean failMmap; + private int mmapCalls; + private int munmapCalls; + private int watermarkFd = -1; + private int watermarkFdCloseCalls; + + private MappingFilesFacade(boolean failMmap) { + this.failMmap = failMmap; + } + + @Override public boolean allocate(int fd, long size) { return INSTANCE.allocate(fd, size); } + @Override public long allocNativePath(String path) { return INSTANCE.allocNativePath(path); } + @Override public int close(int fd) { + if (fd >= 0 && fd == watermarkFd) watermarkFdCloseCalls++; + return INSTANCE.close(fd); + } + @Override public boolean exists(String path) { return INSTANCE.exists(path); } + @Override public void findClose(long findPtr) { INSTANCE.findClose(findPtr); } + @Override public long findFirst(String dir) { return INSTANCE.findFirst(dir); } + @Override public long findName(long findPtr) { return INSTANCE.findName(findPtr); } + @Override public int findNext(long findPtr) { return INSTANCE.findNext(findPtr); } + @Override public int findType(long findPtr) { return INSTANCE.findType(findPtr); } + @Override public void freeNativePath(long pathPtr) { INSTANCE.freeNativePath(pathPtr); } + @Override public int fsync(int fd) { return INSTANCE.fsync(fd); } + @Override public long length(int fd) { return INSTANCE.length(fd); } + @Override public long length(String path) { return INSTANCE.length(path); } + @Override public long length(long pathPtr) { return INSTANCE.length(pathPtr); } + @Override public int lock(int fd) { return INSTANCE.lock(fd); } + @Override public int mkdir(String path, int mode) { return INSTANCE.mkdir(path, mode); } + @Override public long mmap(int fd, long len, long offset, int flags, int memoryTag) { + mmapCalls++; + if (failMmap) return Files.FAILED_MMAP_ADDRESS; + return INSTANCE.mmap(fd, len, offset, flags, memoryTag); + } + @Override public void munmap(long address, long len, int memoryTag) { + munmapCalls++; + INSTANCE.munmap(address, len, memoryTag); + } + @Override public int openCleanRW(String path) { + int fd = INSTANCE.openCleanRW(path); + if (path.endsWith(AckWatermark.FILE_NAME)) watermarkFd = fd; + return fd; + } + @Override public int openCleanRW(long pathPtr) { return INSTANCE.openCleanRW(pathPtr); } + @Override public int openRW(String path) { + int fd = INSTANCE.openRW(path); + if (path.endsWith(AckWatermark.FILE_NAME)) watermarkFd = fd; + return fd; + } + @Override public int openRW(long pathPtr) { return INSTANCE.openRW(pathPtr); } + @Override public long read(int fd, long addr, long len, long offset) { return INSTANCE.read(fd, addr, len, offset); } + @Override public boolean remove(String path) { return INSTANCE.remove(path); } + @Override public boolean remove(long pathPtr) { return INSTANCE.remove(pathPtr); } + @Override public int rename(String oldPath, String newPath) { return INSTANCE.rename(oldPath, newPath); } + @Override public boolean truncate(int fd, long size) { return INSTANCE.truncate(fd, size); } + @Override public long write(int fd, long addr, long len, long offset) { return INSTANCE.write(fd, addr, len, offset); } + } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/BackgroundDrainerDurableAckRetryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/BackgroundDrainerDurableAckRetryTest.java index 0a6b69f8..ad9f5930 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/BackgroundDrainerDurableAckRetryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/BackgroundDrainerDurableAckRetryTest.java @@ -511,8 +511,8 @@ public void testJvmErrorEscapesConnectRetryLoop() throws Exception { // .lock forever with no .failed sentinel and only a throttled WARN as // a trace. A JVM/programming failure is not a transport outage: // retrying cannot clear it, so it must escape the loop on the FIRST - // sweep. run()'s outer catch then quarantines the slot (markFailed + - // FAILED) and its finally releases the lock -- quarantine-and-exit. + // sweep. run() records FAILED without quarantining recoverable data, + // releases the lock in finally, and propagates the Error. CountingListener listener = new CountingListener(); ScriptedFactory factory = ScriptedFactory.alwaysFailing( () -> new LinkageError("simulated JVM failure")); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/BackgroundDrainerSetupFailureTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/BackgroundDrainerSetupFailureTest.java new file mode 100644 index 00000000..a8bd96c9 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/BackgroundDrainerSetupFailureTest.java @@ -0,0 +1,424 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.qwp.client.sf.cursor.AckWatermark; +import io.questdb.client.cutlass.qwp.client.sf.cursor.BackgroundDrainer; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; +import io.questdb.client.cutlass.qwp.client.sf.cursor.OrphanScanner; +import io.questdb.client.std.Files; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.nio.file.Paths; +import java.util.concurrent.atomic.AtomicInteger; + +public class BackgroundDrainerSetupFailureTest { + + private static final long SEGMENT_BYTES = 1L << 20; + private String slotPath; + + @Before + public void setUp() { + slotPath = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-drainer-setup-failure-" + System.nanoTime()).toString(); + Assert.assertEquals(0, Files.mkdir(slotPath, Files.DIR_MODE_DEFAULT)); + } + + @After + public void tearDown() { + removeRecursive(slotPath); + } + + @Test + public void testConnectErrorPropagatesWithoutQuarantine() throws Exception { + TestUtils.assertMemoryLeak(() -> { + seedUnackedFrame(); + LinkageError injected = new LinkageError("injected drainer connect error"); + AtomicInteger connectAttempts = new AtomicInteger(); + BackgroundDrainer drainer = new BackgroundDrainer( + slotPath, + SEGMENT_BYTES, + Long.MAX_VALUE, + () -> { + connectAttempts.incrementAndGet(); + throw injected; + }, + 5_000L, + 1L, + 10L, + true, + 200L); + + LinkageError thrown = null; + try { + drainer.run(); + } catch (LinkageError e) { + thrown = e; + } + + Assert.assertSame("post-publication Error must remain visible to the caller", + injected, thrown); + Assert.assertEquals("connect error must occur on the first attempt", + 1, connectAttempts.get()); + Assert.assertEquals(BackgroundDrainer.DrainOutcome.FAILED, drainer.outcome()); + TestUtils.assertContains(drainer.getLastErrorMessage(), + "injected drainer connect error"); + Assert.assertFalse("a post-publication Error must not quarantine recoverable data", + Files.exists(slotPath + "/" + OrphanScanner.FAILED_SENTINEL_NAME)); + Assert.assertTrue("unacknowledged segment must remain scanner-eligible", + OrphanScanner.isCandidateOrphan(slotPath)); + try (CursorSendEngine ignored = new CursorSendEngine(slotPath, SEGMENT_BYTES)) { + Assert.assertTrue("drainer teardown must release the slot lock", + OrphanScanner.isCandidateOrphan(slotPath)); + } + }); + } + + @Test + public void testConstructionErrorPropagatesWithoutQuarantine() throws Exception { + TestUtils.assertMemoryLeak(() -> { + seedUnackedFrame(); + OutOfMemoryError injected = new OutOfMemoryError("injected drainer construction error"); + CursorSendEngine.setBeforeDeferredCloseCreationHook(() -> { + throw injected; + }); + AtomicInteger connectAttempts = new AtomicInteger(); + BackgroundDrainer drainer = new BackgroundDrainer( + slotPath, + SEGMENT_BYTES, + Long.MAX_VALUE, + () -> { + connectAttempts.incrementAndGet(); + throw new AssertionError("construction error must happen before connect"); + }, + 5_000L, + 1L, + 10L, + true, + 200L); + + OutOfMemoryError thrown = null; + try { + drainer.run(); + } catch (OutOfMemoryError e) { + thrown = e; + } finally { + CursorSendEngine.setBeforeDeferredCloseCreationHook(null); + } + + Assert.assertSame("construction Error must remain visible to the caller", + injected, thrown); + Assert.assertEquals("construction error must precede connect", + 0, connectAttempts.get()); + Assert.assertEquals(BackgroundDrainer.DrainOutcome.FAILED, drainer.outcome()); + TestUtils.assertContains(drainer.getLastErrorMessage(), + "injected drainer construction error"); + Assert.assertFalse("a VM Error must not quarantine recoverable data", + Files.exists(slotPath + "/" + OrphanScanner.FAILED_SENTINEL_NAME)); + Assert.assertTrue("unacknowledged segment must remain scanner-eligible", + OrphanScanner.isCandidateOrphan(slotPath)); + try (CursorSendEngine ignored = new CursorSendEngine(slotPath, SEGMENT_BYTES)) { + Assert.assertTrue("constructor teardown must release the slot lock", + OrphanScanner.isCandidateOrphan(slotPath)); + } + }); + } + + @Test + public void testCorruptRecoveredChainIsQuarantined() throws Exception { + TestUtils.assertMemoryLeak(() -> { + seedUnackedFrame(); + + String segmentPath = slotPath + "/sf-initial.sfa"; + long corruptByte = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT); + int fd = Files.openRW(segmentPath); + Assert.assertTrue("could not open segment for corruption", fd >= 0); + try { + Unsafe.getUnsafe().putByte(corruptByte, (byte) 0); + Assert.assertEquals(1L, Files.write(fd, corruptByte, 1, 0L)); + } finally { + Files.close(fd); + Unsafe.free(corruptByte, 1, MemoryTag.NATIVE_DEFAULT); + } + + AtomicInteger connectAttempts = new AtomicInteger(); + BackgroundDrainer drainer = new BackgroundDrainer( + slotPath, + SEGMENT_BYTES, + Long.MAX_VALUE, + () -> { + connectAttempts.incrementAndGet(); + throw new AssertionError("recovery failure must happen before connect"); + }, + 5_000L, + 1L, + 10L, + true, + 200L); + + drainer.run(); + + Assert.assertEquals("corrupt recovery must precede connect", + 0, connectAttempts.get()); + Assert.assertEquals(BackgroundDrainer.DrainOutcome.FAILED, drainer.outcome()); + Assert.assertTrue("a corrupt durable chain requires operator quarantine", + Files.exists(slotPath + "/" + OrphanScanner.FAILED_SENTINEL_NAME)); + Assert.assertTrue("corrupt segment evidence must remain on disk", + Files.exists(segmentPath)); + }); + } + + @Test + public void testSealedResidueFirstSightHealsAndDoesNotQuarantine() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // A sealed member whose frame accounting is complete but whose + // suffix gap carries legacy pre-sanitization poison. Recovery + // durably zeroes the residue BEFORE failing closed on first + // sight (SegmentRing's sanitize-then-throw), so at the instant + // the drainer sees the throw the chain on disk is already + // healed. The drainer must retry construction over the healed + // chain and reach connect -- not strand the replayable backlog + // behind a .failed sentinel that no orphan scan revisits. + long segSize = MmapSegment.HEADER_SIZE + + 4 * (MmapSegment.FRAME_HEADER_SIZE + 16) + + 12; // sealed suffix gap that can never fit a frame + String p0Path = slotPath + "/p0.sfa"; + String p1Path = slotPath + "/p1.sfa"; + long gapStart; + long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + try { + Unsafe.getUnsafe().setMemory(buf, 16, (byte) 5); + MmapSegment s0 = MmapSegment.create(p0Path, 0, segSize); + for (int i = 0; i < 4; i++) { + s0.tryAppend(buf, 16); + } + gapStart = s0.publishedOffset(); + s0.close(); + MmapSegment s1 = MmapSegment.create(p1Path, 4, segSize); + s1.tryAppend(buf, 16); + s1.close(); + } finally { + Unsafe.free(buf, 16, MemoryTag.NATIVE_DEFAULT); + } + + // Adopt the chain once so the slot carries a manifest, lock and + // watermark like any real producer slot (without a manifest the + // legacy-migration path sanitizes silently and never fails + // closed); close releases the lock. + try (CursorSendEngine seed = new CursorSendEngine(slotPath, segSize)) { + Assert.assertEquals("highest published FSN must cover frames 0..4", + 4L, seed.publishedFsn()); + } + + // Poison the sealed suffix gap the way a pre-sanitization + // client's reseal-after-recovery did. + int fd = Files.openRW(p0Path); + Assert.assertTrue("openRW must succeed", fd >= 0); + long junk = Unsafe.malloc(12, MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < 3; i++) { + Unsafe.getUnsafe().putInt(junk + i * 4L, 0xCAFEBABE); + } + Assert.assertEquals(12L, Files.write(fd, junk, 12, gapStart)); + Files.fsync(fd); + } finally { + Unsafe.free(junk, 12, MemoryTag.NATIVE_DEFAULT); + Files.close(fd); + } + + LinkageError injected = new LinkageError("injected drainer connect error"); + AtomicInteger connectAttempts = new AtomicInteger(); + BackgroundDrainer drainer = new BackgroundDrainer( + slotPath, + segSize, + Long.MAX_VALUE, + () -> { + connectAttempts.incrementAndGet(); + throw injected; + }, + 5_000L, + 1L, + 10L, + true, + 200L); + + LinkageError thrown = null; + try { + drainer.run(); + } catch (LinkageError e) { + thrown = e; + } + + // The heal precedes the first-sight throw: frames intact, + // residue durably zeroed. This holds with or without the + // drainer fix -- it is exactly what makes quarantine wrong. + try (MmapSegment sealed = MmapSegment.openExisting(p0Path)) { + Assert.assertEquals("frames must be untouched", 4L, sealed.frameCount()); + Assert.assertEquals("proven-dead residue must be zeroed on disk", + 0L, sealed.tornTailBytes()); + } + + Assert.assertSame("drainer must reach connect over the healed chain", + injected, thrown); + Assert.assertEquals("exactly one connect attempt after the healed retry", + 1, connectAttempts.get()); + TestUtils.assertContains(drainer.getLastErrorMessage(), + "injected drainer connect error"); + Assert.assertEquals(BackgroundDrainer.DrainOutcome.FAILED, drainer.outcome()); + Assert.assertFalse("a just-healed slot must not be quarantined", + Files.exists(slotPath + "/" + OrphanScanner.FAILED_SENTINEL_NAME)); + Assert.assertTrue("replayable backlog must remain scanner-eligible", + OrphanScanner.isCandidateOrphan(slotPath)); + try (CursorSendEngine engine = new CursorSendEngine(slotPath, segSize)) { + Assert.assertEquals("backlog must remain fully replayable", + 4L, engine.publishedFsn()); + Assert.assertTrue("drainer teardown must release the slot lock", + OrphanScanner.isCandidateOrphan(slotPath)); + } + }); + } + + @Test + public void testLockOpenFailureDoesNotQuarantineRecoverableData() throws Exception { + TestUtils.assertMemoryLeak(() -> { + seedUnackedFrame(); + + String lockPath = slotPath + "/.lock"; + Assert.assertTrue("seeded lock file must exist", Files.exists(lockPath)); + Assert.assertTrue("could not remove seeded lock file", Files.remove(lockPath)); + Assert.assertEquals("could not plant deterministic lock-open failure", + 0, Files.mkdir(lockPath, Files.DIR_MODE_DEFAULT)); + + AtomicInteger connectAttempts = new AtomicInteger(); + BackgroundDrainer drainer = new BackgroundDrainer( + slotPath, + SEGMENT_BYTES, + Long.MAX_VALUE, + () -> { + connectAttempts.incrementAndGet(); + throw new AssertionError("lock setup failure must happen before connect"); + }, + 5_000L, + 1L, + 10L, + true, + 200L); + + drainer.run(); + + Assert.assertEquals("lock failure must precede connect", + 0, connectAttempts.get()); + Assert.assertEquals(BackgroundDrainer.DrainOutcome.FAILED, drainer.outcome()); + TestUtils.assertContains(drainer.getLastErrorMessage(), + "could not open slot lock file"); + Assert.assertFalse("an operational lock-open failure must remain retryable", + Files.exists(slotPath + "/" + OrphanScanner.FAILED_SENTINEL_NAME)); + Assert.assertTrue("unacknowledged segment must remain scanner-eligible", + OrphanScanner.isCandidateOrphan(slotPath)); + }); + } + + @Test + public void testWatermarkOpenFailureDoesNotQuarantineRecoverableData() throws Exception { + TestUtils.assertMemoryLeak(() -> { + seedUnackedFrame(); + + String watermarkPath = slotPath + "/" + AckWatermark.FILE_NAME; + Assert.assertTrue("seeded watermark must exist", Files.exists(watermarkPath)); + Assert.assertTrue("could not remove seeded watermark", Files.remove(watermarkPath)); + Assert.assertEquals("could not plant deterministic watermark-open failure", + 0, Files.mkdir(watermarkPath, Files.DIR_MODE_DEFAULT)); + + AtomicInteger connectAttempts = new AtomicInteger(); + BackgroundDrainer drainer = new BackgroundDrainer( + slotPath, + SEGMENT_BYTES, + Long.MAX_VALUE, + () -> { + connectAttempts.incrementAndGet(); + throw new AssertionError("setup failure must happen before connect"); + }, + 5_000L, + 1L, + 10L, + true, + 200L); + + drainer.run(); + + Assert.assertEquals("watermark failure must precede connect", + 0, connectAttempts.get()); + Assert.assertEquals(BackgroundDrainer.DrainOutcome.FAILED, drainer.outcome()); + TestUtils.assertContains(drainer.getLastErrorMessage(), + "could not open required ack watermark"); + Assert.assertFalse("an operational watermark-open failure must remain retryable", + Files.exists(slotPath + "/" + OrphanScanner.FAILED_SENTINEL_NAME)); + Assert.assertTrue("unacknowledged segment must remain on disk", + OrphanScanner.isCandidateOrphan(slotPath)); + }); + } + + private static void removeRecursive(String path) { + if (path == null || !Files.exists(path)) { + return; + } + long find = Files.findFirst(path); + if (find > 0) { + try { + int rc = 1; + while (rc > 0) { + String name = Files.utf8ToString(Files.findName(find)); + if (name != null && !".".equals(name) && !"..".equals(name)) { + String child = path + "/" + name; + if (!Files.remove(child)) { + removeRecursive(child); + } + } + rc = Files.findNext(find); + } + } finally { + Files.findClose(find); + } + } + Files.remove(path); + } + + private void seedUnackedFrame() { + long buffer = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + try (CursorSendEngine engine = new CursorSendEngine(slotPath, SEGMENT_BYTES)) { + Unsafe.getUnsafe().setMemory(buffer, 16, (byte) 1); + Assert.assertEquals(0L, engine.appendBlocking(buffer, 16)); + } finally { + Unsafe.free(buffer, 16, MemoryTag.NATIVE_DEFAULT); + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineClosePartialEnumerationTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineClosePartialEnumerationTest.java new file mode 100644 index 00000000..9aefc2c1 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineClosePartialEnumerationTest.java @@ -0,0 +1,163 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; +import io.questdb.client.std.Files; +import io.questdb.client.std.FilesFacade; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Proxy; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Close-time unlink under a TORN directory enumeration ({@code findNext} + * fails after the listing already produced entries). A partial listing must + * not drive any unlink: removing only the files the walk happened to see + * could delete the segment holding the highest frame while a lower one + * survives, leaving residual state the retained ack watermark can no longer + * vouch for. The contract is all-or-nothing: abort the cleanup, keep every + * {@code .sfa} file and the watermark, and let the next recovery (or a + * successor's fully-drained close) retry. + *

      + * The sibling {@link CursorSendEngineCloseUnlinkFailureTest} injects a + * failing UNLINK (permission trick, root-skipped); nothing exercised the + * enumeration-abort branch itself. Same determinism trick as the sibling: + * the shared manager is never started, so no worker touches the slot and no + * concurrent enumeration can trip the armed fault. + */ +public class CursorSendEngineClosePartialEnumerationTest { + + private String tmpDir; + + @Before + public void setUp() { + tmpDir = TestUtils.createTmpDir("qdb-engine-close-enum-fault-"); + } + + @After + public void tearDown() { + TestUtils.removeTmpDir(tmpDir); + } + + @Test(timeout = 20_000L) + public void testTornCloseTimeEnumerationUnlinksNothingAndSuccessorRetries() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final AtomicBoolean failFindNext = new AtomicBoolean(); + final AtomicInteger sfaRemoveAttempts = new AtomicInteger(); + FilesFacade faultFacade = (FilesFacade) Proxy.newProxyInstance( + FilesFacade.class.getClassLoader(), + new Class[]{FilesFacade.class}, + (proxy, method, args) -> { + if (failFindNext.get() && "findNext".equals(method.getName())) { + return -1; + } + if ("remove".equals(method.getName()) + && args != null && args.length == 1 + && args[0] instanceof String + && ((String) args[0]).endsWith(".sfa")) { + sfaRemoveAttempts.incrementAndGet(); + } + try { + return method.invoke(FilesFacade.INSTANCE, args); + } catch (InvocationTargetException e) { + throw e.getCause(); + } + }); + + long segSize = 4096L; + String slot = tmpDir + "/slot"; + Assert.assertEquals(0, Files.mkdir(slot, Files.DIR_MODE_DEFAULT)); + SegmentManager manager = new SegmentManager( + segSize, + SegmentManager.DEFAULT_POLL_NANOS, + segSize * 4L, + faultFacade, + System::nanoTime); + long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + try { + CursorSendEngine engine = new CursorSendEngine(slot, segSize, manager); + boolean engineClosed = false; + try { + Assert.assertEquals(0L, engine.appendBlocking(buf, 16)); + Assert.assertTrue(engine.acknowledge(0L)); + + // Fully drained close, but the directory listing tears + // mid-walk: the cleanup must abort BEFORE the first + // unlink. + failFindNext.set(true); + engine.close(); + engineClosed = true; + Assert.assertTrue("close must complete despite the aborted cleanup", + engine.isCloseCompleted()); + Assert.assertEquals( + "a torn enumeration must drive ZERO segment unlinks -- " + + "removing only the files the walk happened to see can " + + "strand residual state the watermark cannot vouch for", + 0, sfaRemoveAttempts.get()); + Assert.assertTrue("the segment file must survive the aborted cleanup", + Files.exists(slot + "/sf-initial.sfa")); + } finally { + if (!engineClosed) { + engine.close(); + } + } + + // Heal the directory walk; a successor adopts the slot, + // recovers the residual (fully acknowledged) state, and its + // own fully-drained close retries the cleanup successfully. + failFindNext.set(false); + CursorSendEngine successor = new CursorSendEngine(slot, segSize, manager); + boolean successorClosed = false; + try { + Assert.assertTrue("successor must recover the residual slot state", + successor.wasRecoveredFromDisk()); + successor.close(); + successorClosed = true; + Assert.assertTrue(successor.isCloseCompleted()); + Assert.assertFalse( + "successor's fully-drained close must retry and complete the unlink", + Files.exists(slot + "/sf-initial.sfa")); + } finally { + if (!successorClosed) { + successor.close(); + } + } + } finally { + Unsafe.free(buf, 16, MemoryTag.NATIVE_DEFAULT); + manager.close(); + } + }); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineCloseUnlinkFailureTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineCloseUnlinkFailureTest.java new file mode 100644 index 00000000..a9df2b67 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineCloseUnlinkFailureTest.java @@ -0,0 +1,209 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.qwp.client.sf.cursor.AckWatermark; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; +import io.questdb.client.std.Files; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Assume; +import org.junit.Before; +import org.junit.Test; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.PosixFilePermissions; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +/** + * Regression for the close-time segment cleanup on a fully-drained slot. + *

      + * {@code CursorSendEngine.finishClose} unlinks the acknowledged {@code .sfa} + * files and then removes the ack watermark. When the unlink fails (transient + * I/O error, permission problem), the residual segment files hold rows the + * server already acknowledged. If the watermark does not cover them, a + * successor engine on the same slot seeds recovery from {@code lowestBase - 1} + * and replays every acknowledged row — duplicates on a non-DEDUP table. + *

      + * The test injects the unlink failure by dropping write permission on the + * slot directory (POSIX: unlink requires a writable parent directory), so it + * is skipped on Windows and when permissions are not enforced (root). + * The shared {@link SegmentManager} is deliberately never started: no worker + * thread exists, so no manager tick can persist the watermark behind the + * test's back, and the close-path quiescence barrier is trivially satisfied — + * fully deterministic, no timing coordination needed. + */ +public class CursorSendEngineCloseUnlinkFailureTest { + + private String tmpDir; + + @Before + public void setUp() { + tmpDir = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-engine-close-unlink-fault-" + System.nanoTime()).toString(); + Assert.assertEquals(0, Files.mkdir(tmpDir, Files.DIR_MODE_DEFAULT)); + } + + @After + public void tearDown() { + if (tmpDir != null) { + removeRecursive(tmpDir); + } + } + + @Test(timeout = 20_000L) + public void testFailedCloseTimeUnlinkMustNotExposeAckedFramesToSuccessor() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long segmentSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 32L; + String slot = tmpDir + "/slot"; + Path slotPath = Paths.get(slot); + long payload = Unsafe.malloc(32, MemoryTag.NATIVE_DEFAULT); + SegmentManager manager = new SegmentManager(segmentSize, TimeUnit.SECONDS.toNanos(60)); + CursorSendEngine pred = null; + CursorSendEngine succ = null; + boolean slotDirReadOnly = false; + try { + fill(payload, 32, (byte) 0x33); + pred = new CursorSendEngine(slot, segmentSize, manager); + Assert.assertEquals(0L, pred.appendBlocking(payload, 32)); + Assert.assertEquals(0L, pred.publishedFsn()); + // The server durably acknowledged FSN 0 in this session. + Assert.assertTrue(pred.acknowledge(0L)); + Assert.assertEquals(0L, pred.ackedFsn()); + + // Inject a close-time unlink failure: drop write permission on + // the slot dir. Prove the injection works with a probe file -- + // root (and some filesystems) ignore directory permissions. + String probePath = slot + "/probe"; + Assert.assertTrue(java.nio.file.Files.exists( + java.nio.file.Files.createFile(Paths.get(probePath)))); + try { + setPermissions(slotPath, "r-xr-xr-x"); + } catch (UnsupportedOperationException e) { + Assume.assumeNoException("POSIX permissions unavailable on this platform", e); + } + slotDirReadOnly = true; + boolean probeRemoved = Files.remove(probePath); + if (probeRemoved) { + setPermissions(slotPath, "rwxr-xr-x"); + slotDirReadOnly = false; + } + Assume.assumeFalse("directory permissions not enforced (running as root?)", + probeRemoved); + + // Fully-drained close: tries to unlink the acknowledged + // segment files and fails. + pred.close(); + Assert.assertTrue("flock release needs no dir write; close must complete", + pred.isCloseCompleted()); + pred = null; + Assert.assertTrue("injected unlink failure must leave the acknowledged segment", + Files.exists(slot + "/sf-initial.sfa")); + + // The transient failure clears before the successor arrives. + setPermissions(slotPath, "rwxr-xr-x"); + slotDirReadOnly = false; + Files.remove(probePath); + + succ = new CursorSendEngine(slot, segmentSize, manager); + Assert.assertTrue(succ.wasRecoveredFromDisk()); + Assert.assertEquals(0L, succ.publishedFsn()); + // THE regression: FSN 0 was acknowledged by the server during + // the predecessor's session. The successor must not see it as + // replayable, or a non-DEDUP table receives duplicate rows. + Assert.assertTrue("successor exposes already-acknowledged frames for replay " + + "[ackedFsn=" + succ.ackedFsn() + + ", publishedFsn=" + succ.publishedFsn() + "]", + succ.ackedFsn() >= succ.publishedFsn()); + + // The successor's own fully-drained close retries the cleanup + // now that the failure has cleared: segments and watermark gone. + succ.close(); + Assert.assertTrue(succ.isCloseCompleted()); + succ = null; + Assert.assertFalse("successor close did not retry the segment unlink", + Files.exists(slot + "/sf-initial.sfa")); + Assert.assertFalse("watermark must be removed once no segment file remains", + Files.exists(slot + "/" + AckWatermark.FILE_NAME)); + } finally { + if (slotDirReadOnly) { + try { + setPermissions(slotPath, "rwxr-xr-x"); + } catch (Throwable ignored) { + } + } + if (pred != null) { + pred.close(); + } + if (succ != null) { + succ.close(); + } + manager.close(); + Unsafe.free(payload, 32, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + private static void fill(long address, int len, byte value) { + for (int i = 0; i < len; i++) { + Unsafe.getUnsafe().putByte(address + i, value); + } + } + + private static void removeRecursive(String dir) { + long find = Files.findFirst(dir); + if (find > 0) { + try { + int rc = 1; + while (rc > 0) { + String name = Files.utf8ToString(Files.findName(find)); + if (name != null && !".".equals(name) && !"..".equals(name)) { + String child = dir + "/" + name; + if (!Files.remove(child)) { + removeRecursive(child); + } + } + rc = Files.findNext(find); + } + } finally { + Files.findClose(find); + } + } + Files.remove(dir); + } + + private static void setPermissions(Path path, String posix) throws Exception { + Set perms = PosixFilePermissions.fromString(posix); + java.nio.file.Files.setPosixFilePermissions(path, perms); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineCloseUnlinkStopOnFirstFailureTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineCloseUnlinkStopOnFirstFailureTest.java new file mode 100644 index 00000000..8fe040d7 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineCloseUnlinkStopOnFirstFailureTest.java @@ -0,0 +1,193 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; +import io.questdb.client.std.Files; +import io.questdb.client.std.FilesFacade; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Proxy; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Close-time unlink STOP-ON-FIRST-FAILURE on a legacy (manifest-less) slot. + * Removal runs in ascending generation order and must stop at the first + * failed unlink so the residue is always a contiguous top slice that passes + * FSN-contiguity at the next recovery. Continuing past a failed + * low-generation remove deletes higher generations and can leave + * non-contiguous residue -- a startup brick on a slot that lost nothing. + *

      + * The siblings cover the OTHER close-cleanup contracts: + * {@link CursorSendEngineClosePartialEnumerationTest} proves a torn + * enumeration drives zero unlinks, and the unlink-failure sibling fails ALL + * removals (permission trick), which cannot discriminate stop-vs-continue + * (every file survives either way). A continue-past-first-failure mutant in + * {@code unlinkAllSegmentFiles} previously survived the whole suite; this + * test kills it via the higher-generation-survival assertions. Same + * determinism trick as the siblings: the shared manager is never started, so + * no worker touches the slot and no concurrent cleanup can trip the armed + * fault. + */ +public class CursorSendEngineCloseUnlinkStopOnFirstFailureTest { + + private static final long SEGMENT_SIZE = 4096L; + + private String tmpDir; + + @Before + public void setUp() { + tmpDir = TestUtils.createTmpDir("qdb-engine-close-unlink-stop-"); + } + + @After + public void tearDown() { + TestUtils.removeTmpDir(tmpDir); + } + + @Test(timeout = 20_000L) + public void testCloseUnlinkStopsAtFirstFailedRemoveOnLegacySlot() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final String slot = tmpDir + "/legacy-slot"; + final String lowPath = slot + "/sf-initial.sfa"; + final String highPath = slot + "/sf-0000000000000002.sfa"; + final AtomicBoolean armLowestRemoveFailure = new AtomicBoolean(); + final AtomicInteger lowRemoveAttempts = new AtomicInteger(); + final AtomicInteger highRemoveAttempts = new AtomicInteger(); + FilesFacade faultFacade = (FilesFacade) Proxy.newProxyInstance( + FilesFacade.class.getClassLoader(), + new Class[]{FilesFacade.class}, + (proxy, method, args) -> { + if ("remove".equals(method.getName()) && args[0] instanceof String) { + String path = (String) args[0]; + if (path.equals(lowPath)) { + lowRemoveAttempts.incrementAndGet(); + if (armLowestRemoveFailure.get()) { + // EBUSY-style transient refusal: no unlink happens. + return false; + } + } else if (path.equals(highPath)) { + highRemoveAttempts.incrementAndGet(); + } + } + try { + return method.invoke(FilesFacade.INSTANCE, args); + } catch (InvocationTargetException e) { + throw e.getCause(); + } + }); + + Assert.assertEquals(0, Files.mkdir(slot, Files.DIR_MODE_DEFAULT)); + // Legacy slot: two FSN-contiguous segments on disk, NO manifest. + long buf = Unsafe.malloc(32, MemoryTag.NATIVE_DEFAULT); + SegmentManager manager = null; + try { + MmapSegment low = MmapSegment.create(lowPath, 0L, SEGMENT_SIZE); + Assert.assertTrue("setup: append must land", low.tryAppend(buf, 32) >= 0); + Assert.assertTrue("setup: append must land", low.tryAppend(buf, 32) >= 0); + low.close(); + MmapSegment high = MmapSegment.create(highPath, 2L, SEGMENT_SIZE); + Assert.assertTrue("setup: append must land", high.tryAppend(buf, 32) >= 0); + high.close(); + + manager = new SegmentManager( + SEGMENT_SIZE, + SegmentManager.DEFAULT_POLL_NANOS, + SEGMENT_SIZE * 4L, + faultFacade, + System::nanoTime); + + CursorSendEngine engine = new CursorSendEngine(slot, SEGMENT_SIZE, manager); + boolean engineClosed = false; + try { + Assert.assertTrue("engine must recover the legacy two-segment chain", + engine.wasRecoveredFromDisk()); + Assert.assertEquals("recovered chain must publish FSNs 0..2", + 2L, engine.publishedFsn()); + Assert.assertTrue(engine.acknowledge(2L)); + + // Fully drained close with the LOWEST-generation removal + // refused once: ascending-order removal must STOP there. + armLowestRemoveFailure.set(true); + engine.close(); + engineClosed = true; + Assert.assertTrue("close must complete despite the aborted cleanup", + engine.isCloseCompleted()); + Assert.assertEquals("cleanup must attempt the lowest generation first", + 1, lowRemoveAttempts.get()); + Assert.assertEquals( + "removal must STOP at the first failed unlink -- continuing would " + + "delete higher generations and could leave non-contiguous " + + "residue that fails FSN-contiguity at the next recovery", + 0, highRemoveAttempts.get()); + Assert.assertTrue("the refused lowest segment must survive", + Files.exists(lowPath)); + Assert.assertTrue("higher-generation segment must survive the stopped cleanup", + Files.exists(highPath)); + } finally { + if (!engineClosed) { + engine.close(); + } + } + + // Heal the fault; a successor adopts the contiguous residue + // and its fully-drained close completes the cleanup. + armLowestRemoveFailure.set(false); + CursorSendEngine successor = new CursorSendEngine(slot, SEGMENT_SIZE, manager); + boolean successorClosed = false; + try { + Assert.assertTrue("successor must recover the contiguous residue", + successor.wasRecoveredFromDisk()); + successor.close(); + successorClosed = true; + Assert.assertTrue(successor.isCloseCompleted()); + Assert.assertFalse("successor's fully-drained close must complete the unlink", + Files.exists(lowPath)); + Assert.assertFalse("successor's fully-drained close must complete the unlink", + Files.exists(highPath)); + } finally { + if (!successorClosed) { + successor.close(); + } + } + } finally { + Unsafe.free(buf, 32, MemoryTag.NATIVE_DEFAULT); + if (manager != null) { + manager.close(); + } + } + }); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineCrashConsistencyTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineCrashConsistencyTest.java new file mode 100644 index 00000000..a7017961 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineCrashConsistencyTest.java @@ -0,0 +1,458 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.qwp.client.sf.cursor.AckWatermark; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; +import io.questdb.client.std.Files; +import io.questdb.client.std.FilesFacade; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.TimeUnit; + +public class CursorSendEngineCrashConsistencyTest { + + @Test + public void testCloseDurabilityOrderAndPostCleanupSyncFailurePropagation() throws Exception { + TestUtils.assertMemoryLeak(() -> { + for (int failAt : new int[]{-1, 4}) { + String root = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-close-crash-" + failAt + "-" + System.nanoTime()).toString(); + String slot = root + "/slot"; + SegmentManager manager = null; + CursorSendEngine engine = null; + long payload = 0; + Throwable failure = null; + try { + Assert.assertEquals(0, Files.mkdir(root, Files.DIR_MODE_DEFAULT)); + CrashImageFilesFacade ff = new CrashImageFilesFacade(slot, failAt); + long segmentSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 32L; + manager = new SegmentManager(segmentSize, TimeUnit.SECONDS.toNanos(60), + SegmentManager.UNLIMITED_TOTAL_BYTES, ff); + payload = Unsafe.malloc(32, MemoryTag.NATIVE_DEFAULT); + engine = new CursorSendEngine(slot, segmentSize, manager); + Unsafe.getUnsafe().setMemory(payload, 32, (byte) 7); + Assert.assertEquals(0L, engine.appendBlocking(payload, 32)); + Assert.assertTrue(engine.acknowledge(0L)); + ff.beginClose(); + try { + engine.close(); + if (failAt >= 0) { + Assert.fail("sync failure was swallowed at boundary " + failAt); + } + } catch (IllegalStateException expected) { + Assert.assertTrue("unexpected close failure: " + expected, failAt >= 0); + } + engine = null; + + Assert.assertFalse("simulated crash replays acknowledged rows at boundary " + failAt, + ff.durableSegments && !ff.durableWatermark); + if (failAt == -1) { + Assert.assertEquals(Arrays.asList("watermark-msync", "watermark-fsync", + "dir-fsync", "segment-remove", "dir-fsync", "watermark-remove"), + ff.events); + } + } catch (Throwable t) { + failure = t; + } finally { + failure = closeEngine(failure, engine); + failure = closeManager(failure, manager); + failure = freePayload(failure, payload); + failure = removeRoot(failure, root); + } + rethrow(failure); + } + }); + } + + @Test + public void testFinalWatermarkBarrierFailureRetainsSlotUntilRetry() throws Exception { + TestUtils.assertMemoryLeak(() -> { + for (String barrier : Arrays.asList("watermark-msync", "watermark-fsync", "dir-fsync")) { + String root = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-close-barrier-" + barrier + "-" + System.nanoTime()).toString(); + String slot = root + "/slot"; + SegmentManager manager = null; + CursorSendEngine predecessor = null; + CursorSendEngine successor = null; + CrashImageFilesFacade ff = null; + long payload = 0; + Throwable failure = null; + try { + Assert.assertEquals(0, Files.mkdir(root, Files.DIR_MODE_DEFAULT)); + ff = new CrashImageFilesFacade(slot, -1); + long segmentSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 32L; + manager = new SegmentManager(segmentSize, TimeUnit.SECONDS.toNanos(60), + SegmentManager.UNLIMITED_TOTAL_BYTES, ff); + payload = Unsafe.malloc(32, MemoryTag.NATIVE_DEFAULT); + predecessor = new CursorSendEngine(slot, segmentSize, manager); + Unsafe.getUnsafe().setMemory(payload, 32, (byte) 7); + Assert.assertEquals(0L, predecessor.appendBlocking(payload, 32)); + Assert.assertTrue(predecessor.acknowledge(0L)); + + ff.failPersistently(barrier); + try { + predecessor.close(); + Assert.fail("expected close to report failed final barrier " + barrier); + } catch (IllegalStateException expected) { + Assert.assertTrue("unexpected close failure: " + expected, + expected.getMessage().contains("ack watermark") + || expected.getMessage().contains("slot directory")); + } + try { + successor = new CursorSendEngine(slot, segmentSize, manager); + Assert.fail("successor acquired a slot whose final ACK barrier failed: " + barrier); + } catch (IllegalStateException expected) { + Assert.assertTrue("successor failed for the wrong reason: " + expected, + expected.getMessage().contains("slot already in use")); + } + + ff.clearPersistentFailure(); + awaitCloseCompleted(predecessor); + predecessor = null; + + successor = new CursorSendEngine(slot, segmentSize, manager); + Assert.assertEquals("successful retry must leave no ACKed frame to replay", + -1L, successor.publishedFsn()); + successor.close(); + Assert.assertTrue(successor.isCloseCompleted()); + successor = null; + } catch (Throwable t) { + failure = t; + } finally { + if (failure != null && ff != null) { + // Let a retained predecessor converge before cleanup. + // The exact red assertion remains attached to failure. + ff.clearPersistentFailure(); + } + failure = closeEngine(failure, successor); + failure = closeEngine(failure, predecessor); + failure = closeManager(failure, manager); + failure = freePayload(failure, payload); + failure = removeRoot(failure, root); + } + rethrow(failure); + } + }); + } + + @Test + public void testRecoveredSlotRejectsMissingWatermark() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String root = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-missing-watermark-" + System.nanoTime()).toString(); + String slot = root + "/slot"; + SegmentManager manager = null; + CursorSendEngine predecessor = null; + CursorSendEngine successor = null; + long payload = 0; + Throwable failure = null; + try { + Assert.assertEquals(0, Files.mkdir(root, Files.DIR_MODE_DEFAULT)); + CrashImageFilesFacade ff = new CrashImageFilesFacade(slot, -1); + long segmentSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 32L; + manager = new SegmentManager(segmentSize, TimeUnit.SECONDS.toNanos(60), + SegmentManager.UNLIMITED_TOTAL_BYTES, ff); + payload = Unsafe.malloc(32, MemoryTag.NATIVE_DEFAULT); + predecessor = new CursorSendEngine(slot, segmentSize, manager); + Unsafe.getUnsafe().setMemory(payload, 32, (byte) 7); + Assert.assertEquals(0L, predecessor.appendBlocking(payload, 32)); + Assert.assertTrue(predecessor.acknowledge(0L)); + ff.beginClose(); + ff.blockSegmentRemove = true; + predecessor.close(); + Assert.assertTrue(predecessor.isCloseCompleted()); + predecessor = null; + Assert.assertTrue("precondition: ACKed segment residue must survive", + Files.exists(slot + "/sf-initial.sfa")); + + ff.failWatermarkOpen = true; + try { + successor = new CursorSendEngine(slot, segmentSize, manager); + Assert.fail("recovered disk slot started without a usable ACK watermark"); + } catch (IllegalStateException expected) { + Assert.assertTrue("successor failed for the wrong reason: " + expected, + expected.getMessage().contains("ack watermark")); + } + + ff.failWatermarkOpen = false; + ff.blockSegmentRemove = false; + successor = new CursorSendEngine(slot, segmentSize, manager); + Assert.assertTrue(successor.wasRecoveredFromDisk()); + Assert.assertTrue("successor exposes predecessor's ACKed frame", + successor.ackedFsn() >= successor.publishedFsn()); + successor.close(); + Assert.assertTrue(successor.isCloseCompleted()); + successor = null; + } catch (Throwable t) { + failure = t; + } finally { + failure = closeEngine(failure, successor); + failure = closeEngine(failure, predecessor); + failure = closeManager(failure, manager); + failure = freePayload(failure, payload); + failure = removeRoot(failure, root); + } + rethrow(failure); + }); + } + + private static Throwable addCleanupFailure(Throwable failure, Throwable cleanupFailure) { + if (failure == null) { + return cleanupFailure; + } + if (failure != cleanupFailure) { + failure.addSuppressed(cleanupFailure); + } + return failure; + } + + private static void awaitCloseCompleted(CursorSendEngine engine) { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (!engine.isCloseCompleted() && System.nanoTime() < deadline) { + Thread.yield(); + } + Assert.assertTrue("close retry did not complete", engine.isCloseCompleted()); + } + + private static Throwable closeEngine(Throwable failure, CursorSendEngine engine) { + if (engine != null) { + try { + engine.close(); + } catch (Throwable cleanupFailure) { + failure = addCleanupFailure(failure, cleanupFailure); + } + } + return failure; + } + + private static Throwable closeManager(Throwable failure, SegmentManager manager) { + if (manager != null) { + try { + manager.close(); + } catch (Throwable cleanupFailure) { + failure = addCleanupFailure(failure, cleanupFailure); + } + } + return failure; + } + + private static Throwable freePayload(Throwable failure, long payload) { + if (payload != 0) { + try { + Unsafe.free(payload, 32, MemoryTag.NATIVE_DEFAULT); + } catch (Throwable cleanupFailure) { + failure = addCleanupFailure(failure, cleanupFailure); + } + } + return failure; + } + + private static void removeRecursive(String dir) { + long find = Files.findFirst(dir); + if (find > 0) { + try { + int rc = 1; + while (rc > 0) { + String name = Files.utf8ToString(Files.findName(find)); + if (name != null && !".".equals(name) && !"..".equals(name)) { + String child = dir + "/" + name; + if (!Files.remove(child)) { + removeRecursive(child); + } + } + rc = Files.findNext(find); + } + } finally { + Files.findClose(find); + } + } + Files.remove(dir); + } + + private static Throwable removeRoot(Throwable failure, String root) { + try { + removeRecursive(root); + Assert.assertFalse("test directory was not removed: " + root, Files.exists(root)); + } catch (Throwable cleanupFailure) { + failure = addCleanupFailure(failure, cleanupFailure); + } + return failure; + } + + private static void rethrow(Throwable failure) { + if (failure != null) { + CursorSendEngineCrashConsistencyTest.throwUnchecked(failure); + } + } + + @SuppressWarnings("unchecked") + private static void throwUnchecked(Throwable failure) throws T { + throw (T) failure; + } + + private static final class CrashImageFilesFacade implements FilesFacade { + private final List events = new ArrayList<>(); + private final int failAt; + private final String slot; + private boolean active; + private boolean blockSegmentRemove; + private boolean durableSegments = true; + private boolean durableWatermark; + private int eventIndex; + private boolean failWatermarkOpen; + private String persistentFailure; + private int watermarkFd = -1; + + private CrashImageFilesFacade(String slot, int failAt) { + this.slot = slot; + this.failAt = failAt; + } + + private void beginClose() { + active = true; + events.clear(); + eventIndex = 0; + } + + private void clearPersistentFailure() { + persistentFailure = null; + } + + private boolean fail(String event) { + events.add(event); + return event.equals(persistentFailure) || eventIndex++ == failAt; + } + + private void failPersistently(String event) { + active = true; + events.clear(); + eventIndex = 0; + persistentFailure = event; + } + + @Override + public boolean allocate(int fd, long size) { return INSTANCE.allocate(fd, size); } + @Override + public long allocNativePath(String path) { return INSTANCE.allocNativePath(path); } + @Override + public int close(int fd) { return INSTANCE.close(fd); } + @Override + public boolean exists(String path) { return INSTANCE.exists(path); } + @Override + public void findClose(long findPtr) { INSTANCE.findClose(findPtr); } + @Override + public long findFirst(String dir) { return INSTANCE.findFirst(dir); } + @Override + public long findName(long findPtr) { return INSTANCE.findName(findPtr); } + @Override + public int findNext(long findPtr) { return INSTANCE.findNext(findPtr); } + @Override + public int findType(long findPtr) { return INSTANCE.findType(findPtr); } + @Override + public void freeNativePath(long pathPtr) { INSTANCE.freeNativePath(pathPtr); } + @Override + public int fsync(int fd) { + return active && fd == watermarkFd && fail("watermark-fsync") ? -1 : INSTANCE.fsync(fd); + } + @Override + public int fsyncDir(String dir) { + if (active && slot.equals(dir)) { + if (fail("dir-fsync")) return -1; + if (events.contains("segment-remove")) { + durableSegments = false; + } else { + durableWatermark = true; + } + } + return INSTANCE.fsyncDir(dir); + } + @Override + public long length(int fd) { return INSTANCE.length(fd); } + @Override + public long length(String path) { return INSTANCE.length(path); } + @Override + public long length(long pathPtr) { return INSTANCE.length(pathPtr); } + @Override + public int lock(int fd) { return INSTANCE.lock(fd); } + @Override + public int mkdir(String path, int mode) { return INSTANCE.mkdir(path, mode); } + @Override + public int msync(long addr, long len, boolean async) { + return active && fail("watermark-msync") ? -1 : INSTANCE.msync(addr, len, async); + } + @Override + public int openCleanRW(String path) { + if (failWatermarkOpen && path.equals(slot + "/" + AckWatermark.FILE_NAME)) return -1; + int fd = INSTANCE.openCleanRW(path); + if (path.equals(slot + "/" + AckWatermark.FILE_NAME)) watermarkFd = fd; + return fd; + } + @Override + public int openCleanRW(long pathPtr) { return INSTANCE.openCleanRW(pathPtr); } + @Override + public int openRW(String path) { + if (failWatermarkOpen && path.equals(slot + "/" + AckWatermark.FILE_NAME)) return -1; + int fd = INSTANCE.openRW(path); + if (path.equals(slot + "/" + AckWatermark.FILE_NAME)) watermarkFd = fd; + return fd; + } + @Override + public int openRW(long pathPtr) { return INSTANCE.openRW(pathPtr); } + @Override + public long read(int fd, long addr, long len, long offset) { + return INSTANCE.read(fd, addr, len, offset); + } + @Override + public boolean remove(String path) { + if (active && path.endsWith(".sfa")) { + if (blockSegmentRemove || fail("segment-remove")) return false; + } else if (active && path.equals(slot + "/" + AckWatermark.FILE_NAME)) { + if (fail("watermark-remove")) return false; + } + return INSTANCE.remove(path); + } + @Override + public boolean remove(long pathPtr) { return INSTANCE.remove(pathPtr); } + @Override + public int rename(String oldPath, String newPath) { return INSTANCE.rename(oldPath, newPath); } + @Override + public boolean truncate(int fd, long size) { return INSTANCE.truncate(fd, size); } + @Override + public long write(int fd, long addr, long len, long offset) { + return INSTANCE.write(fd, addr, len, offset); + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java new file mode 100644 index 00000000..4c59bcfe --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineSlotReacquisitionTest.java @@ -0,0 +1,852 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentRing; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SlotLock; +import io.questdb.client.std.Files; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.nio.file.Paths; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +/** + * Engine-level regression for the shutdown hazard where + * {@link CursorSendEngine#close()} released the slot lock, closed the ring + * and watermark, and unlinked segment files while the shared + * {@link SegmentManager} worker was still mid service pass for the engine's + * ring. A replacement engine could acquire the same slot the moment the + * lock was released, after which the stale worker's abandon/trim path could + * unlink a segment path the replacement was actively writing through — + * store-and-forward data loss after restart. + *

      + * The fix makes {@code close()} run a quiescence barrier + * ({@link SegmentManager#awaitRingQuiescence}) after {@code deregister} and + * refuse to release any worker-reachable resource (ring, watermark, segment + * files, slot lock) until the barrier confirms the worker cannot touch the + * slot again. On barrier timeout an owned-manager engine hands cleanup + * ownership to the worker's exit path (see + * {@code SegmentManager.deferUntilWorkerExit}); a shared-manager engine + * deliberately leaks and a later {@code close()} retries the cleanup. + */ +public class CursorSendEngineSlotReacquisitionTest { + + private String tmpDir; + + @Before + public void setUp() { + tmpDir = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-engine-slot-reacq-" + System.nanoTime()).toString(); + Assert.assertEquals(0, Files.mkdir(tmpDir, Files.DIR_MODE_DEFAULT)); + } + + @After + public void tearDown() { + if (tmpDir == null) return; + rmDirRecursive(tmpDir); + Files.remove(tmpDir); + } + + /** + * The structural guarantee: while the manager worker is provably still + * inside a service pass for the engine's ring, repeated direct + * {@code close()} calls must NOT hand the slot to anyone else. With the + * duplicate-cleanup-owner branch reverted, the second close runs cleanup + * inline and the mid-test {@code SlotLock.acquire} probe succeeds. Once + * the pass finishes, its deferred cleanup must run exactly once and + * release the slot without another close retry. + */ + @Test(timeout = 30_000L) + public void testRepeatedCloseRetainsSlotWhileWorkerIsMidServicePass() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long segSize = MmapSegment.HEADER_SIZE + (MmapSegment.FRAME_HEADER_SIZE + 32); + String slot = tmpDir + "/slot"; + // 60 s poll: the worker only acts when explicitly woken, so the + // single pass we park below is the only pass in flight. + SegmentManager manager = new SegmentManager(segSize, TimeUnit.SECONDS.toNanos(60)); + CountDownLatch cleanupFinished = new CountDownLatch(1); + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(); + AtomicInteger cleanupCount = new AtomicInteger(); + AtomicReference hookErr = new AtomicReference<>(); + boolean managerClosed = false; + CursorSendEngine engine = null; + try { + manager.setAfterRingCleanupHook(() -> { + cleanupCount.incrementAndGet(); + cleanupFinished.countDown(); + }); + manager.setBeforeInstallSyncHook(() -> { + if (!fired.compareAndSet(false, true)) return; + workerBlocked.countDown(); + try { + if (!releaseWorker.await(20, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, + new AssertionError("timed out waiting for test to release worker")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + manager.start(); + + // Shared manager: ownsManager=false, so engine close() cannot + // fall back on manager.close()'s join — the per-ring barrier + // is the only protection, which is exactly what we pin here. + engine = new CursorSendEngine(slot, segSize, manager); + Assert.assertTrue("worker never reached the install hook", + workerBlocked.await(5, TimeUnit.SECONDS)); + + // Barrier must time out fast: the worker is parked inside the + // service pass for this engine's ring. + manager.setWorkerJoinTimeoutMillis(50L); + engine.close(); + Assert.assertFalse("incomplete close must remain observable to the owner", + engine.isCloseCompleted()); + + // Exercise CursorSendEngine.close() directly again while the + // same pass already owns deferredClose. Sender.close() cannot + // reach this branch because its second call is a no-op. + engine.close(); + Assert.assertFalse("repeated close must not steal deferred cleanup ownership", + engine.isCloseCompleted()); + Assert.assertEquals("cleanup ran while the worker pass was blocked", + 0, cleanupCount.get()); + + // The slot must still be locked: a replacement engine (or raw + // SlotLock) acquiring it now would race the stale worker. + try { + SlotLock probe = SlotLock.acquire(slot); + probe.close(); + Assert.fail("engine.close() released the slot lock while the manager " + + "worker was still mid service pass for its ring — a " + + "replacement engine could acquire the slot and have its " + + "segment files unlinked by the stale worker"); + } catch (Exception expected) { + // good — slot retained. + } + + // Let the worker finish its pass (it abandons the spare: the + // ring was deregistered by the close attempt above). + releaseWorker.countDown(); + Assert.assertTrue("ring pass did not finish deferred cleanup", + cleanupFinished.await(5, TimeUnit.SECONDS)); + Assert.assertEquals("ring-pass cleanup must run exactly once", + 1, cleanupCount.get()); + Assert.assertTrue("ring-pass cleanup must report complete cleanup", + engine.isCloseCompleted()); + engine = null; + + try (SlotLock probe = SlotLock.acquire(slot)) { + Assert.assertNotNull("slot must be acquirable after a completed close", probe); + } catch (Exception e) { + throw new AssertionError("ring-pass cleanup did not release the slot lock", e); + } + + manager.close(); + managerClosed = true; + if (hookErr.get() != null) { + throw new AssertionError("install hook failed", hookErr.get()); + } + } finally { + manager.setAfterRingCleanupHook(null); + manager.setBeforeInstallSyncHook(null); + releaseWorker.countDown(); + if (!managerClosed) { + manager.close(); + } + } + }); + } + + /** + * Owned-manager twin of {@link #testRepeatedCloseRetainsSlotWhileWorkerIsMidServicePass}: + * the ONLY construction shape production uses (Sender.build, BackgroundDrainer, + * QwpWebSocketSender.connect all own their manager). The owned close path does + * not run the per-ring barrier at all — it relies on {@code manager.close()}'s + * bounded join and the {@code isWorkerReaped()} check. If that check regressed + * to report quiescence unconditionally (or {@code isWorkerReaped()} itself + * returned true while the worker is alive), close() would release the slot + * lock mid service pass and the shared-manager tests would stay green — this + * test is the red gate for the production path. + *

      + * Determinism: the owned manager starts inside the engine ctor (1 ms poll), + * so its first spare-install pass races test setup. We first wait until the + * initial hot spare is installed — after that the worker cannot enter another + * install pass until a rotation consumes the spare, so the park hook installed + * afterwards can neither be missed nor fire early. Two appends then fill the + * active segment and rotate onto the spare; the worker's next poll tick + * re-enters the install pass and parks in the hook. + */ + @Test(timeout = 30_000L) + public void testOwnedEngineCloseRetainsSlotWhileWorkerIsMidServicePass() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final int payloadLen = 32; + long segSize = MmapSegment.HEADER_SIZE + (MmapSegment.FRAME_HEADER_SIZE + payloadLen); + String slot = tmpDir + "/owned-parked-slot"; + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(); + AtomicReference hookErr = new AtomicReference<>(); + // Production shape: private, owned manager (ownsManager=true). + CursorSendEngine engine = new CursorSendEngine(slot, segSize); + SegmentManager manager = engine.getManagerForTesting(); + long buf = Unsafe.malloc(payloadLen, MemoryTag.NATIVE_DEFAULT); + try { + // Phase 1: let the worker finish the initial spare install so + // the hook below can only fire on the rotation-triggered pass. + SegmentRing ring = engine.getRingForTesting(); + long deadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (ring.needsHotSpare()) { + if (System.nanoTime() > deadlineNs) { + throw new AssertionError("manager worker never installed the initial hot spare"); + } + Thread.sleep(1); + } + manager.setBeforeInstallSyncHook(() -> { + if (!fired.compareAndSet(false, true)) return; + workerBlocked.countDown(); + try { + if (!releaseWorker.await(20, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, + new AssertionError("timed out waiting for test to release worker")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + + // Phase 2: one frame fills the active segment exactly; the + // second forces rotation onto the spare. needsHotSpare() is + // true again, so the worker's next tick parks in the hook. + Unsafe.getUnsafe().putLong(buf, 0L); + Assert.assertEquals(0L, engine.appendBlocking(buf, payloadLen)); + Assert.assertEquals(1L, engine.appendBlocking(buf, payloadLen)); + Assert.assertTrue("worker never re-entered a spare-install pass", + workerBlocked.await(5, TimeUnit.SECONDS)); + + // Phase 3: owned close with the worker provably mid service + // pass. manager.close()'s 50 ms join times out, the worker is + // not reaped, and close() must retain every worker-reachable + // resource — above all the slot flock. + manager.setWorkerJoinTimeoutMillis(50L); + engine.close(); + Assert.assertFalse("incomplete owned close must remain observable to the owner", + engine.isCloseCompleted()); + try { + SlotLock probe = SlotLock.acquire(slot); + probe.close(); + Assert.fail("owned engine.close() released the slot lock while its manager " + + "worker was still mid service pass — a replacement engine could " + + "acquire the slot and have its segment files unlinked by the " + + "stale worker (the production SF-data-loss hazard)"); + } catch (Exception expected) { + // good — slot retained. + } + + // Phase 4: release the worker (it abandons the spare: the ring + // was deregistered by the close attempt) and retry. The join + // now reaps the worker and the full cleanup must complete. + releaseWorker.countDown(); + manager.setWorkerJoinTimeoutMillis(TimeUnit.SECONDS.toMillis(60)); + engine.close(); + Assert.assertTrue("retried owned close must report complete cleanup", + engine.isCloseCompleted()); + try (SlotLock probe = SlotLock.acquire(slot)) { + Assert.assertNotNull("slot must be acquirable after a completed close", probe); + } catch (Exception e) { + throw new AssertionError("retried owned close() did not release the slot lock", e); + } + if (hookErr.get() != null) { + throw new AssertionError("install hook failed", hookErr.get()); + } + } finally { + Unsafe.free(buf, payloadLen, MemoryTag.NATIVE_DEFAULT); + manager.setBeforeInstallSyncHook(null); + releaseWorker.countDown(); + manager.setWorkerJoinTimeoutMillis(TimeUnit.SECONDS.toMillis(60)); + try { + engine.close(); + } catch (Throwable ignored) { + } + } + }); + } + + /** + * The ownership handoff (owned manager): when close() cannot confirm + * worker quiescence within the bounded join, the terminal cleanup (ring, + * watermark, flock release) transfers to the worker's exit path — the + * worker is provably the last thread able to touch the slot directory. + * Once the parked pass is released the worker must run the cleanup + * itself, WITHOUT any retried {@code close()}: {@code isCloseCompleted()} + * flips true and the slot becomes acquirable again. This is what lets a + * pool recover a retired slot instead of losing its capacity until + * process exit. + */ + @Test(timeout = 30_000L) + public void testOwnedEngineCloseHandsCleanupToWorkerExit() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final int payloadLen = 32; + long segSize = MmapSegment.HEADER_SIZE + (MmapSegment.FRAME_HEADER_SIZE + payloadLen); + String slot = tmpDir + "/owned-handoff-slot"; + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(); + AtomicReference hookErr = new AtomicReference<>(); + // Production shape: private, owned manager (ownsManager=true). + CursorSendEngine engine = new CursorSendEngine(slot, segSize); + SegmentManager manager = engine.getManagerForTesting(); + long buf = Unsafe.malloc(payloadLen, MemoryTag.NATIVE_DEFAULT); + try { + // Phase 1: wait out the initial spare install so the park hook + // can only fire on the rotation-triggered pass (see + // testOwnedEngineCloseRetainsSlotWhileWorkerIsMidServicePass). + SegmentRing ring = engine.getRingForTesting(); + long deadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (ring.needsHotSpare()) { + if (System.nanoTime() > deadlineNs) { + throw new AssertionError("manager worker never installed the initial hot spare"); + } + Thread.sleep(1); + } + manager.setBeforeInstallSyncHook(() -> { + if (!fired.compareAndSet(false, true)) return; + workerBlocked.countDown(); + try { + if (!releaseWorker.await(20, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, + new AssertionError("timed out waiting for test to release worker")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + + // Phase 2: fill the active segment and rotate onto the spare; + // the worker's next tick re-enters the install pass and parks. + Unsafe.getUnsafe().putLong(buf, 0L); + Assert.assertEquals(0L, engine.appendBlocking(buf, payloadLen)); + Assert.assertEquals(1L, engine.appendBlocking(buf, payloadLen)); + Assert.assertTrue("worker never re-entered a spare-install pass", + workerBlocked.await(5, TimeUnit.SECONDS)); + + // Phase 3: owned close with the worker provably mid-pass. The + // bounded join times out; cleanup ownership is handed to the + // worker's exit path. Every worker-reachable resource — above + // all the slot flock — must still be retained at this point. + manager.setWorkerJoinTimeoutMillis(50L); + engine.close(); + Assert.assertFalse("close must stay incomplete while the worker holds the handoff", + engine.isCloseCompleted()); + try { + SlotLock probe = SlotLock.acquire(slot); + probe.close(); + Assert.fail("engine.close() released the slot lock while its manager worker " + + "was still mid service pass — the handoff must not weaken the " + + "quiescence gate"); + } catch (Exception expected) { + // good — slot retained while the worker can still touch it. + } + + // Phase 4 — the contract under test: release the worker and do + // NOT retry close(). The worker finishes its pass, exits, and + // runs the deferred cleanup itself, flipping isCloseCompleted + // and releasing the flock with no further caller action. + releaseWorker.countDown(); + long cleanupDeadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (!engine.isCloseCompleted()) { + if (System.nanoTime() > cleanupDeadlineNs) { + throw new AssertionError( + "deferred cleanup never ran on manager-worker exit — the slot " + + "would stay retired until process exit"); + } + Thread.sleep(1); + } + try (SlotLock probe = SlotLock.acquire(slot)) { + Assert.assertNotNull("slot must be acquirable after the worker-exit cleanup", probe); + } catch (Exception e) { + throw new AssertionError("worker-exit cleanup did not release the slot lock", e); + } + if (hookErr.get() != null) { + throw new AssertionError("install hook failed", hookErr.get()); + } + } finally { + Unsafe.free(buf, payloadLen, MemoryTag.NATIVE_DEFAULT); + manager.setBeforeInstallSyncHook(null); + releaseWorker.countDown(); + manager.setWorkerJoinTimeoutMillis(TimeUnit.SECONDS.toMillis(60)); + try { + engine.close(); + } catch (Throwable ignored) { + } + } + }); + } + + /** + * Registration-failure twin of + * {@link #testOwnedEngineCloseHandsCleanupToWorkerExit}: when + * {@code deferUntilWorkerExit} itself throws (allocation failure while + * building the handoff), close() must NOT mistake the swallowed throw + * for "worker already exited" and run the terminal cleanup inline — the + * worker is provably still mid service pass, so releasing the ring, + * watermark or slot flock here is the original stale-worker UAF/data-loss + * hazard. Every worker-reachable resource must be retained and the close + * must stay incomplete; a retried close() after the worker exits + * converges and releases the slot. + */ + @Test(timeout = 30_000L) + public void testOwnedEngineCloseRetainsSlotWhenHandoffRegistrationFails() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final int payloadLen = 32; + long segSize = MmapSegment.HEADER_SIZE + (MmapSegment.FRAME_HEADER_SIZE + payloadLen); + String slot = tmpDir + "/owned-regfail-slot"; + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(); + AtomicReference hookErr = new AtomicReference<>(); + // Production shape: private, owned manager (ownsManager=true). + CursorSendEngine engine = new CursorSendEngine(slot, segSize); + SegmentManager manager = engine.getManagerForTesting(); + long buf = Unsafe.malloc(payloadLen, MemoryTag.NATIVE_DEFAULT); + try { + // Phase 1: wait out the initial spare install so the park hook + // can only fire on the rotation-triggered pass (see + // testOwnedEngineCloseRetainsSlotWhileWorkerIsMidServicePass). + SegmentRing ring = engine.getRingForTesting(); + long deadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (ring.needsHotSpare()) { + if (System.nanoTime() > deadlineNs) { + throw new AssertionError("manager worker never installed the initial hot spare"); + } + Thread.sleep(1); + } + manager.setBeforeInstallSyncHook(() -> { + if (!fired.compareAndSet(false, true)) return; + workerBlocked.countDown(); + try { + if (!releaseWorker.await(20, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, + new AssertionError("timed out waiting for test to release worker")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + + // Phase 2: fill the active segment and rotate onto the spare; + // the worker's next tick re-enters the install pass and parks. + Unsafe.getUnsafe().putLong(buf, 0L); + Assert.assertEquals(0L, engine.appendBlocking(buf, payloadLen)); + Assert.assertEquals(1L, engine.appendBlocking(buf, payloadLen)); + Assert.assertTrue("worker never re-entered a spare-install pass", + workerBlocked.await(5, TimeUnit.SECONDS)); + + // Phase 3: make the handoff registration throw — simulating + // an OutOfMemoryError allocating the cleanup lambda/list — + // with the worker provably still mid service pass. close() + // must retain everything: no inline finishClose, no flock + // release, closeCompleted stays false. + manager.setBeforeExitCleanupRegistrationHook(() -> { + throw new OutOfMemoryError("simulated allocation failure registering exit cleanup"); + }); + manager.setWorkerJoinTimeoutMillis(50L); + engine.close(); + Assert.assertFalse("close must stay incomplete when handoff registration fails", + engine.isCloseCompleted()); + try { + SlotLock probe = SlotLock.acquire(slot); + probe.close(); + Assert.fail("engine.close() released the slot lock after a failed handoff " + + "registration while the manager worker was still mid service " + + "pass — the swallowed throw was mistaken for proof the worker " + + "exited (stale-worker UAF/data-loss hazard)"); + } catch (Exception expected) { + // good — slot retained while the worker can still touch it. + } + + // Phase 4: clear the fault, release the worker (its loop was + // already stopped by the close attempt, so it exits), and + // retry close(). The retry must converge via isWorkerReaped() + // and release the slot. + manager.setBeforeExitCleanupRegistrationHook(null); + releaseWorker.countDown(); + manager.setWorkerJoinTimeoutMillis(TimeUnit.SECONDS.toMillis(60)); + engine.close(); + Assert.assertTrue("retried close must report complete cleanup", + engine.isCloseCompleted()); + try (SlotLock probe = SlotLock.acquire(slot)) { + Assert.assertNotNull("slot must be acquirable after the retried close", probe); + } catch (Exception e) { + throw new AssertionError("retried close() did not release the slot lock", e); + } + if (hookErr.get() != null) { + throw new AssertionError("install hook failed", hookErr.get()); + } + } finally { + Unsafe.free(buf, payloadLen, MemoryTag.NATIVE_DEFAULT); + manager.setBeforeInstallSyncHook(null); + manager.setBeforeExitCleanupRegistrationHook(null); + releaseWorker.countDown(); + manager.setWorkerJoinTimeoutMillis(TimeUnit.SECONDS.toMillis(60)); + try { + engine.close(); + } catch (Throwable ignored) { + } + } + }); + } + + /** + * Exactly-once contention on the terminal-cleanup claim + * ({@code terminalCleanupClaimed} CAS): after a timed-out owned close + * handed cleanup to the worker's exit path, a retried {@code close()} + * that races the worker MID-{@code finishClose} must neither re-run the + * terminal cleanup (double munmap / double flock release) nor block on + * the worker, nor publish completion on the worker's behalf. The race + * window is made deterministic by parking the worker inside + * {@code finishClose} (via {@code beforeFlockReleaseHook}) while the + * retried close() converges through {@code isWorkerReaped()} and loses + * the CAS. + */ + @Test(timeout = 30_000L) + public void testTerminalCleanupRunsExactlyOnceWhenRetriedCloseRacesWorkerHandoff() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final int payloadLen = 32; + long segSize = MmapSegment.HEADER_SIZE + (MmapSegment.FRAME_HEADER_SIZE + payloadLen); + String slot = tmpDir + "/cas-contention-slot"; + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + CountDownLatch inFinishClose = new CountDownLatch(1); + CountDownLatch releaseFinishClose = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(); + AtomicInteger finishCloseRuns = new AtomicInteger(); + AtomicReference hookErr = new AtomicReference<>(); + // Production shape: private, owned manager (ownsManager=true). + CursorSendEngine engine = new CursorSendEngine(slot, segSize); + SegmentManager manager = engine.getManagerForTesting(); + long buf = Unsafe.malloc(payloadLen, MemoryTag.NATIVE_DEFAULT); + try { + // Phase 1: wait out the initial spare install so the park hook + // can only fire on the rotation-triggered pass. + SegmentRing ring = engine.getRingForTesting(); + long deadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (ring.needsHotSpare()) { + if (System.nanoTime() > deadlineNs) { + throw new AssertionError("manager worker never installed the initial hot spare"); + } + Thread.sleep(1); + } + manager.setBeforeInstallSyncHook(() -> { + if (!fired.compareAndSet(false, true)) return; + workerBlocked.countDown(); + try { + if (!releaseWorker.await(20, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, + new AssertionError("timed out waiting for test to release worker")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + // Counts terminal-cleanup executions and parks the FIRST one + // (the worker's deferred cleanup) mid-finishClose, before the + // flock release — the exact window a retried close() races. + engine.setBeforeFlockReleaseHook(() -> { + if (finishCloseRuns.incrementAndGet() == 1) { + inFinishClose.countDown(); + try { + if (!releaseFinishClose.await(20, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, new AssertionError( + "timed out waiting for test to release finishClose")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + } + }); + + // Phase 2: rotate onto the spare so the worker parks in the + // next install pass. + Unsafe.getUnsafe().putLong(buf, 0L); + Assert.assertEquals(0L, engine.appendBlocking(buf, payloadLen)); + Assert.assertEquals(1L, engine.appendBlocking(buf, payloadLen)); + Assert.assertTrue("worker never re-entered a spare-install pass", + workerBlocked.await(5, TimeUnit.SECONDS)); + + // Phase 3: timed-out close — cleanup ownership transfers to + // the worker's exit path. + manager.setWorkerJoinTimeoutMillis(50L); + engine.close(); + Assert.assertFalse("close must stay incomplete while the worker holds the handoff", + engine.isCloseCompleted()); + + // Phase 4: release the pass. The worker exits its loop, wins + // the cleanup CAS, enters finishClose and parks in the hook — + // mid-cleanup, flock still held, completion unpublished. + releaseWorker.countDown(); + Assert.assertTrue("worker never entered the deferred finishClose", + inFinishClose.await(10, TimeUnit.SECONDS)); + Assert.assertEquals(1, finishCloseRuns.get()); + Assert.assertFalse("completion must not be observable mid-finishClose", + engine.isCloseCompleted()); + + // Phase 5 — the contention under test: a retried close() while + // the worker is parked INSIDE finishClose. The worker loop has + // already exited (workerLoopExited=true precedes the deferred + // cleanups), so the short bounded join reaps the manager state + // and close() converges to the CAS — which it must LOSE, + // returning promptly without touching ring/watermark/flock. + engine.close(); + Assert.assertEquals( + "retried close() re-ran the terminal cleanup while the worker's " + + "deferred cleanup was mid-flight — ring/watermark/flock would " + + "be double-released", + 1, finishCloseRuns.get()); + Assert.assertFalse("retried close() must not publish completion on the worker's behalf", + engine.isCloseCompleted()); + try { + SlotLock probe = SlotLock.acquire(slot); + probe.close(); + Assert.fail("slot lock observable as released while the worker was still " + + "parked before its flock release"); + } catch (Exception expected) { + // good — flock still held by the parked cleanup. + } + + // Phase 6: let the worker finish. Completion publishes, the + // slot becomes acquirable, and the cleanup count stays at 1. + releaseFinishClose.countDown(); + long cleanupDeadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (!engine.isCloseCompleted()) { + if (System.nanoTime() > cleanupDeadlineNs) { + throw new AssertionError("deferred cleanup never completed after release"); + } + Thread.sleep(1); + } + Assert.assertEquals(1, finishCloseRuns.get()); + try (SlotLock probe = SlotLock.acquire(slot)) { + Assert.assertNotNull("slot must be acquirable after the worker-exit cleanup", probe); + } + // A final close() takes the fast no-op path. + engine.close(); + Assert.assertEquals("post-completion close() must be a no-op", + 1, finishCloseRuns.get()); + if (hookErr.get() != null) { + throw new AssertionError("hook failed", hookErr.get()); + } + } finally { + Unsafe.free(buf, payloadLen, MemoryTag.NATIVE_DEFAULT); + manager.setBeforeInstallSyncHook(null); + engine.setBeforeFlockReleaseHook(null); + releaseWorker.countDown(); + releaseFinishClose.countDown(); + manager.setWorkerJoinTimeoutMillis(TimeUnit.SECONDS.toMillis(60)); + try { + engine.close(); + } catch (Throwable ignored) { + } + } + }); + } + + /** + * Memory-mode twin of {@link #testOwnedEngineCloseHandsCleanupToWorkerExit}: + * {@code sfDir == null}, so there is no slot lock, no watermark and no + * segment files — but the ring's malloc'd native segments are still + * worker-reachable, so the timed-out close must take the same handoff + * path with every SF-only resource null. Pins that (a) the handoff + * branch tolerates null slotLock/watermark/sfDir without NPE, (b) the + * close stays incomplete while the worker can still touch the ring, and + * (c) the worker-exit cleanup completes the close and frees the ring's + * native memory (assertMemoryLeak is the leak oracle here). + */ + @Test(timeout = 30_000L) + public void testMemoryModeOwnedCloseHandsCleanupToWorkerExit() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final int payloadLen = 32; + long segSize = MmapSegment.HEADER_SIZE + (MmapSegment.FRAME_HEADER_SIZE + payloadLen); + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(); + AtomicReference hookErr = new AtomicReference<>(); + // Memory mode: null sfDir, private owned manager — the exact + // shape non-SF async ingest uses. + CursorSendEngine engine = new CursorSendEngine(null, segSize); + SegmentManager manager = engine.getManagerForTesting(); + long buf = Unsafe.malloc(payloadLen, MemoryTag.NATIVE_DEFAULT); + try { + // Phase 1: wait out the initial spare install so the park hook + // can only fire on the rotation-triggered pass. + SegmentRing ring = engine.getRingForTesting(); + long deadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (ring.needsHotSpare()) { + if (System.nanoTime() > deadlineNs) { + throw new AssertionError("manager worker never installed the initial hot spare"); + } + Thread.sleep(1); + } + manager.setBeforeInstallSyncHook(() -> { + if (!fired.compareAndSet(false, true)) return; + workerBlocked.countDown(); + try { + if (!releaseWorker.await(20, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, + new AssertionError("timed out waiting for test to release worker")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + + // Phase 2: rotate onto the spare; the worker's next tick + // re-enters the (in-memory) install pass and parks. + Unsafe.getUnsafe().putLong(buf, 0L); + Assert.assertEquals(0L, engine.appendBlocking(buf, payloadLen)); + Assert.assertEquals(1L, engine.appendBlocking(buf, payloadLen)); + Assert.assertTrue("worker never re-entered a spare-install pass", + workerBlocked.await(5, TimeUnit.SECONDS)); + + // Phase 3: timed-out memory-mode close. The worker can still + // touch the ring's native memory, so the close must hand off + // and stay incomplete — releasing the ring here would be a + // use-after-free on the worker's install path. + manager.setWorkerJoinTimeoutMillis(50L); + engine.close(); + Assert.assertFalse( + "memory-mode close must stay incomplete while the worker is mid service pass", + engine.isCloseCompleted()); + + // Phase 4: release the worker; its exit path must run the + // deferred cleanup (null slotLock/watermark/sfDir) and flip + // completion with no further caller action. + releaseWorker.countDown(); + long cleanupDeadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (!engine.isCloseCompleted()) { + if (System.nanoTime() > cleanupDeadlineNs) { + throw new AssertionError( + "deferred memory-mode cleanup never ran on manager-worker exit — " + + "the ring's native segments would leak for the process lifetime"); + } + Thread.sleep(1); + } + if (hookErr.get() != null) { + throw new AssertionError("install hook failed", hookErr.get()); + } + } finally { + Unsafe.free(buf, payloadLen, MemoryTag.NATIVE_DEFAULT); + manager.setBeforeInstallSyncHook(null); + releaseWorker.countDown(); + manager.setWorkerJoinTimeoutMillis(TimeUnit.SECONDS.toMillis(60)); + try { + engine.close(); + } catch (Throwable ignored) { + } + } + }); + } + + /** + * An engine that owns its manager must use the whole-manager stop/join as + * its only quiescence barrier. Calling the per-ring barrier first would + * give a stuck worker two independent timeout budgets. + */ + @Test(timeout = 30_000L) + public void testOwnedManagerCloseSkipsPerRingQuiescenceWait() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String slot = tmpDir + "/owned-slot"; + CursorSendEngine engine = new CursorSendEngine(slot, 4L * 1024 * 1024); + SegmentManager manager = engine.getManagerForTesting(); + AtomicBoolean perRingAwaited = new AtomicBoolean(); + try { + manager.setBeforeRingQuiescenceAwaitHook(() -> perRingAwaited.set(true)); + engine.close(); + Assert.assertTrue("owned engine close did not complete", engine.isCloseCompleted()); + Assert.assertFalse("owned engine close spent a separate per-ring wait budget", + perRingAwaited.get()); + } finally { + manager.setBeforeRingQuiescenceAwaitHook(null); + engine.close(); + } + }); + } + + /** + * Plain-positive path: after a normal close (worker quiesces promptly), + * a second engine must be able to acquire and use the same slot. + */ + @Test(timeout = 30_000L) + public void testSameSlotReacquirableAfterNormalClose() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String slot = tmpDir + "/slot"; + CursorSendEngine first = new CursorSendEngine(slot, 4L * 1024 * 1024); + first.close(); + CursorSendEngine second = new CursorSendEngine(slot, 4L * 1024 * 1024); + try { + Assert.assertFalse("fully-drained close must leave no segments to recover", + second.wasRecoveredFromDisk()); + } finally { + second.close(); + } + }); + } + + private static void rmDirRecursive(String dir) { + if (!Files.exists(dir)) return; + long find = Files.findFirst(dir); + if (find <= 0) return; + try { + int rc = 1; + while (rc > 0) { + String name = Files.utf8ToString(Files.findName(find)); + if (name != null && !".".equals(name) && !"..".equals(name)) { + String child = dir + "/" + name; + if (!Files.remove(child)) { + rmDirRecursive(child); + Files.remove(child); + } + } + rc = Files.findNext(find); + } + } finally { + Files.findClose(find); + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java index f4de1ff2..0cf5ab69 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java @@ -28,6 +28,7 @@ import io.questdb.client.cutlass.qwp.client.sf.cursor.AckWatermark; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; +import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegmentException; import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; import io.questdb.client.cutlass.qwp.client.sf.cursor.SlotLock; import io.questdb.client.std.Files; @@ -104,6 +105,55 @@ public void testAcknowledgePropagatesToRing() throws Exception { }); } + @Test + public void testCheckDurabilitySurfacesLatchedFailureToSenderEntryPoints() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // QwpWebSocketSender.flushAndGetSequence()/awaitAckedFsn() call + // engine.checkDurability() as their FIRST durability act, so this + // delegation seam is what stands between a latched periodic + // data-barrier failure and a producer that keeps publishing into + // an unsyncable slot. It had zero test references before this + // pin. Contract: quiet when clean, throws the LATCHED instance + // (not a copy) on every call until the manager's healed pass + // clears it -- callers poll it, so it must be repeatable, not + // one-shot. + try (CursorSendEngine engine = new CursorSendEngine(tmpDir, 4096)) { + engine.checkDurability(); // clean: must not throw + MmapSegmentException failure = new MmapSegmentException("injected data-sync failure"); + engine.getRingForTesting().recordDurabilityFailureForTesting(failure); + for (int i = 0; i < 2; i++) { + try { + engine.checkDurability(); + fail("latched durability failure must surface on call #" + i); + } catch (MmapSegmentException expected) { + assertTrue("the latched instance itself must surface", + expected == failure); + } + } + } + }); + } + + @Test + public void testAppendChecksLatchedDurabilityFailureBeforePublishing() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + try (CursorSendEngine engine = new CursorSendEngine(tmpDir, 4096)) { + MmapSegmentException failure = new MmapSegmentException("injected data-sync failure"); + engine.getRingForTesting().recordDurabilityFailureForTesting(failure); + try { + engine.appendBlocking(buf, 16); + fail("expected latched durability failure"); + } catch (MmapSegmentException expected) { + assertTrue(expected == failure); + } + assertEquals(-1L, engine.publishedFsn()); + } finally { + Unsafe.free(buf, 16, MemoryTag.NATIVE_DEFAULT); + } + }); + } + @Test public void testAppendBlockingNeverFailsUnderManagerSupply() throws Exception { TestUtils.assertMemoryLeak(() -> { @@ -197,6 +247,25 @@ public void testCloseIsIdempotent() throws Exception { }); } + @Test + public void testCallbackCreationFailurePrecedesOwnedManagerResources() throws Exception { + TestUtils.assertMemoryLeak(() -> { + CursorSendEngine.setBeforeDeferredCloseCreationHook(() -> { + throw new OutOfMemoryError("simulated bound callback allocation failure"); + }); + try { + try { + new CursorSendEngine(tmpDir, 4096); + fail("expected callback allocation failure"); + } catch (OutOfMemoryError expected) { + assertEquals("simulated bound callback allocation failure", expected.getMessage()); + } + } finally { + CursorSendEngine.setBeforeDeferredCloseCreationHook(null); + } + }); + } + @Test public void testConstructorFailureAfterOwnedManagerStartCleansResources() throws Exception { TestUtils.assertMemoryLeak(() -> { @@ -381,6 +450,53 @@ public void testManagerPersistedWatermarkSurvivesRestart() throws Exception { }); } + @Test + public void testPeriodicRotationWaitsForDurablePredecessor() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long segmentSize = MmapSegment.HEADER_SIZE + + 2L * (MmapSegment.FRAME_HEADER_SIZE + 64L); + long buf = Unsafe.malloc(64, MemoryTag.NATIVE_DEFAULT); + CursorSendEngine engine = new CursorSendEngine( + tmpDir, segmentSize, segmentSize * 4L, + TimeUnit.SECONDS.toNanos(5), TimeUnit.HOURS.toNanos(1)); + try { + assertEquals(0L, engine.appendBlocking(buf, 64)); + assertEquals(1L, engine.appendBlocking(buf, 64)); + // The third append needs rotation. Its predecessor is dirty and + // the periodic deadline is an hour away, so only the explicit + // rotation request can make progress. + assertEquals(2L, engine.appendBlocking(buf, 64)); + ObjList sealed = engine.getRingForTesting().getSealedSegments(); + assertEquals(1, sealed.size()); + assertTrue(sealed.getQuick(0).isPublishedDurable()); + MmapSegment active = engine.getRingForTesting().getActive(); + assertFalse(active.isPublishedDurable()); + engine.close(); + assertTrue("close must sync the unacknowledged active", active.isPublishedDurable()); + } finally { + engine.close(); + Unsafe.free(buf, 64, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + @Test + public void testPeriodicValidationPrecedesOwnedManagerAllocation() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try { + new CursorSendEngine( + null, + 4096L, + 8192L, + CursorSendEngine.DEFAULT_APPEND_DEADLINE_NANOS, + 1L); + fail("expected periodic memory-mode validation failure"); + } catch (IllegalArgumentException expected) { + assertTrue(expected.getMessage().contains("requires disk store-and-forward mode")); + } + }); + } + @Test public void testRestartIntoNonEmptySfDirContinuesFsnSequence() throws Exception { TestUtils.assertMemoryLeak(() -> { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopBlockedSendCloseTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopBlockedSendCloseTest.java new file mode 100644 index 00000000..20039fa5 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopBlockedSendCloseTest.java @@ -0,0 +1,263 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.DefaultHttpClientConfiguration; +import io.questdb.client.cutlass.http.client.WebSocketClient; +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; +import io.questdb.client.network.PlainSocketFactory; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +public class CursorWebSocketSendLoopBlockedSendCloseTest { + + @Test(timeout = 30_000L) + public void testCloseBreaksBlockedSendBeforeJoiningWorker() throws Exception { + TestUtils.assertMemoryLeak(() -> { + BlockingSendClient client = new BlockingSendClient(true); + CursorSendEngine engine = new CursorSendEngine(null, 64 * 1024); + CursorWebSocketSendLoop loop = new CursorWebSocketSendLoop( + client, + engine, + 0L, + CursorWebSocketSendLoop.DEFAULT_PARK_NANOS, + null, + 100L, + 1_000L, + 5_000L, + false + ); + long payload = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + Thread closer = null; + AtomicReference closeFailure = new AtomicReference<>(); + try { + Unsafe.getUnsafe().putLong(payload, 0x0102030405060708L); + Unsafe.getUnsafe().putLong(payload + 8, 0x1112131415161718L); + Assert.assertEquals(0L, engine.appendBlocking(payload, 16)); + loop.start(); + Assert.assertTrue("I/O worker never entered the blocking send", + client.sendEntered.await(5, TimeUnit.SECONDS)); + + closer = new Thread(() -> { + try { + loop.close(); + } catch (Throwable t) { + closeFailure.set(t); + } + }, "blocked-send-closer"); + closer.start(); + + Assert.assertTrue("close did not break the traffic path before joining", + client.trafficClosed.await(5, TimeUnit.SECONDS)); + Assert.assertEquals("traffic path must close exactly once", 1, client.trafficCloseCount.get()); + Assert.assertEquals("the closer must break traffic, not the I/O worker", + closer, client.trafficCloseThread.get()); + Assert.assertTrue("blocked send did not observe traffic-path closure", + client.sendExited.await(5, TimeUnit.SECONDS)); + + closer.join(TimeUnit.SECONDS.toMillis(5)); + Assert.assertFalse("close did not join the I/O worker", closer.isAlive()); + Assert.assertNull("close failed", closeFailure.get()); + Assert.assertNull("ordinary close must not manufacture a terminal error", loop.getTerminalError()); + + Thread ioThread = client.sendThread.get(); + Assert.assertNotNull(ioThread); + // close() returns when the worker counts shutdownLatch down -- + // the worker's last action before its exit tail -- so the + // thread can be observably alive for a scheduling beat after + // close() returns. Quiescence is the latch plus the cleanup + // asserts below (cleanup runs before the countdown), not + // thread death: join briefly instead of asserting the race. + ioThread.join(TimeUnit.SECONDS.toMillis(5)); + Assert.assertFalse("I/O worker did not exit after close returned", ioThread.isAlive()); + Assert.assertEquals("full client cleanup must run exactly once", 1, client.cleanupCount.get()); + Assert.assertEquals("the I/O worker must own cleanup before publishing exit", + ioThread, client.cleanupThread.get()); + Assert.assertFalse("close must not manufacture caller interruption", + closer.isInterrupted()); + } finally { + client.releaseSend.countDown(); + if (closer != null) { + closer.join(TimeUnit.SECONDS.toMillis(5)); + } + loop.close(); + engine.close(); + client.close(); + Unsafe.free(payload, 16, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + @Test(timeout = 30_000L) + public void testUnsupportedCustomTransportFailsWithoutDestroyingWorkerResources() throws Exception { + TestUtils.assertMemoryLeak(() -> { + BlockingSendClient client = new BlockingSendClient(false); + CursorSendEngine engine = new CursorSendEngine(null, 64 * 1024); + CursorWebSocketSendLoop loop = new CursorWebSocketSendLoop( + client, + engine, + 0L, + CursorWebSocketSendLoop.DEFAULT_PARK_NANOS, + null, + 100L, + 1_000L, + 5_000L, + false + ); + long payload = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + Thread closer = null; + AtomicReference closeFailure = new AtomicReference<>(); + CountDownLatch closeEntered = new CountDownLatch(1); + try { + Unsafe.getUnsafe().putLong(payload, 0x0102030405060708L); + Unsafe.getUnsafe().putLong(payload + 8, 0x1112131415161718L); + Assert.assertEquals(0L, engine.appendBlocking(payload, 16)); + loop.start(); + Assert.assertTrue("I/O worker never entered the blocking send", + client.sendEntered.await(5, TimeUnit.SECONDS)); + + closer = new Thread(() -> { + closeEntered.countDown(); + try { + loop.close(); + } catch (Throwable t) { + closeFailure.set(t); + } + }, "unsupported-transport-closer"); + closer.start(); + + Assert.assertTrue("close did not start", closeEntered.await(5, TimeUnit.SECONDS)); + closer.join(TimeUnit.SECONDS.toMillis(5)); + Assert.assertFalse("unsupported transport made close join indefinitely", closer.isAlive()); + Assert.assertTrue(closeFailure.get() instanceof LineSenderException); + Assert.assertTrue(closeFailure.get().getCause() instanceof UnsupportedOperationException); + Assert.assertEquals("unsupported cancellation must not release the blocked send", + 1L, client.sendExited.getCount()); + Assert.assertEquals("unsupported cancellation must not perform full cleanup", 0, client.cleanupCount.get()); + Assert.assertTrue("worker must retain resource ownership", client.sendThread.get().isAlive()); + + client.releaseSend.countDown(); + Assert.assertTrue("released send did not exit", client.sendExited.await(5, TimeUnit.SECONDS)); + Assert.assertTrue("worker did not complete delegated cleanup", client.cleanupDone.await(5, TimeUnit.SECONDS)); + client.sendThread.get().join(TimeUnit.SECONDS.toMillis(5)); + Assert.assertFalse("worker lingered after the custom transport was released", client.sendThread.get().isAlive()); + Assert.assertEquals(1, client.cleanupCount.get()); + Assert.assertNull("ordinary worker exit must remain non-terminal", loop.getTerminalError()); + } finally { + client.releaseSend.countDown(); + if (closer != null) { + closer.join(TimeUnit.SECONDS.toMillis(5)); + } + Thread ioThread = client.sendThread.get(); + if (ioThread != null) { + ioThread.join(TimeUnit.SECONDS.toMillis(5)); + } + loop.close(); + engine.close(); + client.close(); + Unsafe.free(payload, 16, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + private static final class BlockingSendClient extends WebSocketClient { + private final AtomicBoolean cleanupClaimed = new AtomicBoolean(); + private final AtomicInteger cleanupCount = new AtomicInteger(); + private final CountDownLatch cleanupDone = new CountDownLatch(1); + private final AtomicReference cleanupThread = new AtomicReference<>(); + private final boolean trafficShutdownSupported; + private final CountDownLatch releaseSend = new CountDownLatch(1); + private final CountDownLatch sendEntered = new CountDownLatch(1); + private final CountDownLatch sendExited = new CountDownLatch(1); + private final AtomicReference sendThread = new AtomicReference<>(); + private final AtomicInteger trafficCloseCount = new AtomicInteger(); + private final AtomicReference trafficCloseThread = new AtomicReference<>(); + private final CountDownLatch trafficClosed = new CountDownLatch(1); + + private BlockingSendClient(boolean trafficShutdownSupported) { + super(DefaultHttpClientConfiguration.INSTANCE, PlainSocketFactory.INSTANCE); + this.trafficShutdownSupported = trafficShutdownSupported; + } + + @Override + public void close() { + if (cleanupClaimed.compareAndSet(false, true)) { + cleanupCount.incrementAndGet(); + cleanupThread.set(Thread.currentThread()); + cleanupDone.countDown(); + } + super.close(); + } + + @Override + public void closeTraffic() { + if (!trafficShutdownSupported) { + throw new UnsupportedOperationException("custom transport has no safe cancellation capability"); + } + trafficCloseThread.compareAndSet(null, Thread.currentThread()); + trafficCloseCount.incrementAndGet(); + trafficClosed.countDown(); + releaseSend.countDown(); + } + + @Override + public void sendBinary(long dataPtr, int length, int timeout) { + sendThread.set(Thread.currentThread()); + sendEntered.countDown(); + try { + if (!releaseSend.await(10, TimeUnit.SECONDS)) { + throw new AssertionError("traffic path was not closed"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError("I/O worker was interrupted instead of traffic being closed", e); + } finally { + sendExited.countDown(); + } + throw new LineSenderException("traffic path closed"); + } + + @Override + protected void ioWait(int timeout, int op) { + throw new UnsupportedOperationException("stub: no socket"); + } + + @Override + protected void setupIoWait() { + // no-op + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopConnectPhaseCloseTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopConnectPhaseCloseTest.java new file mode 100644 index 00000000..0225d071 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopConnectPhaseCloseTest.java @@ -0,0 +1,469 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.DefaultHttpClientConfiguration; +import io.questdb.client.cutlass.http.client.WebSocketClient; +import io.questdb.client.cutlass.http.client.WebSocketFrameHandler; +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; +import io.questdb.client.network.PlainSocketFactory; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +/** + * TRUE-CANCELLATION proof (P1): {@code close()} must cancel a connect attempt + * that is blocked inside {@code reconnectFactory.reconnect(...)} (a black-holed + * native connect that neither unpark nor interrupt can cancel). The in-flight + * {@link WebSocketClient} is walk-local -- it is NOT the loop's {@code client} + * field (that field is {@code null} on the async-initial connect and points at + * the stale pre-drop client on a mid-flight reconnect) -- so before the fix + * {@code close()}'s field-client {@code closeTraffic()} could not reach it and + * {@code close()} blocked on the untimed {@code shutdownLatch.await()} for the + * whole OS SYN-retry window (~60-130s per endpoint). + *

      + * The fix publishes the in-flight client to a race-safe + * {@link CursorWebSocketSendLoop.ConnectCancellation} handle before the + * blocking connect, and {@code close()} breaks that client's traffic. These + * tests assert TRUE cancellation, not just return: the fake in-flight client's + * {@code closeTraffic()} is the ONLY thing that unblocks the parked + * {@code reconnect()}, and the tests witness that {@code closeTraffic()} was + * invoked (exactly once, by the closer thread) AND that {@code close()} + * returned. Deterministic latches only; {@code @Test(timeout=...)} fails a + * regression fast. + */ +public class CursorWebSocketSendLoopConnectPhaseCloseTest { + + /** + * Generous budget: with the fix, close() breaks the in-flight connect and + * returns well within this. Without the fix, close() blocks on the untimed + * shutdown latch for the entire (simulated) connect and this budget lapses. + */ + private static final long CLOSE_BUDGET_MILLIS = 5_000L; + + /** + * Small, injectable bounded-await backstop for the TOCTOU test. Shrunk from + * the production {@code DEFAULT_CLOSE_SHUTDOWN_AWAIT_MILLIS} (30 s) via + * {@link CursorWebSocketSendLoop#setShutdownAwaitTimeoutMillis(long)} so the + * timeout branch fires fast and deterministically -- no multi-second real + * wait -- while still leaving CLOSE_BUDGET_MILLIS of slack for the closer + * thread to return. + */ + private static final long BACKSTOP_MILLIS = 500L; + + /** + * Async-initial-connect path: the loop is built with a {@code null} client + * and a reconnect factory, so the I/O thread drives the very first connect + * through connectLoop while the {@code client} field stays {@code null}. + */ + @Test(timeout = 30_000L) + public void testCloseCancelsAsyncInitialConnectViaInFlightClient() throws Exception { + runConnectCancelledByCloseTraffic(false); + } + + /** + * Mid-flight-reconnect path: an initial client is installed, its first + * receive fails, so connectLoop reconnects and blocks. The {@code client} + * field then points at the stale pre-drop client, never the in-flight one. + */ + @Test(timeout = 30_000L) + public void testCloseCancelsMidFlightReconnectViaInFlightClient() throws Exception { + runConnectCancelledByCloseTraffic(true); + } + + /** + * BACKSTOP (bounded-await) proof: models the pathological, uninterruptible + * TOCTOU case where {@code cancel()}'s {@code closeTraffic()} CANNOT break + * the in-flight connect (as if cancellation landed after the pre-connect + * guard but before native fd creation, making closeTraffic() a no-op). The + * connect stays parked, so round-2 cancellation does NOT release the latch. + * Asserts {@code close()} STILL returns within the bounded backstop (does + * not hang) and surfaces the failed-stop contract: a loud + * {@link LineSenderException} whose message names the stalled I/O thread and + * the timeout, with client/engine teardown delegated to the I/O thread's + * own exit path (no destructive close under the still-live worker). + */ + @Test(timeout = 30_000L) + public void testCloseReturnsBoundedWhenCancellationCannotBreakConnect() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final AtomicReference published = new AtomicReference<>(); + final CountDownLatch connectEntered = new CountDownLatch(1); + // The ONLY thing that releases the parked connect. closeTraffic() + // deliberately does NOT touch it, modelling a cancel that cannot + // break the connect; the test itself counts it down, AFTER the + // backstop assertions, so the worker can unwind for the leak check. + final CountDownLatch release = new CountDownLatch(1); + + final CursorWebSocketSendLoop.ReconnectFactory factory = new CursorWebSocketSendLoop.ReconnectFactory() { + @Override + public WebSocketClient reconnect() throws Exception { + return reconnect(null); + } + + @Override + public WebSocketClient reconnect(CursorWebSocketSendLoop.ConnectCancellation cancellation) { + UninterruptibleInFlightClient c = new UninterruptibleInFlightClient(release); + published.set(c); + if (cancellation != null) { + cancellation.publish(c); + } + connectEntered.countDown(); + // Uninterruptible AND uncancellable: closeTraffic() does not + // release this park; only the test's release latch does. + c.awaitRelease(); + c.close(); + throw new LineSenderException("connect ended after release"); + } + }; + + final CursorSendEngine engine = new CursorSendEngine(null, 64 * 1024); + final CursorWebSocketSendLoop loop = new CursorWebSocketSendLoop( + null, + engine, + 0L, + CursorWebSocketSendLoop.DEFAULT_PARK_NANOS, + factory, + /* reconnectMaxDurationMillis */ 60_000L, + /* reconnectInitialBackoffMillis */ 1_000L, + /* reconnectMaxBackoffMillis */ 5_000L, + false + ); + // Shrink the bounded-await backstop so the timeout branch fires fast + // (no multi-second real wait); production uses the 30s default. + loop.setShutdownAwaitTimeoutMillis(BACKSTOP_MILLIS); + + final AtomicReference closeFailure = new AtomicReference<>(); + Thread closer = null; + try { + loop.start(); + Assert.assertTrue("I/O worker never entered the blocking connect", + connectEntered.await(5, TimeUnit.SECONDS)); + + closer = new Thread(() -> { + try { + loop.close(); + } catch (Throwable t) { + closeFailure.set(t); + } + }, "backstop-closer"); + closer.start(); + + closer.join(CLOSE_BUDGET_MILLIS); + final boolean closedInTime = !closer.isAlive(); + + final UninterruptibleInFlightClient inFlight = published.get(); + Assert.assertNotNull("no in-flight client was ever published", inFlight); + // cancel() DID attempt to break the connect -- but closeTraffic() + // is a no-op here, so the connect is still parked. + Assert.assertTrue("close() must have attempted to cancel the in-flight connect", + inFlight.trafficCloseCount.get() >= 1); + Assert.assertEquals("the parked connect must NOT have been released by closeTraffic()", + 1L, release.getCount()); + + // Decisive: close() returns within the bounded backstop even + // though cancellation could not break the connect. + Assert.assertTrue( + "close() must return within the bounded backstop (" + CLOSE_BUDGET_MILLIS + + "ms) even when cancellation cannot break the connect; instead it hung", + closedInTime); + + // Failed-stop contract: loud LineSenderException naming the + // stalled thread and the timeout. + final Throwable failure = closeFailure.get(); + Assert.assertNotNull("close() must loud-fail when the backstop times out", failure); + Assert.assertTrue("failed stop must be a LineSenderException, was " + failure, + failure instanceof LineSenderException); + Assert.assertTrue("message must name the stalled I/O thread: " + failure.getMessage(), + failure.getMessage().contains("cursor I/O thread did not stop")); + Assert.assertTrue("message must attribute the failed stop to the backstop timeout: " + + failure.getMessage(), + failure.getMessage().contains("timed out")); + // Teardown is delegated to the I/O thread's exit path, not done + // destructively under the live worker: no terminal manufactured. + Assert.assertNull("backstop timeout must not manufacture a terminal error", + loop.getTerminalError()); + } finally { + // Now let the parked connect unwind so the worker exits and the + // leak check sees a clean teardown. Restore a generous backstop + // so the reconciling close() below awaits the worker's exit. + loop.setShutdownAwaitTimeoutMillis(CLOSE_BUDGET_MILLIS); + release.countDown(); + if (closer != null) { + closer.join(TimeUnit.SECONDS.toMillis(5)); + } + loop.close(); + engine.close(); + } + }); + } + + private void runConnectCancelledByCloseTraffic(boolean withInitialClient) throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Allocate the initial client INSIDE the leak-checked block so its + // native buffers are inside the baseline, not counted as over-free. + final WebSocketClient initialClient = withInitialClient ? new FailingInitialClient() : null; + final AtomicReference published = new AtomicReference<>(); + final CountDownLatch connectEntered = new CountDownLatch(1); + + // A reconnect factory that models the real connect walk: it creates + // the client it is about to block on, PUBLISHES it to the loop's + // cancellation handle before the blocking connect, then parks in a + // way that ONLY closeTraffic() (i.e. cancellation) can release -- + // exactly a black-holed native connect that unpark/interrupt cannot + // cancel. + final CursorWebSocketSendLoop.ReconnectFactory factory = new CursorWebSocketSendLoop.ReconnectFactory() { + @Override + public WebSocketClient reconnect() throws Exception { + return reconnect(null); + } + + @Override + public WebSocketClient reconnect(CursorWebSocketSendLoop.ConnectCancellation cancellation) { + InFlightConnectClient c = new InFlightConnectClient(); + published.set(c); + if (cancellation != null) { + cancellation.publish(c); + } + connectEntered.countDown(); + // Black-holed connect: returns only once closeTraffic() breaks it. + c.awaitTrafficBreak(); + // Cancelled: dispose the half-built client (frees native + // buffers) and surface a transport error, exactly as the + // real connect walk's catch does on a broken connect. + c.close(); + throw new LineSenderException("connect cancelled by closeTraffic()"); + } + }; + + final CursorSendEngine engine = new CursorSendEngine(null, 64 * 1024); + final CursorWebSocketSendLoop loop = new CursorWebSocketSendLoop( + initialClient, + engine, + 0L, + CursorWebSocketSendLoop.DEFAULT_PARK_NANOS, + factory, + /* reconnectMaxDurationMillis */ 60_000L, + /* reconnectInitialBackoffMillis */ 1_000L, + /* reconnectMaxBackoffMillis */ 5_000L, + false + ); + + final AtomicReference closeFailure = new AtomicReference<>(); + Thread closer = null; + try { + loop.start(); + Assert.assertTrue("I/O worker never entered the blocking connect", + connectEntered.await(5, TimeUnit.SECONDS)); + + closer = new Thread(() -> { + try { + loop.close(); + } catch (Throwable t) { + closeFailure.set(t); + } + }, "connect-phase-closer"); + closer.start(); + + closer.join(CLOSE_BUDGET_MILLIS); + final boolean closedInTime = !closer.isAlive(); + + final InFlightConnectClient inFlight = published.get(); + Assert.assertNotNull("no in-flight client was ever published to the cancellation handle", + inFlight); + + // Let the worker finish unwinding regardless of outcome so the + // memory-leak check sees a clean teardown. + inFlight.trafficBroken.countDown(); + closer.join(TimeUnit.SECONDS.toMillis(10)); + + Assert.assertNull("close() must not fail", closeFailure.get()); + Assert.assertTrue( + "close() must cancel the connect blocked inside reconnect() and return " + + "within " + CLOSE_BUDGET_MILLIS + "ms; instead it blocked on the " + + "untimed shutdown latch for the whole connect", + closedInTime); + // Decisive: closeTraffic() on the IN-FLIGHT client is what + // unblocked the parked connect -- and it was the closer thread, + // not the I/O worker, that broke it. + Assert.assertEquals( + "close() must break the in-flight connect's traffic exactly once", + 1, inFlight.trafficCloseCount.get()); + Assert.assertEquals( + "the closer thread (not the I/O worker) must cancel the in-flight connect", + closer, inFlight.trafficCloseThread.get()); + Assert.assertNull("ordinary connect-phase close must not manufacture a terminal error", + loop.getTerminalError()); + } finally { + InFlightConnectClient inFlight = published.get(); + if (inFlight != null) { + inFlight.trafficBroken.countDown(); + } + if (closer != null) { + closer.join(TimeUnit.SECONDS.toMillis(5)); + } + loop.close(); + engine.close(); + if (initialClient != null) { + initialClient.close(); + } + } + }); + } + + /** + * The fake in-flight client. It never really connects: the factory parks in + * {@link #awaitTrafficBreak()} until {@link #closeTraffic()} releases it, + * which is the ONLY path out -- neither unpark nor interrupt can cancel it. + */ + private static final class InFlightConnectClient extends WebSocketClient { + final CountDownLatch trafficBroken = new CountDownLatch(1); + final AtomicInteger trafficCloseCount = new AtomicInteger(); + final AtomicReference trafficCloseThread = new AtomicReference<>(); + + private InFlightConnectClient() { + super(DefaultHttpClientConfiguration.INSTANCE, PlainSocketFactory.INSTANCE); + } + + @Override + public void closeTraffic() { + trafficCloseThread.compareAndSet(null, Thread.currentThread()); + trafficCloseCount.incrementAndGet(); + trafficBroken.countDown(); + } + + void awaitTrafficBreak() { + // Uninterruptible: only closeTraffic()'s countDown may release us, + // faithfully modelling a native connect that unpark/interrupt cannot + // cancel. + boolean interrupted = false; + while (trafficBroken.getCount() != 0L) { + try { + trafficBroken.await(); + } catch (InterruptedException e) { + interrupted = true; + } + } + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + + @Override + protected void ioWait(int timeout, int op) { + throw new UnsupportedOperationException("stub: no socket"); + } + + @Override + protected void setupIoWait() { + // no-op + } + } + + /** + * The fake in-flight client for the BACKSTOP test. Its {@link #closeTraffic()} + * is a no-op w.r.t. releasing the park -- it only records that cancellation + * was attempted -- so a {@code close()}->{@code cancel()} CANNOT unblock the + * parked connect. This models the pathological uninterruptible/TOCTOU case; + * the ONLY release is the test-owned {@code release} latch, counted down + * AFTER the backstop assertions. + */ + private static final class UninterruptibleInFlightClient extends WebSocketClient { + final AtomicInteger trafficCloseCount = new AtomicInteger(); + private final CountDownLatch release; + + private UninterruptibleInFlightClient(CountDownLatch release) { + super(DefaultHttpClientConfiguration.INSTANCE, PlainSocketFactory.INSTANCE); + this.release = release; + } + + @Override + public void closeTraffic() { + // Records the cancel attempt but does NOT release the parked + // connect: exactly a closeTraffic() that lands before the native fd + // exists (a no-op), so the connect blocks on regardless. + trafficCloseCount.incrementAndGet(); + } + + void awaitRelease() { + // Uninterruptible: neither unpark, interrupt, nor closeTraffic() + // releases us -- only the test's release latch. + boolean interrupted = false; + while (release.getCount() != 0L) { + try { + release.await(); + } catch (InterruptedException e) { + interrupted = true; + } + } + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + + @Override + protected void ioWait(int timeout, int op) { + throw new UnsupportedOperationException("stub: no socket"); + } + + @Override + protected void setupIoWait() { + // no-op + } + } + + /** + * A pre-installed client whose first receive fails, driving the I/O loop + * into a reconnect so the in-flight client is published while the + * {@code client} field still points here (the mid-flight-reconnect path). + */ + private static final class FailingInitialClient extends WebSocketClient { + + private FailingInitialClient() { + super(DefaultHttpClientConfiguration.INSTANCE, PlainSocketFactory.INSTANCE); + } + + @Override + public boolean tryReceiveFrame(WebSocketFrameHandler handler) { + throw new LineSenderException("initial wire dropped"); + } + + @Override + protected void ioWait(int timeout, int op) { + throw new UnsupportedOperationException("stub: no socket"); + } + + @Override + protected void setupIoWait() { + // no-op + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EngineClosePublishAfterFlockReleaseTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EngineClosePublishAfterFlockReleaseTest.java new file mode 100644 index 00000000..81fd36ef --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EngineClosePublishAfterFlockReleaseTest.java @@ -0,0 +1,247 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SlotLock; +import io.questdb.client.std.Files; +import io.questdb.client.test.tools.TestUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.nio.file.Paths; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * Regression test for the completion/release ordering in + * {@code CursorSendEngine.finishClose()}: {@code closeCompleted} must be + * published strictly AFTER the slot flock release is confirmed, never + * before. + * + *

      The bug being pinned: {@code closeCompleted = true} used to be written + * before {@code slotLock} was closed. A pool thread could observe completion + * through {@code QwpWebSocketSender.isSlotLockReleased()}, free the slot + * index in {@code SenderPool.reprobeRetiredSlots()}, and admit a replacement + * sender whose {@code SlotLock.acquire} then collided with the still-open + * flock fd — a spurious "sf slot already in use" construction failure naming + * the process's own pid as the holder. + * + *

      The window between the publish and the {@code Files.close(fd)} is + * microseconds wide in the wild; {@code setBeforeFlockReleaseHook} makes it + * deterministic. The closing thread is parked between terminal cleanup and + * the flock release, and the test asserts from outside the window's two + * halves of the contract: + *

        + *
      • inside the window: {@code isCloseCompleted()} is still false AND a + * fresh {@code SlotLock.acquire} on the slot fails (proving the flock + * is genuinely held — i.e. reporting completion here would have been + * a lie);
        • + *
      • after the window: {@code isCloseCompleted()} is true AND a fresh + * {@code SlotLock.acquire} succeeds (completion still implies + * reusability — the reorder did not break the happy path).
        • + *
      + */ +public class EngineClosePublishAfterFlockReleaseTest { + + private String sfDir; + + @Before + public void setUp() { + sfDir = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-engine-close-publish-order-" + System.nanoTime()).toString(); + } + + @After + public void tearDown() { + if (sfDir == null) return; + long find = Files.findFirst(sfDir); + if (find > 0) { + try { + int rc = 1; + while (rc > 0) { + String name = Files.utf8ToString(Files.findName(find)); + if (name != null && !".".equals(name) && !"..".equals(name)) { + Files.remove(sfDir + "/" + name); + } + rc = Files.findNext(find); + } + } finally { + Files.findClose(find); + } + } + Files.remove(sfDir); + } + + @Test(timeout = 30_000L) + public void testCloseCompletedPublishedOnlyAfterConfirmedFlockRelease() throws Exception { + TestUtils.assertMemoryLeak(() -> { + CursorSendEngine engine = new CursorSendEngine(sfDir, 4L * 1024 * 1024); + CountDownLatch inWindow = new CountDownLatch(1); + CountDownLatch proceed = new CountDownLatch(1); + engine.setBeforeFlockReleaseHook(() -> { + inWindow.countDown(); + try { + // Bounded so a failed assertion on the main thread can + // never wedge the closer past the test timeout. + proceed.await(20, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + + AtomicReference closerError = new AtomicReference<>(); + Thread closer = new Thread(() -> { + try { + engine.close(); + } catch (Throwable t) { + closerError.set(t); + } + }, "engine-closer"); + closer.start(); + + try { + assertTrue("closer thread never reached the cleanup/release window", + inWindow.await(10, TimeUnit.SECONDS)); + + // Inside the window: terminal cleanup has run, the flock has + // NOT been released. Completion must not be observable yet — + // this is the exact read a pool thread performs before + // freeing the slot index. + assertFalse("closeCompleted was published before the flock release; " + + "a pool observing this would free the slot index and a " + + "replacement sender would collide with the still-held flock", + engine.isCloseCompleted()); + + // Prove the window is real: the flock is genuinely still + // held, so acquisition by a "replacement engine" fails. + try { + SlotLock probe = SlotLock.acquire(sfDir); + probe.close(); + fail("scaffolding error: expected the slot flock to still be " + + "held inside the pre-release window, but a fresh " + + "SlotLock.acquire succeeded"); + } catch (IllegalStateException expected) { + // good — slot is locked, which is why completion must + // not have been published yet. + } + } finally { + proceed.countDown(); + closer.join(10_000L); + } + assertFalse("closer thread did not finish", closer.isAlive()); + assertNull("engine.close() threw", closerError.get()); + + // After the window: the release is confirmed, so completion must + // now be latched, and completion must still imply reusability. + assertTrue("closeCompleted must latch once the flock release is confirmed", + engine.isCloseCompleted()); + try (SlotLock ignored = SlotLock.acquire(sfDir)) { + // good — completion implies the slot dir is acquirable. + } catch (IllegalStateException stillHeld) { + fail("closeCompleted reported true but the slot flock is still held: " + + stillHeld.getMessage()); + } + }); + } + + /** + * The error half of the publish-after-release contract: when + * {@code SlotLock.release()} reports failure (the OS refused the explicit + * unlock, so the flock may still be held), {@code closeCompleted} must + * stay false until a later close confirms release. A pool observing + * {@code isCloseCompleted() == false} keeps the slot retired while the + * flock is genuinely held, then recovers it once the retry completes. + *

      + * The lock's fd is swapped to a known-bad descriptor for a deterministic + * native unlock failure, then restored before retrying through the engine's + * public close API. + */ + @Test(timeout = 30_000L) + public void testUnconfirmedFlockReleaseKeepsCloseIncompleteUntilRetry() throws Exception { + TestUtils.assertMemoryLeak(() -> { + CursorSendEngine engine = new CursorSendEngine(sfDir, 4L * 1024 * 1024); + SlotLock slotLock = engine.getSlotLockForTesting(); + assertNotNull("disk-mode engine must hold a slot lock", slotLock); + + // This test owns the explicit close() retry. Prevent the shared + // asynchronous retry driver from racing it and from surviving into + // another test class; FlockReleaseRetryDriverTest covers the real + // driver lifecycle separately. A start failure synchronously clears + // the shared retry queue and resets the engine's scheduling flag. + CursorSendEngine.setFlockReleaseRetryThreadFactory(task -> { + throw new IllegalStateException("retry driver disabled for explicit close retry test"); + }); + try { + try (SlotLock.ReleaseFailureForTesting releaseFailure = + slotLock.injectReleaseFailureForTesting()) { + engine.close(); + assertFalse( + "closeCompleted was published despite an unconfirmed flock release; " + + "a pool observing this would free the slot index while the " + + "flock fd is still open", + engine.isCloseCompleted()); + // The REAL flock is still held — the incomplete report is true. + try { + SlotLock probe = SlotLock.acquire(sfDir); + probe.close(); + fail("slot must not be acquirable while the original flock fd is still open"); + } catch (IllegalStateException expected) { + // good — incomplete close really means "still locked". + } + + // Remove the injected fault. The retry must only re-attempt + // the retained fd release: ring/watermark cleanup stays + // exactly-once, while completion and slot reusability recover. + releaseFailure.close(); + engine.close(); + assertTrue("retried close() must complete after the flock release succeeds", + engine.isCloseCompleted()); + try (SlotLock ignored = SlotLock.acquire(sfDir)) { + // good — eventual completion implies reusable capacity. + } + } + } finally { + try { + // The guard has restored the real fd on every exit. Release + // it if an assertion failed before the successful retry. + if (!engine.isCloseCompleted()) { + assertTrue("restored fd must release cleanly", slotLock.release()); + } + } finally { + CursorSendEngine.setFlockReleaseRetryThreadFactory(null); + } + } + }); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/FlockReleaseRetryDriverTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/FlockReleaseRetryDriverTest.java new file mode 100644 index 00000000..967a4fab --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/FlockReleaseRetryDriverTest.java @@ -0,0 +1,482 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SlotLock; +import io.questdb.client.std.Files; +import io.questdb.client.test.tools.TestUtils; +import org.junit.After; +import org.junit.Test; + +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class FlockReleaseRetryDriverTest { + + private final List sfDirs = new ArrayList<>(); + + @After + public void tearDown() { + CursorSendEngine.setAfterFlockReleaseRetryFailureHook(null); + CursorSendEngine.setFlockReleaseRetryParkOverride(null); + CursorSendEngine.setFlockReleaseRetryThreadFactory(null); + for (String sfDir : sfDirs) { + removeDir(sfDir); + } + } + + /** + * Driver-start failure must not strand retired capacity until process + * exit. {@code Sender.close()} is a one-shot no-op by contract, so the + * only recovery surface a pool has is its retired-slot probe + * ({@code isSlotLockReleased()}, called from the housekeeper tick and + * capacity-starved borrows) — that probe must re-arm the shared retry + * driver once thread creation works again. + */ + @Test(timeout = 30_000L) + public void testDriverStartFailureRecoversViaPoolProbe() throws Exception { + TestUtils.assertMemoryLeak(() -> { + AtomicInteger starts = new AtomicInteger(); + CursorSendEngine.setFlockReleaseRetryThreadFactory(task -> new Thread(task) { + @Override + public synchronized void start() { + starts.incrementAndGet(); + throw new IllegalStateException("injected start failure"); + } + }); + + CursorSendEngine engine = new CursorSendEngine( + newSfDir("probe-rearm"), 4L * 1024 * 1024); + SlotLock slotLock = engine.getSlotLockForTesting(); + QwpWebSocketSender sender = QwpWebSocketSender.createForTesting("localhost", 1); + sender.setCursorEngine(engine, true); + AtomicReference rearmedDriver = new AtomicReference<>(); + boolean recovered = false; + SlotLock.ReleaseFailureForTesting releaseFailure = slotLock.injectReleaseFailureForTesting(); + try { + sender.close(); + assertEquals("retry driver start must be attempted once", 1, starts.get()); + assertFalse("failed unlock must remain unpublished", engine.isCloseCompleted()); + + // Sender.close() is idempotent: repeat calls never reach the + // engine again, so they cannot restart the failed driver. + sender.close(); + assertEquals("no-op close must not retry the driver start", 1, starts.get()); + + // While the fault persists, each pool probe re-attempts the + // re-arm (and fails again) without publishing a release. + assertFalse("failed unlock must keep the slot reported as held", + sender.isSlotLockReleased()); + assertTrue("pool probe must re-attempt the driver start", + starts.get() >= 2); + + // The transient condition clears: thread creation works again + // (the failed start drained the queue, so the factory swap is + // legal) and the flock fd is back. + CursorSendEngine.setFlockReleaseRetryThreadFactory(task -> { + Thread thread = new Thread(task, "test-rearmed-flock-release-retry"); + rearmedDriver.set(thread); + return thread; + }); + releaseFailure.close(); + CountDownLatch released = new CountDownLatch(1); + engine.setSlotLockReleaseListener(released::countDown); + + // Production recovery surface: the pool re-probes the retired + // slot through isSlotLockReleased(). + sender.isSlotLockReleased(); + assertTrue("pool probe must re-arm the flock-release retry after " + + "a driver start failure", + released.await(10, TimeUnit.SECONDS)); + assertTrue(engine.isCloseCompleted()); + assertTrue("probe must expose the recovered release", + sender.isSlotLockReleased()); + recovered = true; + } finally { + Thread driver = rearmedDriver.get(); + if (driver != null) { + driver.join(10_000L); + } + if (!recovered) { + CursorSendEngine.setFlockReleaseRetryThreadFactory(null); + releaseFailure.close(); + if (!slotLock.release()) { + fail("restored flock fd did not release"); + } + } + } + CursorSendEngine.setFlockReleaseRetryThreadFactory(null); + }); + } + + @Test(timeout = 30_000L) + public void testPersistentFailuresShareOneRetryThread() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final int engineCount = 32; + AtomicInteger retryFailures = new AtomicInteger(); + AtomicInteger threadsCreated = new AtomicInteger(); + AtomicReference retryThreadRef = new AtomicReference<>(); + CountDownLatch retryFailuresObserved = new CountDownLatch(engineCount * 2); + CountDownLatch retryThreadStarted = new CountDownLatch(1); + CountDownLatch runRetryDriver = new CountDownLatch(1); + CursorSendEngine.setAfterFlockReleaseRetryFailureHook(() -> { + retryFailures.incrementAndGet(); + retryFailuresObserved.countDown(); + }); + CursorSendEngine.setFlockReleaseRetryThreadFactory(task -> { + threadsCreated.incrementAndGet(); + Thread thread = new Thread(() -> { + retryThreadStarted.countDown(); + try { + runRetryDriver.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + task.run(); + }, "test-shared-flock-release-retry"); + retryThreadRef.set(thread); + return thread; + }); + + List engines = new ArrayList<>(engineCount); + List releaseFailures = new ArrayList<>(engineCount); + boolean failuresRestored = false; + try { + for (int i = 0; i < engineCount; i++) { + String sfDir = newSfDir("persistent-" + i); + CursorSendEngine engine = new CursorSendEngine(sfDir, 4L * 1024 * 1024); + SlotLock slotLock = engine.getSlotLockForTesting(); + engines.add(engine); + boolean isTracked = false; + SlotLock.ReleaseFailureForTesting releaseFailure = + slotLock.injectReleaseFailureForTesting(); + try { + releaseFailures.add(releaseFailure); + isTracked = true; + engine.close(); + } finally { + if (!isTracked) { + releaseFailure.close(); + } + } + assertFalse("injected unlock failure must keep close incomplete", + engine.isCloseCompleted()); + if (i == 0) { + assertTrue("retry thread was not started", + retryThreadStarted.await(10, TimeUnit.SECONDS)); + } + } + + assertEquals("persistent failures must share one retry thread", + 1, threadsCreated.get()); + runRetryDriver.countDown(); + assertTrue("driver did not perform two failed rounds", + retryFailuresObserved.await(10, TimeUnit.SECONDS)); + assertTrue("driver did not retain persistent failures", + retryFailures.get() >= engineCount * 2); + for (CursorSendEngine engine : engines) { + assertFalse("failed releases must remain unpublished", + engine.isCloseCompleted()); + } + + restoreReleaseFailures(releaseFailures); + failuresRestored = true; + Thread retryThread = retryThreadRef.get(); + retryThread.join(10_000L); + assertFalse("shared retry thread retained lifecycle resources after drain", + retryThread.isAlive()); + for (CursorSendEngine engine : engines) { + assertTrue("driver must release every restored flock", + engine.isCloseCompleted()); + } + assertEquals("retries must not create another thread", + 1, threadsCreated.get()); + } finally { + if (!failuresRestored) { + restoreReleaseFailures(releaseFailures); + } + runRetryDriver.countDown(); + Thread retryThread = retryThreadRef.get(); + if (retryThread != null) { + retryThread.join(10_000L); + } + CursorSendEngine.setAfterFlockReleaseRetryFailureHook(null); + } + CursorSendEngine.setFlockReleaseRetryThreadFactory(null); + }); + } + + /** + * Schedule inspection for the shared driver's retry cadence: the + * inter-round park must grow exponentially from 100 ms and cap at 5 s, + * so a persistent unlock failure does not burn 10 syscalls per second + * per engine forever. The park override replaces the real park, so the + * test coordinates rounds without wall-clock waits. + */ + @Test(timeout = 30_000L) + public void testRetryBackoffDoublesToCap() throws Exception { + TestUtils.assertMemoryLeak(() -> { + List parks = new ArrayList<>(); + Semaphore parked = new Semaphore(0); + Semaphore proceed = new Semaphore(0); + AtomicReference driverRef = new AtomicReference<>(); + CursorSendEngine.setFlockReleaseRetryParkOverride(nanos -> { + parks.add(nanos); + parked.release(); + proceed.acquireUninterruptibly(); + }); + CursorSendEngine.setFlockReleaseRetryThreadFactory(task -> { + Thread thread = new Thread(task, "test-backoff-flock-release-retry"); + driverRef.set(thread); + return thread; + }); + + CursorSendEngine engine = new CursorSendEngine( + newSfDir("backoff-cap"), 4L * 1024 * 1024); + SlotLock slotLock = engine.getSlotLockForTesting(); + boolean failureRestored = false; + SlotLock.ReleaseFailureForTesting releaseFailure = slotLock.injectReleaseFailureForTesting(); + try { + engine.close(); + assertFalse("injected unlock failure must keep close incomplete", + engine.isCloseCompleted()); + + // Eight failed rounds: enough to observe the full ramp and + // two capped parks. + for (int round = 1; round <= 8; round++) { + assertTrue("driver did not park after failed round " + round, + parked.tryAcquire(10, TimeUnit.SECONDS)); + if (round < 8) { + proceed.release(); + } + } + + releaseFailure.close(); + failureRestored = true; + proceed.release(); + Thread driver = driverRef.get(); + driver.join(10_000L); + assertFalse("driver did not drain after the release succeeded", + driver.isAlive()); + assertTrue("restored flock must be released", engine.isCloseCompleted()); + + List expected = new ArrayList<>(); + expected.add(100_000_000L); + expected.add(200_000_000L); + expected.add(400_000_000L); + expected.add(800_000_000L); + expected.add(1_600_000_000L); + expected.add(3_200_000_000L); + expected.add(5_000_000_000L); + expected.add(5_000_000_000L); + assertEquals("retry parks must double from 100ms and cap at 5s", + expected, parks); + } finally { + if (!failureRestored) { + releaseFailure.close(); + } + proceed.release(1_000); + Thread driver = driverRef.get(); + if (driver != null) { + driver.join(10_000L); + } + } + CursorSendEngine.setFlockReleaseRetryParkOverride(null); + CursorSendEngine.setFlockReleaseRetryThreadFactory(null); + }); + } + + /** + * A successful release in a round is progress: the driver must reset its + * backoff to the base so the remaining engines are retried promptly + * while the failure condition is clearing. + */ + @Test(timeout = 30_000L) + public void testRetryBackoffResetsOnProgress() throws Exception { + TestUtils.assertMemoryLeak(() -> { + List parks = new ArrayList<>(); + Semaphore parked = new Semaphore(0); + Semaphore proceed = new Semaphore(0); + AtomicReference driverRef = new AtomicReference<>(); + CursorSendEngine.setFlockReleaseRetryParkOverride(nanos -> { + parks.add(nanos); + parked.release(); + proceed.acquireUninterruptibly(); + }); + CursorSendEngine.setFlockReleaseRetryThreadFactory(task -> { + Thread thread = new Thread(task, "test-reset-flock-release-retry"); + driverRef.set(thread); + return thread; + }); + + CursorSendEngine engineA = new CursorSendEngine( + newSfDir("backoff-reset-a"), 4L * 1024 * 1024); + CursorSendEngine engineB = new CursorSendEngine( + newSfDir("backoff-reset-b"), 4L * 1024 * 1024); + SlotLock slotLockA = engineA.getSlotLockForTesting(); + SlotLock slotLockB = engineB.getSlotLockForTesting(); + try { + try (SlotLock.ReleaseFailureForTesting releaseFailureA = + slotLockA.injectReleaseFailureForTesting(); + SlotLock.ReleaseFailureForTesting releaseFailureB = + slotLockB.injectReleaseFailureForTesting()) { + engineA.close(); + engineB.close(); + + // Three failed rounds ramp the backoff to 400ms. + for (int round = 1; round <= 3; round++) { + assertTrue("driver did not park after failed round " + round, + parked.tryAcquire(10, TimeUnit.SECONDS)); + if (round < 3) { + proceed.release(); + } + } + + // Engine A recovers; round 4 has one success and one failure, + // so its park must be back at the 100ms base. + releaseFailureA.close(); + proceed.release(); + assertTrue("driver did not park after the mixed round", + parked.tryAcquire(10, TimeUnit.SECONDS)); + assertTrue("recovered engine must publish completion", + engineA.isCloseCompleted()); + + releaseFailureB.close(); + proceed.release(); + Thread driver = driverRef.get(); + driver.join(10_000L); + assertFalse("driver did not drain after both releases succeeded", + driver.isAlive()); + assertTrue(engineB.isCloseCompleted()); + + List expected = new ArrayList<>(); + expected.add(100_000_000L); + expected.add(200_000_000L); + expected.add(400_000_000L); + expected.add(100_000_000L); + assertEquals("a successful release must reset the backoff to base", + expected, parks); + } + } finally { + proceed.release(1_000); + Thread driver = driverRef.get(); + if (driver != null) { + driver.join(10_000L); + } + } + CursorSendEngine.setFlockReleaseRetryParkOverride(null); + CursorSendEngine.setFlockReleaseRetryThreadFactory(null); + }); + } + + @Test(timeout = 30_000L) + public void testRetryThreadStartFailureLeavesExplicitCloseRetryable() throws Exception { + TestUtils.assertMemoryLeak(() -> { + AtomicInteger starts = new AtomicInteger(); + CursorSendEngine.setFlockReleaseRetryThreadFactory(task -> new Thread(task) { + @Override + public synchronized void start() { + starts.incrementAndGet(); + throw new IllegalStateException("injected start failure"); + } + }); + + CursorSendEngine engine = new CursorSendEngine( + newSfDir("start-failure"), 4L * 1024 * 1024); + SlotLock slotLock = engine.getSlotLockForTesting(); + SlotLock.ReleaseFailureForTesting releaseFailure = slotLock.injectReleaseFailureForTesting(); + try { + engine.close(); + assertEquals("retry driver start must be attempted once", 1, starts.get()); + assertFalse("failed unlock must remain unpublished", engine.isCloseCompleted()); + + // This also proves the failed driver cleared its queue: the + // setter rejects replacement while any engine remains queued. + CursorSendEngine.setFlockReleaseRetryThreadFactory(null); + releaseFailure.close(); + engine.close(); + assertTrue("explicit close must recover after retry-thread start failure", + engine.isCloseCompleted()); + } finally { + if (!engine.isCloseCompleted()) { + CursorSendEngine.setFlockReleaseRetryThreadFactory(null); + releaseFailure.close(); + if (!slotLock.release()) { + fail("restored flock fd did not release"); + } + } + } + }); + } + + private static void removeDir(String sfDir) { + long find = Files.findFirst(sfDir); + if (find > 0) { + try { + int rc = 1; + while (rc > 0) { + String name = Files.utf8ToString(Files.findName(find)); + if (name != null && !".".equals(name) && !"..".equals(name)) { + Files.remove(sfDir + "/" + name); + } + rc = Files.findNext(find); + } + } finally { + Files.findClose(find); + } + } + Files.remove(sfDir); + } + + private static void restoreReleaseFailures( + List releaseFailures + ) { + for (int i = 0; i < releaseFailures.size(); i++) { + releaseFailures.get(i).close(); + } + } + + private String newSfDir(String suffix) { + String sfDir = Paths.get( + System.getProperty("java.io.tmpdir"), + "qdb-flock-release-retry-" + suffix + "-" + System.nanoTime() + ).toString(); + sfDirs.add(sfDir); + return sfDir; + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentRecoveryFaultTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentRecoveryFaultTest.java index 861f761c..fbe179e8 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentRecoveryFaultTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentRecoveryFaultTest.java @@ -25,6 +25,7 @@ package io.questdb.client.test.cutlass.qwp.client.sf.cursor; import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; +import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegmentCorruptionException; import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegmentException; import io.questdb.client.std.Files; import io.questdb.client.std.FilesFacade; @@ -36,61 +37,17 @@ import org.junit.Test; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; /** - * Regression guard for the recovery-time SIGBUS hazard in {@link MmapSegment}. - *

      - * On recovery, {@link MmapSegment#openExisting} maps a persisted {@code .sfa} - * to its stat length and scans frames straight out of the mapping. When a prior - * session left a sparse segment tail -- a truncate-based pre-allocation that - * never materialized the tail blocks, as happens on ZFS -- a read of an - * unbacked page raises the JVM's recoverable - * {@code InternalError("...unsafe memory access operation...")} (a translated - * SIGBUS). Recovery must treat that page as the boundary of recoverable data, - * keep every frame below it, and hand back a usable segment -- not let the - * error abort recovery of the whole slot (the reported ZFS-CI flake). - *

      - * These tests drive the production entry point ({@code openExisting}), - * not the private scan methods via reflection. That matters for two reasons: - *

        - *
      • It exercises the real recovery path end to end.
        • - *
      • On pre-21 JDKs the mmap-fault {@code InternalError} is delivered - * imprecisely ("a fault occurred in a recent unsafe memory access - * operation in compiled Java code") and escapes a reflective - * {@code Method.invoke} frame instead of being caught inside the scan -- - * so a reflection-based test spuriously fails on the shipping JDK 8/11/17 - * even though the direct-call production path catches it fine.
        • - *
      - * The fault-delivery mechanism the fix rests on was verified directly on the - * shipping/CI Java floor -- JDK 8 (Temurin 1.8.0_492) -- not merely inferred - * from the adjacent pre-21 LTS releases: the whole class passes there in both - * interpreter ({@code -Xint}) and JIT modes, HotSpot emits the exact pre-21 - * message above, and a direct {@code try/catch} catches the fault in - * interpreter, C1, and C2 modes. {@code isMmapAccessFault}'s shared - * {@code "unsafe memory access operation"} fragment matches that message while - * the JDK 21+-only needle it replaced does not -- the guard is live on JDK 8. - * The unbacked tail is produced portably by truncating the file down (dropping - * the tail blocks) and back up to the mapping size (leaving a sparse hole). A - * hole-faulting filesystem (ZFS) then faults on the read exactly as in - * production -- the case the fix must survive rather than fold the CRC through - * the native, JNI-side {@code Crc32c} where a SIGBUS is uncatchable and aborts - * the JVM. A hole-zero-filling filesystem (ext4) instead reads the hole back as - * zeroes, which fails the frame CRC; either way recovery must stop at the same - * boundary and recover the same frames. - *

      - * Fail-on-revert on any filesystem. The sparse-hole tests above only - * fault on ZFS: on ext4/xfs the within-EOF hole zero-fills, so the scan stops - * via the CRC-mismatch / bad-magic branch and they stay green even with the - * mmap-fault guard reverted -- no regression protection on the ext4/xfs CI - * runners. The two {@code MapPastEof} tests below close that gap portably. - * They truncate the file down (freeing the tail blocks) and hand - * {@code openExisting} a {@link FilesFacade} that reports the original, larger - * length, so the mapping extends past real end-of-file. A read of a page beyond - * real EOF raises SIGBUS on every filesystem -- the same catchable - * {@code InternalError} an unbacked ZFS page raises -- so they exercise the - * real fault path (and fail on revert) on ext4/xfs too, not only on ZFS. + * Regression guards for recovery through positioned file reads. Recovery must + * consume every byte it validates before mmap creation so sparse or unbacked + * pages never raise SIGBUS during validation. Sparse holes are ordinary zero-filled + * reads; negative reads, premature EOF, and file-size changes are operational + * errors that fail recovery closed without returning a live mapping or mutating + * the segment. */ public class MmapSegmentRecoveryFaultTest { @@ -125,8 +82,8 @@ public void testRecoveryKeepsFramesBeforeUnbackedTail() throws Exception { long boundary = writeSegment(path, 7L, new int[]{payloadLen}); assertEquals("frame must fill exactly one page", page, boundary); - // Drop the tail blocks, then re-extend logically so [page, SEGMENT_BYTES) - // is an unbacked hole under the recovery mapping. + // Drop the tail blocks, then re-extend logically so + // [page, SEGMENT_BYTES) is an unbacked hole in the persisted file. punchSparseTail(path, page); try (MmapSegment seg = MmapSegment.openExisting(path)) { @@ -138,15 +95,9 @@ public void testRecoveryKeepsFramesBeforeUnbackedTail() throws Exception { } /** - * The harder case: a frame whose 8-byte header sits on a backed page but - * whose payload reaches into the unbacked hole (a torn write leaves a real - * positive {@code payloadLen} with the payload spanning the boundary). The - * CRC fold therefore reads across the backed-to-unbacked edge. Recovery - * must reject that frame and keep the one below it -- and, crucially, must - * do so via {@code Unsafe} reads: the native, JNI-side {@code Crc32c} over - * an unbacked page raises a SIGBUS that HotSpot cannot translate, aborting - * the whole JVM (verified: an {@code hs_err} in - * {@code Java_io_questdb_client_std_Crc32c_update}). + * The harder sparse case: a frame header is backed but its payload reaches + * into a hole. Positioned reads return zeroes for the hole, so CRC + * validation rejects that frame without dereferencing the mapping. */ @Test public void testRecoverySurvivesPayloadReachingUnbackedPage() throws Exception { @@ -159,10 +110,16 @@ public void testRecoverySurvivesPayloadReachingUnbackedPage() throws Exception { // the backed->unbacked edge. final long frame2Offset = boundary - 16; final int payloadLen2 = (int) page; - final int payloadLen1 = (int) (frame2Offset - MmapSegment.HEADER_SIZE - MmapSegment.FRAME_HEADER_SIZE); + final int payloadLen1 = (int) ( + frame2Offset - MmapSegment.HEADER_SIZE - MmapSegment.FRAME_HEADER_SIZE + ); long used = writeSegment(path, 11L, new int[]{payloadLen1, payloadLen2}); - assertEquals("frame 2's header must end 8 bytes below the page boundary", boundary - 8, frame2Offset + MmapSegment.FRAME_HEADER_SIZE); + assertEquals( + "frame 2's header must end 8 bytes below the page boundary", + boundary - 8, + frame2Offset + MmapSegment.FRAME_HEADER_SIZE + ); assertTrue("frame 2 payload must reach past the boundary", used > boundary); punchSparseTail(path, boundary); @@ -178,18 +135,8 @@ public void testRecoverySurvivesPayloadReachingUnbackedPage() throws Exception { } /** - * M1 regression: the header block (magic/version/baseSeq) is read before - * {@code scanFrames}, so an unbacked page 0 faults ahead of the guarded - * scan. {@link MmapSegment#openExisting} must surface that as a - * {@link MmapSegmentException} -- the per-file signal {@code SegmentRing} - * catches to skip just this {@code .sfa} -- and never let the raw - * {@code InternalError} escape and abort recovery of every sibling segment. - *

      - * Portable across filesystems: on a hole-faulting FS (ZFS) the fault is - * converted to a {@code MmapSegmentException} in {@code openExisting}'s - * catch; on a hole-zero-filling FS (ext4) page 0 reads back as zeroes, so - * the magic check fails and throws {@code MmapSegmentException} directly. - * Either way the file is skippable, not fatal. + * A sparse page zero header is positively identified as corrupt from the + * bytes returned by positioned read and is therefore skippable per-file. */ @Test public void testUnbackedHeaderPageIsSkippableNotFatal() throws Exception { @@ -200,101 +147,172 @@ public void testUnbackedHeaderPageIsSkippableNotFatal() throws Exception { punchSparseTail(path, 0L); try { MmapSegment.openExisting(path).close(); - fail("expected MmapSegmentException for an unbacked header page"); + fail("expected corruption for a sparse zero header"); + } catch (MmapSegmentCorruptionException expected) { + // ok -- SegmentRing's narrow corruption catch skips just this + // file instead of aborting recovery of the whole slot. + } catch (MmapSegmentException unexpected) { + fail("expected quarantinable corruption subtype, got " + unexpected); + } + }); + } + + @Test + public void testLargeFrameRecoveryCrossesReadBuffer() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final String path = tmpDir + "/seg-large-frame.sfa"; + // Put frame 2's length field across the first 64 KiB boundary, + // then make its payload span several recovery-buffer refills. + final int firstPayloadLen = 64 * 1024 + - MmapSegment.HEADER_SIZE - MmapSegment.FRAME_HEADER_SIZE - 5; + final int largePayloadLen = 3 * 64 * 1024 + 17; + assertEquals(64 * 1024 - 5, + MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + firstPayloadLen); + long expectedEnd = writeSegment(path, 13L, new int[]{firstPayloadLen, largePayloadLen, 31}); + try (MmapSegment seg = MmapSegment.openExisting(path)) { + assertEquals(3L, seg.frameCount()); + assertEquals(expectedEnd, seg.publishedOffset()); + assertEquals(0L, seg.tornTailBytes()); + } + }); + } + + @Test + public void testReadErrorFailsClosedBeforeMmap() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final String path = tmpDir + "/seg-read-error.sfa"; + writeSegment(path, 17L, new int[]{256}); + RecoveryReadFacade ff = new RecoveryReadFacade(); + ff.failReadWithError = true; + ff.stopReadsAt = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 32L; + try { + MmapSegment.openExisting(ff, path).close(); + fail("expected positioned-read failure"); } catch (MmapSegmentException expected) { - // ok -- SegmentRing's per-file catch skips just this file - // instead of aborting recovery of the whole slot. + assertFalse("operational read errors must not be quarantinable corruption", + expected instanceof MmapSegmentCorruptionException); + assertTrue(expected.getMessage(), expected.getMessage().contains("could not read")); } + assertEquals("mapping must not start after a failed scan", 0, ff.mmapCalls); + assertEquals("open descriptor must be closed", 1, ff.closeCalls); + assertTrue("read failure must not mutate the segment", Files.exists(path)); }); } - /** - * Portable fail-on-revert guard for the recovery mmap-fault handling on the - * scan path. Unlike {@link #testRecoveryKeepsFramesBeforeUnbackedTail} - * (which only faults on ZFS), this maps the file past real EOF via the - * length-injecting facade, so the scan's read of the beyond-EOF page faults - * on ext4/xfs too. The fix must recognize that fault and keep - * recovery safe -- never a JVM abort, never a raw {@code InternalError} - * escaping into {@code SegmentRing}'s recovery loop. Revert the - * {@code scanFrames}/{@code openExisting} mmap-fault guard (or fold the CRC - * back through native {@code Crc32c}) and this errors or aborts the fork. - *

      - * Two handled outcomes are accepted, because which one occurs depends on - * whether the recovery methods are JIT-compiled at fault time: - *

        - *
      • Interpreter / C1: {@code scanFrames}'s own - * {@code catch (InternalError)} fires, so the frame below the tear is - * recovered and a usable segment is returned.
        • - *
      • C2: once {@code scanFrames} is inlined into - * {@code openExisting}, HotSpot delivers the async unsafe-access - * {@code InternalError} to {@code openExisting}'s outer - * {@code catch (Throwable)} instead of the inlined inner one, which - * converts the file to a skippable {@link MmapSegmentException}. - * Still fully handled -- {@code SegmentRing} skips just this - * {@code .sfa} rather than aborting the slot.
        • - *
      - * (The C2 delivery imprecision is a property of HotSpot's async - * unsafe-access fault handling, not of this seam; the seam only makes it - * reproducible off ZFS.) - */ @Test - public void testScanFaultOnMapPastEofIsHandledAnyFilesystem() throws Exception { + public void testReadErrorAfterDetectedTornBytesStillFailsClosed() throws Exception { TestUtils.assertMemoryLeak(() -> { - final String path = tmpDir + "/seg-mappasteof-scan.sfa"; - final long page = Files.PAGE_SIZE; - // One frame that ends exactly on the first page boundary. - final int payloadLen = (int) (page - MmapSegment.HEADER_SIZE - MmapSegment.FRAME_HEADER_SIZE); - long boundary = writeSegment(path, 5L, new int[]{payloadLen}); - assertEquals("frame must fill exactly one page", page, boundary); - // Free every block past the first page: the file is now exactly one - // (fully backed) page, with nothing beyond it on disk. - truncateTo(path, page); - // Report twice the real length so openExisting maps a second, - // beyond-EOF page; the scan faults reading it on any filesystem. - FilesFacade ff = new MapPastEofFacade(path, 2 * page); + final String path = tmpDir + "/seg-torn-then-read-error.sfa"; + long lastGood = writeSegment(path, 18L, new int[]{256}); + + // Make the failed-frame header non-zero in the first recovery + // buffer. Recovery must still read the rest of the suffix so an + // operational error in a later buffer cannot hide behind the + // already-established torn-tail signal. + int fd = Files.openRW(path); + assertTrue("openRW failed", fd >= 0); + long marker = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT); + try { + Unsafe.getUnsafe().putByte(marker, (byte) 1); + assertEquals(1L, Files.write(fd, marker, 1L, lastGood)); + } finally { + Unsafe.free(marker, 1, MemoryTag.NATIVE_DEFAULT); + Files.close(fd); + } + + RecoveryReadFacade ff = new RecoveryReadFacade(); + ff.failReadWithError = true; + ff.stopReadsAt = 64L * 1024L; + try { + MmapSegment.openExisting(ff, path).close(); + fail("expected later positioned-read failure"); + } catch (MmapSegmentException expected) { + assertFalse(expected instanceof MmapSegmentCorruptionException); + assertTrue(expected.getMessage(), expected.getMessage().contains("could not read")); + } + assertTrue("fault must occur after recovery consumed the first buffer", ff.readCalls > 1); + assertEquals("mapping must not start after any suffix read fails", 0, ff.mmapCalls); + assertEquals("open descriptor must be closed", 1, ff.closeCalls); + assertTrue("read failure must not mutate the segment", Files.exists(path)); + }); + } + + @Test + public void testShortReadFailsClosedBeforeMmap() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final String path = tmpDir + "/seg-short-read.sfa"; + writeSegment(path, 19L, new int[]{256}); + RecoveryReadFacade ff = new RecoveryReadFacade(); + ff.stopReadsAt = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 32L; + try { + MmapSegment.openExisting(ff, path).close(); + fail("expected premature EOF failure"); + } catch (MmapSegmentException expected) { + assertFalse("premature EOF must remain an operational failure", + expected instanceof MmapSegmentCorruptionException); + assertTrue(expected.getMessage(), expected.getMessage().contains("short read")); + } + assertEquals("mapping must not start after a failed scan", 0, ff.mmapCalls); + assertEquals("open descriptor must be closed", 1, ff.closeCalls); + assertTrue("short read must not mutate the segment", Files.exists(path)); + }); + } + + @Test + public void testShortReadsAreRetried() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final String path = tmpDir + "/seg-partial-reads.sfa"; + long expectedEnd = writeSegment(path, 23L, new int[]{31, 127, 4097}); + RecoveryReadFacade ff = new RecoveryReadFacade(); + ff.maxReadSize = 1024; try (MmapSegment seg = MmapSegment.openExisting(ff, path)) { - // Interpreter / C1: graceful partial recovery. - assertEquals("the frame below the beyond-EOF page must be recovered", 1L, seg.frameCount()); - assertEquals("scan must stop at the beyond-EOF boundary", page, seg.publishedOffset()); - assertEquals("a beyond-EOF page is not a torn write", 0L, seg.tornTailBytes()); - } catch (MmapSegmentException skippedUnderC2) { - // C2: the inlined fault escaped to openExisting's outer catch and - // was converted to a per-file skip. Assert it is the recognized - // mmap fault (not some other data error) so a revert -- which - // lets a raw InternalError through instead -- still fails here. - assertTrue(skippedUnderC2.getMessage(), - skippedUnderC2.getMessage().contains("unsafe memory access operation")); + assertEquals(3L, seg.frameCount()); + assertEquals(expectedEnd, seg.publishedOffset()); } + assertTrue("recovery must loop over partial positioned reads", ff.readCalls > 1); + assertEquals(1, ff.mmapCalls); + assertEquals(1, ff.closeCalls); }); } - /** - * Portable fail-on-revert guard for the {@code openExisting} header-block - * guard. The file is truncated to empty and the facade reports a full page, - * so the very first header read (magic) lands on a beyond-EOF page and - * faults on any filesystem. {@code openExisting} must convert that to a - * {@link MmapSegmentException} -- the per-file signal {@code SegmentRing} - * skips on -- not let the raw {@code InternalError} escape and abort recovery - * of every sibling. Revert the header-block conversion and this throws - * {@code InternalError} instead of {@code MmapSegmentException}. - */ @Test - public void testHeaderFaultOnMapPastEofIsSkippableAnyFilesystem() throws Exception { + public void testSizeChangeFailsClosedBeforeMmap() throws Exception { TestUtils.assertMemoryLeak(() -> { - final String path = tmpDir + "/seg-mappasteof-header.sfa"; - final long page = Files.PAGE_SIZE; - writeSegment(path, 9L, new int[]{64}); - // Free every block: the file is now empty, so even page 0 (the - // header) is beyond EOF under the reported one-page mapping. - truncateTo(path, 0L); - FilesFacade ff = new MapPastEofFacade(path, page); + final String path = tmpDir + "/seg-size-change.sfa"; + writeSegment(path, 29L, new int[]{64}); + RecoveryReadFacade ff = new RecoveryReadFacade(); + ff.changeLengthAfterScan = true; try { MmapSegment.openExisting(ff, path).close(); - fail("expected MmapSegmentException for a beyond-EOF header page"); + fail("expected unstable-size failure"); } catch (MmapSegmentException expected) { - // ok -- SegmentRing's per-file catch skips just this file - // instead of aborting recovery of the whole slot. + assertFalse(expected instanceof MmapSegmentCorruptionException); + assertTrue(expected.getMessage(), expected.getMessage().contains("size changed")); } + assertEquals("mapping must not start for an unstable file", 0, ff.mmapCalls); + assertEquals("open descriptor must be closed", 1, ff.closeCalls); + assertTrue("size-race detection must not mutate the segment", Files.exists(path)); + }); + } + + @Test + public void testSizeChangeWhileMappingFailsClosedAndUnmaps() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final String path = tmpDir + "/seg-size-change-while-mapping.sfa"; + writeSegment(path, 31L, new int[]{64}); + RecoveryReadFacade ff = new RecoveryReadFacade(); + ff.changeLengthOnMmap = true; + try { + MmapSegment.openExisting(ff, path).close(); + fail("expected size change during mmap to fail recovery"); + } catch (MmapSegmentException expected) { + assertFalse(expected instanceof MmapSegmentCorruptionException); + assertTrue(expected.getMessage(), expected.getMessage().contains("size changed while mapping")); + } + assertEquals("scan should map only after validation", 1, ff.mmapCalls); + assertEquals("rejected mapping must be released", 1, ff.munmapCalls); + assertEquals("open descriptor must be closed", 1, ff.closeCalls); + assertTrue("injected size observation must not mutate the segment", Files.exists(path)); }); } @@ -311,9 +329,7 @@ private static long writeSegment(String path, long baseSeq, int[] payloadLens) { } long buf = Unsafe.malloc(maxLen, MemoryTag.NATIVE_DEFAULT); try { - for (int i = 0; i < maxLen; i++) { - Unsafe.getUnsafe().putByte(buf + i, (byte) (i | 1)); // all non-zero - } + Unsafe.getUnsafe().setMemory(buf, maxLen, (byte) 1); try (MmapSegment seg = MmapSegment.create(path, baseSeq, SEGMENT_BYTES)) { for (int len : payloadLens) { assertTrue("append must fit", seg.tryAppend(buf, len) >= 0); @@ -329,8 +345,8 @@ private static long writeSegment(String path, long baseSeq, int[] payloadLens) { * Turns {@code [keepBytes, SEGMENT_BYTES)} of the file into an unbacked * sparse hole: truncate down to {@code keepBytes} (frees the tail blocks), * then back up to {@code SEGMENT_BYTES} (re-extends the logical size without - * allocating blocks). Recovery maps the full stat length, so the hole is - * inside the mapping -- reads of it fault on ZFS and zero-fill on ext4. + * allocating blocks). Positioned recovery reads observe the hole as zeroes + * without dereferencing it through mmap. */ private static void punchSparseTail(String path, long keepBytes) { int fd = Files.openRW(path); @@ -344,36 +360,20 @@ private static void punchSparseTail(String path, long keepBytes) { } /** - * Shrinks the file to {@code keepBytes}, freeing every block past it, and - * leaves it there (no re-extend). Combined with a facade that reports a - * larger length, the freed region becomes a beyond-EOF part of the mapping - * that faults on read on any filesystem. + * Positioned-read fault seam. Calls not involved in recovery scanning + * delegate to the production {@link FilesFacade#INSTANCE}. */ - private static void truncateTo(String path, long keepBytes) { - int fd = Files.openRW(path); - assertTrue("openRW failed", fd >= 0); - try { - assertTrue("truncate failed", Files.truncate(fd, keepBytes)); - } finally { - Files.close(fd); - } - } - - /** - * A {@link FilesFacade} that reports an inflated stat length for one target - * path so {@code openExisting} maps that file past end-of-file (see - * {@link FilesFacade#length(String)}); every other call, including - * {@code length} for any other path, delegates to the production - * {@link FilesFacade#INSTANCE}. - */ - private static final class MapPastEofFacade implements FilesFacade { - private final long reportedLength; - private final String targetPath; - - MapPastEofFacade(String targetPath, long reportedLength) { - this.targetPath = targetPath; - this.reportedLength = reportedLength; - } + private static final class RecoveryReadFacade implements FilesFacade { + private boolean changeLengthAfterScan; + private boolean changeLengthOnMmap; + private int closeCalls; + private boolean failReadWithError; + private int lengthCalls; + private int maxReadSize = Integer.MAX_VALUE; + private int mmapCalls; + private int munmapCalls; + private int readCalls; + private long stopReadsAt = Long.MAX_VALUE; @Override public boolean allocate(int fd, long size) { @@ -387,6 +387,7 @@ public long allocNativePath(String path) { @Override public int close(int fd) { + closeCalls++; return INSTANCE.close(fd); } @@ -432,7 +433,13 @@ public int fsync(int fd) { @Override public long length(int fd) { - return INSTANCE.length(fd); + long length = INSTANCE.length(fd); + lengthCalls++; + if ((changeLengthAfterScan && lengthCalls > 1) + || (changeLengthOnMmap && mmapCalls > 0)) { + return length - 1L; + } + return length; } @Override @@ -442,7 +449,7 @@ public long length(long pathPtr) { @Override public long length(String path) { - return targetPath.equals(path) ? reportedLength : INSTANCE.length(path); + return INSTANCE.length(path); } @Override @@ -455,6 +462,18 @@ public int mkdir(String path, int mode) { return INSTANCE.mkdir(path, mode); } + @Override + public long mmap(int fd, long len, long offset, int flags, int memoryTag) { + mmapCalls++; + return INSTANCE.mmap(fd, len, offset, flags, memoryTag); + } + + @Override + public void munmap(long address, long len, int memoryTag) { + munmapCalls++; + INSTANCE.munmap(address, len, memoryTag); + } + @Override public int openCleanRW(String path) { return INSTANCE.openCleanRW(path); @@ -477,7 +496,15 @@ public int openRW(long pathPtr) { @Override public long read(int fd, long addr, long len, long offset) { - return INSTANCE.read(fd, addr, len, offset); + readCalls++; + if (offset >= stopReadsAt) { + return failReadWithError ? -1L : 0L; + } + long delegatedLen = Math.min(len, (long) maxReadSize); + if (delegatedLen > stopReadsAt - offset) { + delegatedLen = stopReadsAt - offset; + } + return INSTANCE.read(fd, addr, delegatedLen, offset); } @Override diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentTest.java index 177ee5f6..b63a0e97 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentTest.java @@ -35,6 +35,7 @@ import org.junit.Before; import org.junit.Test; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; @@ -55,6 +56,60 @@ public void tearDown() { TestUtils.removeTmpDir(tmpDir); } + @Test + public void testBarrierChecksMsyncAndFsyncFailures() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long payload = Unsafe.malloc(32, MemoryTag.NATIVE_DEFAULT); + try { + FaultyFilesFacade success = new FaultyFilesFacade(); + try (MmapSegment segment = MmapSegment.create( + success, tmpDir + "/barrier-ok.sfa", 0L, 4096L)) { + fillPattern(payload, 32, 1); + assertTrue(segment.tryAppend(payload, 32) >= 0L); + assertFalse(segment.isPublishedDurable()); + assertEquals(segment.publishedOffset(), segment.syncPublished()); + assertTrue(segment.isPublishedDurable()); + assertEquals(1, success.msyncCalls); + assertEquals(1, success.fsyncCalls); + } + + FaultyFilesFacade msyncFailure = new FaultyFilesFacade(); + msyncFailure.failOnMsync = true; + try (MmapSegment segment = MmapSegment.create( + msyncFailure, tmpDir + "/barrier-msync-fail.sfa", 0L, 4096L)) { + assertTrue(segment.tryAppend(payload, 32) >= 0L); + try { + segment.syncPublished(); + fail("expected msync failure"); + } catch (MmapSegmentException expected) { + assertTrue(expected.getMessage().contains("sync segment data")); + } + assertFalse(segment.isPublishedDurable()); + assertEquals(1, msyncFailure.msyncCalls); + assertEquals(0, msyncFailure.fsyncCalls); + } + + FaultyFilesFacade fsyncFailure = new FaultyFilesFacade(); + fsyncFailure.failOnFsync = true; + try (MmapSegment segment = MmapSegment.create( + fsyncFailure, tmpDir + "/barrier-fsync-fail.sfa", 0L, 4096L)) { + assertTrue(segment.tryAppend(payload, 32) >= 0L); + try { + segment.syncPublished(); + fail("expected fsync failure"); + } catch (MmapSegmentException expected) { + assertTrue(expected.getMessage().contains("sync segment file")); + } + assertFalse(segment.isPublishedDurable()); + assertEquals(1, fsyncFailure.msyncCalls); + assertEquals(1, fsyncFailure.fsyncCalls); + } + } finally { + Unsafe.free(payload, 32, MemoryTag.NATIVE_DEFAULT); + } + }); + } + @Test public void testCapacityRemainingAccountsForFrameEnvelope() throws Exception { TestUtils.assertMemoryLeak(() -> { @@ -80,6 +135,27 @@ public void testCapacityRemainingAccountsForFrameEnvelope() throws Exception { }); } + @Test + public void testCompatibilityMsyncKeepsLegacyCallPattern() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long payload = Unsafe.malloc(32, MemoryTag.NATIVE_DEFAULT); + try { + FaultyFilesFacade filesFacade = new FaultyFilesFacade(); + try (MmapSegment segment = MmapSegment.create( + filesFacade, tmpDir + "/compat-msync.sfa", 0L, 4096L)) { + assertTrue(segment.tryAppend(payload, 32) >= 0L); + segment.msync(); + segment.msync(); + assertEquals(2, filesFacade.msyncCalls); + assertEquals(0, filesFacade.fsyncCalls); + assertFalse(segment.isPublishedDurable()); + } + } finally { + Unsafe.free(payload, 32, MemoryTag.NATIVE_DEFAULT); + } + }); + } + @Test public void testCreateAppendCloseReopenScansAllFrames() throws Exception { TestUtils.assertMemoryLeak(() -> { @@ -128,7 +204,7 @@ public void testCreateFailsCleanlyWhenAllocateReturnsFalse() throws Exception { assertTrue(expected.getMessage(), expected.getMessage().contains("pre-allocation failed")); } - assertEquals("openCleanRW must run exactly once", 1, ff.openCleanRWCalls); + assertEquals("exclusive create must run exactly once", 1, ff.openRWExclusiveCalls); assertEquals("allocate must run exactly once", 1, ff.allocateCalls); assertEquals("fd must be closed on allocate failure", 1, ff.closeCalls); assertEquals("file must be removed on allocate failure", 1, ff.removeCalls); @@ -138,28 +214,31 @@ public void testCreateFailsCleanlyWhenAllocateReturnsFalse() throws Exception { } @Test - public void testCreateFailsCleanlyWhenOpenCleanRWReturnsMinusOne() throws Exception { + public void testCreateFailsCleanlyWhenExclusiveOpenReturnsMinusOne() throws Exception { TestUtils.assertMemoryLeak(() -> { String path = tmpDir + "/seg-noopen.sfa"; long sizeBytes = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 64; FaultyFilesFacade ff = new FaultyFilesFacade(); - ff.failOnOpenCleanRW = true; + ff.failOnOpenRWExclusive = true; try { MmapSegment.create(ff, path, 0L, sizeBytes).close(); - fail("expected MmapSegmentException from openCleanRW returning -1"); + fail("expected MmapSegmentException from openRWExclusive returning -1"); } catch (MmapSegmentException expected) { assertTrue(expected.getMessage(), - expected.getMessage().contains("openCleanRW failed")); + expected.getMessage().contains("exclusive create failed")); } - assertEquals("openCleanRW must run exactly once", 1, ff.openCleanRWCalls); - assertEquals("allocate must not run after openCleanRW failure", + assertEquals("exclusive create must run exactly once", 1, ff.openRWExclusiveCalls); + assertEquals("allocate must not run after exclusive-create failure", 0, ff.allocateCalls); assertEquals("close must not be called when no fd was opened", 0, ff.closeCalls); - assertEquals("remove must not be called when openCleanRW failed", + // With O_EXCL semantics a create failure can mean "path already + // exists and belongs to another lifecycle" -- create() must NOT + // unlink a file it never owned. + assertEquals("remove must not be called when exclusive create failed", 0, ff.removeCalls); - assertFalse("no file should exist when openCleanRW failed", + assertFalse("no file should exist when exclusive create failed", Files.exists(path)); }); } @@ -198,7 +277,7 @@ public void testCreateRepeatedAllocateFailuresDoNotAccumulateOrphans() throws Ex } assertEquals("no orphan files may survive repeated allocate failures", 0, survivors); - assertEquals(attempts, ff.openCleanRWCalls); + assertEquals(attempts, ff.openRWExclusiveCalls); assertEquals(attempts, ff.allocateCalls); assertEquals(attempts, ff.closeCalls); assertEquals(attempts, ff.removeCalls); @@ -212,9 +291,9 @@ public void testFirstFrameCrcCorruptionFlagsTornTailAndPreservesFile() throws Ex // frames are followed by garbage. None cover frame[0] itself // being corrupt — yet a single bit-flip on the CRC of frame[0] // at rest (bit-rot, partial-page-write at crash) is the - // worst-case data-loss trigger: scanFrames bails at HEADER_SIZE - // and frameCount drops to 0, even though valid frames still - // sit on disk past the corrupt header. + // worst-case data-loss trigger: the recovery scan bails at + // HEADER_SIZE and frameCount drops to 0, even though valid frames + // still sit on disk past the corrupt header. // // Contract: tornTailBytes() must be non-zero (because non-zero // bytes exist past the last good frame), and openExisting @@ -255,7 +334,7 @@ public void testFirstFrameCrcCorruptionFlagsTornTailAndPreservesFile() throws Ex Files.exists(path)); try (MmapSegment seg = MmapSegment.openExisting(path)) { - assertEquals("scanFrames must bail at the corrupt frame[0]", + assertEquals("recovery scan must bail at the corrupt frame[0]", 0L, seg.frameCount()); assertEquals("publishedOffset must rewind to the header end", MmapSegment.HEADER_SIZE, seg.publishedOffset()); @@ -265,6 +344,16 @@ public void testFirstFrameCrcCorruptionFlagsTornTailAndPreservesFile() throws Ex + "hot-spare leftover; got " + seg.tornTailBytes(), seg.tornTailBytes() > 0L); } + // A second open must see the SAME evidence: openExisting is + // observe-only, so the valid-CRC frames past the corrupt + // frame[0] -- the only copy of that data -- stay on disk for + // operator extraction (chain-level recovery of such a member + // fails closed without mutating it). + try (MmapSegment seg = MmapSegment.openExisting(path)) { + assertEquals(0L, seg.frameCount()); + assertEquals("observe-only recovery must preserve the residue", + 4096L - MmapSegment.HEADER_SIZE, seg.tornTailBytes()); + } assertTrue("openExisting must not unlink the corrupt file", Files.exists(path)); } finally { @@ -273,6 +362,47 @@ public void testFirstFrameCrcCorruptionFlagsTornTailAndPreservesFile() throws Ex }); } + @Test + public void testAllZeroCorruptFrameHeaderDoesNotHideNonZeroSuffix() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String path = tmpDir + "/seg-zero-frame-header.sfa"; + long payload = Unsafe.malloc(32, MemoryTag.NATIVE_DEFAULT); + try { + fillPattern(payload, 32, 0); + try (MmapSegment seg = MmapSegment.create(path, 0L, 4096)) { + assertTrue(seg.tryAppend(payload, 32) >= 0L); + assertTrue(seg.tryAppend(payload, 32) >= 0L); + seg.msync(); + } + + // Zero both the CRC and length of frame[0]. Looking only at + // this 8-byte header would misclassify the segment as a clean + // empty spare even though its payload and frame[1] remain. + int fd = Files.openRW(path); + assertTrue("openRW must succeed", fd >= 0); + long zeroHeader = Unsafe.malloc(MmapSegment.FRAME_HEADER_SIZE, MemoryTag.NATIVE_DEFAULT); + try { + Unsafe.getUnsafe().setMemory(zeroHeader, MmapSegment.FRAME_HEADER_SIZE, (byte) 0); + assertEquals(MmapSegment.FRAME_HEADER_SIZE, + Files.write(fd, zeroHeader, MmapSegment.FRAME_HEADER_SIZE, MmapSegment.HEADER_SIZE)); + Files.fsync(fd); + } finally { + Unsafe.free(zeroHeader, MmapSegment.FRAME_HEADER_SIZE, MemoryTag.NATIVE_DEFAULT); + Files.close(fd); + } + + try (MmapSegment seg = MmapSegment.openExisting(path)) { + assertEquals(0L, seg.frameCount()); + assertEquals(MmapSegment.HEADER_SIZE, seg.publishedOffset()); + assertEquals("non-zero bytes later in the suffix must preserve corruption evidence", + 4096L - MmapSegment.HEADER_SIZE, seg.tornTailBytes()); + } + } finally { + Unsafe.free(payload, 32, MemoryTag.NATIVE_DEFAULT); + } + }); + } + @Test public void testFullSegmentRejectsFurtherAppends() throws Exception { TestUtils.assertMemoryLeak(() -> { @@ -372,6 +502,315 @@ public void testRecoveryDoesNotFlagFreshUnusedSegment() throws Exception { }); } + @Test + public void testOpenExistingObservesTornTailWithoutMutation() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // openExisting must be a pure observer: a torn tail can hide + // unreachable valid-CRC frames past a mid-file tear -- the only + // copy of real payloads -- and whether they may be destroyed is + // a chain-level decision (SegmentRing sanitizes the segment it + // resumes as active, plus proven-dead sealed residue, and fails + // closed preserving the bytes otherwise). Repeated opens must + // keep reporting the same observation over identical bytes. + String path = tmpDir + "/seg-observe.sfa"; + long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + long lastGood; + try { + try (MmapSegment seg = MmapSegment.create(path, 0L, 4096)) { + for (int i = 0; i < 3; i++) { + fillPattern(buf, 16, i); + seg.tryAppend(buf, 16); + } + lastGood = seg.publishedOffset(); + long addr = seg.address(); + for (long off = lastGood; off + 4 <= 4096; off += 4) { + Unsafe.getUnsafe().putInt(addr + off, 0xCAFEBABE); + } + seg.msync(); + } + byte[] before = java.nio.file.Files.readAllBytes(java.nio.file.Paths.get(path)); + for (int open = 0; open < 2; open++) { + try (MmapSegment seg = MmapSegment.openExisting(path)) { + assertEquals("open #" + open + " must report the residue", + 4096L - lastGood, seg.tornTailBytes()); + assertEquals(3L, seg.frameCount()); + assertTrue(seg.hasUnsanitizedTornTail()); + } + assertArrayEquals( + "observe-only recovery must not change a single byte (open #" + open + ')', + before, java.nio.file.Files.readAllBytes(java.nio.file.Paths.get(path))); + } + } finally { + Unsafe.free(buf, 16, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + @Test + public void testSanitizeTornTailZeroesResidueOnDisk() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Once ring recovery proves the residue is not load-bearing it + // calls sanitizeTornTail: the open keeps reporting the observed + // residue (operator signal), but after sanitization a later open + // of the untouched file must find a clean zero suffix and intact + // frames. + String path = tmpDir + "/seg-zeroed.sfa"; + long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + long lastGood; + try { + try (MmapSegment seg = MmapSegment.create(path, 0L, 4096)) { + for (int i = 0; i < 3; i++) { + fillPattern(buf, 16, i); + seg.tryAppend(buf, 16); + } + lastGood = seg.publishedOffset(); + long addr = seg.address(); + for (long off = lastGood; off + 4 <= 4096; off += 4) { + Unsafe.getUnsafe().putInt(addr + off, 0xCAFEBABE); + } + seg.msync(); + } + try (MmapSegment seg = MmapSegment.openExisting(path)) { + assertEquals("recovery must report the observed residue", + 4096L - lastGood, seg.tornTailBytes()); + assertTrue(seg.hasUnsanitizedTornTail()); + seg.sanitizeTornTail(); + assertEquals("the observation must survive sanitization for diagnostics", + 4096L - lastGood, seg.tornTailBytes()); + assertFalse(seg.hasUnsanitizedTornTail()); + // Idempotent: a second call is a no-op, not a re-zero. + seg.sanitizeTornTail(); + } + try (MmapSegment seg = MmapSegment.openExisting(path)) { + assertEquals("sanitize must have zeroed the residue on disk", + 0L, seg.tornTailBytes()); + assertEquals(lastGood, seg.publishedOffset()); + assertEquals(3L, seg.frameCount()); + } + } finally { + Unsafe.free(buf, 16, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + @Test + public void testResealGapResidueCannotSurviveRecoveryAppends() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Two-crash reseal regression at segment level. Crash #1 leaves + // residue up to the file end; recovery resumes at lastGood; the + // resumed writer fills the segment but its last frame stops short + // of the file end (the remaining gap cannot fit another frame). + // Pre-fix the stale residue survived in that gap, so the segment + // -- sealed as-is by rotation -- failed the sealed-suffix-must- + // be-zero check on the NEXT recovery, permanently failing startup. + long segSize = MmapSegment.HEADER_SIZE + + 4 * (MmapSegment.FRAME_HEADER_SIZE + 16) + + 12; // reseal gap: a 5th 24-byte frame can never fit + String path = tmpDir + "/seg-reseal.sfa"; + long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + try { + // Session 1: two frames, then a crash mid-write near the end + // -- garbage from the torn frame all the way to the file end. + try (MmapSegment seg = MmapSegment.create(path, 0L, segSize)) { + fillPattern(buf, 16, 0); + seg.tryAppend(buf, 16); + fillPattern(buf, 16, 1); + seg.tryAppend(buf, 16); + long addr = seg.address(); + for (long off = seg.publishedOffset(); off + 4 <= segSize; off += 4) { + Unsafe.getUnsafe().putInt(addr + off, 0xCAFEBABE); + } + seg.msync(); + } + // Recovery #1 + session 2: fill the segment to its rotation + // point. The 4th frame ends 12 bytes short of the file end -- + // a region session 2 never overwrites. Sanitize before the + // appends, exactly as SegmentRing.recover does for the + // segment it resumes as active. + try (MmapSegment seg = MmapSegment.openExisting(path)) { + assertEquals(2L, seg.frameCount()); + seg.sanitizeTornTail(); + fillPattern(buf, 16, 2); + assertTrue(seg.tryAppend(buf, 16) >= 0); + fillPattern(buf, 16, 3); + assertTrue(seg.tryAppend(buf, 16) >= 0); + assertEquals("next append must not fit (rotation would seal here)", + -1L, seg.tryAppend(buf, 16)); + seg.msync(); + } + // Recovery #2: the state a sealed segment presents at the next + // startup. Its suffix must be clean or ring recovery bricks. + try (MmapSegment seg = MmapSegment.openExisting(path)) { + assertEquals("all four frames must recover", 4L, seg.frameCount()); + assertEquals("no residue may survive in the reseal gap: a sealed " + + "segment with a non-zero suffix permanently fails " + + "ring recovery", + 0L, seg.tornTailBytes()); + } + } finally { + Unsafe.free(buf, 16, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + @Test + public void testDiscardedStaleFrameNotResurrectedByLaterRecovery() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // The frame envelope [crc(len|payload)][len][payload] binds + // neither position nor FSN, so a stale frame with a valid CRC + // sitting past the tear -- byte-aligned with the resumed writer's + // frames, natural with fixed-size records -- would be silently + // re-adopted by the next recovery scan at a recycled FSN. + // Recovery #1 discarded it; recovery #2 must not bring it back. + String path = tmpDir + "/seg-stale.sfa"; + long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + long frameB; + try { + try (MmapSegment seg = MmapSegment.create(path, 0L, 4096)) { + fillPattern(buf, 16, 0); + seg.tryAppend(buf, 16); // frame A, FSN 0 + fillPattern(buf, 16, 1); + frameB = seg.tryAppend(buf, 16); // frame B, FSN 1 + fillPattern(buf, 16, 2); + seg.tryAppend(buf, 16); // frame C, FSN 2 + // The crash tears frame B only: flip its CRC. C keeps a + // valid CRC at a frame-aligned offset past the tear. + long addr = seg.address(); + int crc = Unsafe.getUnsafe().getInt(addr + frameB); + Unsafe.getUnsafe().putInt(addr + frameB, crc ^ 0x5A5A5A5A); + seg.msync(); + } + // Recovery #1: B fails CRC, so the scan stops at B; B and C + // are discarded, and the resume-as-active sanitization (the + // same call SegmentRing.recover makes) zeroes them on disk. + try (MmapSegment seg = MmapSegment.openExisting(path)) { + assertEquals("scan must stop at the torn frame", 1L, seg.frameCount()); + seg.sanitizeTornTail(); + // The resumed writer re-issues FSN 1 with a fresh payload + // of the old B's exact size -- the byte-aligned case. + fillPattern(buf, 16, 7); + assertEquals(frameB, seg.tryAppend(buf, 16)); + seg.msync(); + } + // Recovery #2: pre-fix the scan walked A, B' and then adopted + // the STALE C (valid CRC) as a live frame -- data recovery #1 + // had already discarded, resurrected behind the engine's back. + try (MmapSegment seg = MmapSegment.openExisting(path)) { + assertEquals("stale frame C must stay discarded, not be " + + "resurrected at a recycled FSN", 2L, seg.frameCount()); + assertEquals(0L, seg.tornTailBytes()); + } + } finally { + Unsafe.free(buf, 16, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + @Test + public void testSanitizeTornTailSyncFailureFailsClosed() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // The sanitize barrier is load-bearing: if the zeroes cannot be + // made durable, ring recovery must fail closed rather than expose + // a segment whose reseal could permanently fail the next startup. + // openExisting itself runs no barrier -- it never writes. A + // failed attempt may still leave zeroes in the page cache; that + // is safe (a retry either sees a clean tail or re-zeroes), so + // the follow-up open with a healthy facade must succeed. + long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + try { + String msyncPath = tmpDir + "/seg-zero-msync-fail.sfa"; + String fsyncPath = tmpDir + "/seg-zero-fsync-fail.sfa"; + for (String path : new String[]{msyncPath, fsyncPath}) { + try (MmapSegment seg = MmapSegment.create(path, 0L, 4096)) { + fillPattern(buf, 16, 0); + seg.tryAppend(buf, 16); + long addr = seg.address(); + Unsafe.getUnsafe().putInt(addr + seg.publishedOffset(), 0xCAFEBABE); + Unsafe.getUnsafe().putInt(addr + seg.publishedOffset() + 4, 16); + seg.msync(); + } + } + + FaultyFilesFacade msyncFailure = new FaultyFilesFacade(); + msyncFailure.failOnMsync = true; + try (MmapSegment seg = MmapSegment.openExisting(msyncFailure, msyncPath)) { + assertEquals("observe-only open must not run the zeroing barrier", + 0, msyncFailure.msyncCalls); + try { + seg.sanitizeTornTail(); + fail("expected sanitize to abort when the zeroed tail cannot be msync'd"); + } catch (MmapSegmentException expected) { + assertTrue(expected.getMessage(), + expected.getMessage().contains("zeroed torn tail")); + } + assertTrue("a failed barrier must leave the residue unsanitized", + seg.hasUnsanitizedTornTail()); + } + assertEquals(1, msyncFailure.msyncCalls); + assertEquals(0, msyncFailure.fsyncCalls); + try (MmapSegment seg = MmapSegment.openExisting(msyncPath)) { + assertEquals(1L, seg.frameCount()); + } + + FaultyFilesFacade fsyncFailure = new FaultyFilesFacade(); + fsyncFailure.failOnFsync = true; + try (MmapSegment seg = MmapSegment.openExisting(fsyncFailure, fsyncPath)) { + try { + seg.sanitizeTornTail(); + fail("expected sanitize to abort when the zeroed tail cannot be fsync'd"); + } catch (MmapSegmentException expected) { + assertTrue(expected.getMessage(), + expected.getMessage().contains("zeroed torn tail")); + } + assertTrue("a failed barrier must leave the residue unsanitized", + seg.hasUnsanitizedTornTail()); + } + assertEquals(1, fsyncFailure.msyncCalls); + assertEquals(1, fsyncFailure.fsyncCalls); + try (MmapSegment seg = MmapSegment.openExisting(fsyncPath)) { + assertEquals(1L, seg.frameCount()); + } + } finally { + Unsafe.free(buf, 16, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + @Test + public void testSanitizeTornTailRefusedAfterAppendsResume() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Zeroing [residueStart, EOF) after appends resumed would destroy + // freshly appended frames, so that ordering is rejected outright: + // sanitization is a recovery-time-only operation that must run + // before the segment takes traffic. + String path = tmpDir + "/seg-sanitize-late.sfa"; + long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + try { + try (MmapSegment seg = MmapSegment.create(path, 0L, 4096)) { + fillPattern(buf, 16, 0); + seg.tryAppend(buf, 16); + long addr = seg.address(); + Unsafe.getUnsafe().putInt(addr + seg.publishedOffset(), 0xCAFEBABE); + seg.msync(); + } + try (MmapSegment seg = MmapSegment.openExisting(path)) { + assertTrue(seg.hasUnsanitizedTornTail()); + fillPattern(buf, 16, 1); + assertTrue(seg.tryAppend(buf, 16) >= 0); + try { + seg.sanitizeTornTail(); + fail("sanitize after appends must be refused"); + } catch (MmapSegmentException expected) { + assertTrue(expected.getMessage(), + expected.getMessage().contains("before appends resume")); + } + } + } finally { + Unsafe.free(buf, 16, MemoryTag.NATIVE_DEFAULT); + } + }); + } + @Test public void testRecoverySignalsTornTailWithByteCount() throws Exception { TestUtils.assertMemoryLeak(() -> { @@ -494,10 +933,34 @@ private static final class FaultyFilesFacade implements FilesFacade { int allocateCalls; int closeCalls; boolean failOnAllocate; + boolean failOnFsync; + boolean failOnMsync; boolean failOnOpenCleanRW; + boolean failOnOpenRWExclusive; + int fsyncCalls; + int msyncCalls; int openCleanRWCalls; + int openRWExclusiveCalls; int removeCalls; + @Override + public int openRWExclusive(String path) { + openRWExclusiveCalls++; + if (failOnOpenRWExclusive) { + return -1; + } + return INSTANCE.openRWExclusive(path); + } + + @Override + public int openRWExclusive(long pathPtr) { + openRWExclusiveCalls++; + if (failOnOpenRWExclusive) { + return -1; + } + return INSTANCE.openRWExclusive(pathPtr); + } + @Override public long allocNativePath(String path) { return INSTANCE.allocNativePath(path); @@ -555,7 +1018,8 @@ public void freeNativePath(long pathPtr) { @Override public int fsync(int fd) { - return INSTANCE.fsync(fd); + fsyncCalls++; + return failOnFsync ? -1 : INSTANCE.fsync(fd); } @Override @@ -578,6 +1042,12 @@ public int mkdir(String path, int mode) { return INSTANCE.mkdir(path, mode); } + @Override + public int msync(long addr, long len, boolean async) { + msyncCalls++; + return failOnMsync ? -1 : INSTANCE.msync(addr, len, async); + } + @Override public int openCleanRW(String path) { openCleanRWCalls++; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PrReviewRedTests.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PrReviewRedTests.java index e551c6f8..ae987fee 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PrReviewRedTests.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PrReviewRedTests.java @@ -79,8 +79,8 @@ public void tearDown() { /** * Finding C1 / C10 — first-frame CRC corruption silently deletes the segment. *

      - * If frame[0] of a recovered .sfa fails CRC validation, scanFrames returns - * lastGood=HEADER_SIZE, countFrames returns 0, and SegmentRing.openExisting + * If frame[0] of a recovered .sfa fails CRC validation, the recovery scan + * returns lastGood=HEADER_SIZE and frameCount=0, and SegmentRing.openExisting * unlinks the file as an "empty hot-spare leftover" — destroying every frame * that physically followed the corrupt header. The torn-tail WARN inside * MmapSegment.openExisting is dropped on the floor. @@ -128,8 +128,8 @@ public void testC1_recoveryMustNotUnlinkSegmentWithCorruptFirstFrame() throws Ex // Run recovery. SegmentRing recovered = SegmentRing.openExisting(tmpDir, 64 * 1024); try { - // The bug: openExisting sees frameCount=0 (because scanFrames - // bailed at the corrupt frame[0]) and treats the segment as + // The bug: openExisting sees frameCount=0 (because the recovery + // scan bailed at the corrupt frame[0]) and treats the segment as // an "empty hot-spare leftover" — closing AND UNLINKING the // file. The user's frames 1, 2, 3 are gone forever; the only // record was a WARN log line that's already been emitted. diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCloseRaceTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCloseRaceTest.java index 3d7c6a7c..46ea1b0d 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCloseRaceTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCloseRaceTest.java @@ -27,16 +27,13 @@ import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentRing; -import io.questdb.client.std.bytes.DirectByteSink; import io.questdb.client.std.Files; -import io.questdb.client.std.str.DirectUtf8Sink; import io.questdb.client.test.tools.TestUtils; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import java.lang.reflect.Field; import java.nio.file.Paths; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -116,12 +113,9 @@ public void testManagerDoesNotInstallSpareIntoClosedRing() throws Exception { manager.close(); } - Field hotSpareField = SegmentRing.class.getDeclaredField("hotSpare"); - hotSpareField.setAccessible(true); - int leaked = 0; for (int i = 0; i < ITERATIONS; i++) { - Object hs = hotSpareField.get(rings[i]); + Object hs = rings[i].getHotSpareForTesting(); if (hs != null) { leaked++; // Don't leak in the test: close the survivor. @@ -204,6 +198,391 @@ public void testCloseDoesNotFreePathScratchWhenWorkerStillAlive() throws Excepti }); } + /** + * Pins the claim-time registration gate at the top of + * {@code SegmentManager.serviceRing}: a snapshot entry whose ring was + * deregistered BEFORE the worker claims it must be skipped entirely — + * never claimed as {@code inService}, never handed to + * {@code serviceRing0}. This is the exact guarantee + * {@code CursorSendEngine.close()} relies on when it releases the ring, + * watermark and slot flock right after + * {@code awaitRingQuiescence(ring) == true}: the deregistering thread may + * already be freeing those resources, so a stale snapshot entry must not + * be touched at all (spare install, watermark write, drainTrimmable, or + * path building under the slot dir). + *

      + * Deterministic shape: three rings A, B, C registered before start, so + * the worker's first snapshot is exactly [A, B, C]. The worker parks in + * A's spare-install pass; while it is parked, B is deregistered (and a + * quiescence barrier for B must pass immediately — no pass for B is in + * flight). After release the worker walks the rest of its stale + * snapshot: it must skip B and service C. Every serviced ring is + * recorded from inside the worker's own pass (via the trim-sync hook + * reading {@code inService}), so the assertion is exact — no timing + * grace, no sleeps. + */ + @Test(timeout = 15_000L) + public void testStaleSnapshotEntrySkippedAfterDeregisterBeforeServiceClaim() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long segSize = MmapSegment.HEADER_SIZE + (MmapSegment.FRAME_HEADER_SIZE + 32); + SegmentManager manager = new SegmentManager(segSize, TimeUnit.SECONDS.toNanos(60)); + SegmentRing[] rings = new SegmentRing[3]; + String[] slots = new String[3]; + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(); + AtomicReference hookErr = new AtomicReference<>(); + // Rings the worker actually claimed and serviced, recorded from + // the worker thread itself at the trim-sync point every service + // pass reaches (spare needed or not). + java.util.Set serviced = + java.util.Collections.synchronizedSet( + java.util.Collections.newSetFromMap(new java.util.IdentityHashMap<>())); + boolean managerClosed = false; + try { + for (int i = 0; i < 3; i++) { + slots[i] = tmpDir + "/claim-skip-slot-" + i; + Assert.assertEquals(0, Files.mkdir(slots[i], Files.DIR_MODE_DEFAULT)); + MmapSegment initial = MmapSegment.create( + slots[i] + "/sf-initial.sfa", 0L, segSize); + rings[i] = new SegmentRing(initial, segSize); + } + manager.setBeforeInstallSyncHook(() -> { + if (!fired.compareAndSet(false, true)) return; + workerBlocked.countDown(); + try { + if (!releaseWorker.await(10, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, + new AssertionError("timed out waiting for test to release worker")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + manager.setBeforeTrimSyncHook(() -> { + try { + Object ring = readInServiceRing(manager); + if (ring != null) { + serviced.add(ring); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + // Register all three BEFORE start: the worker's first snapshot + // is [A, B, C] and every fresh ring wants a hot spare, so the + // install hook parks the worker inside A's pass. + manager.register(rings[0], slots[0]); + manager.register(rings[1], slots[1]); + manager.register(rings[2], slots[2]); + manager.start(); + Assert.assertTrue("worker did not reach A's install pass", + workerBlocked.await(5, TimeUnit.SECONDS)); + + // B is deregistered while its snapshot entry is still ahead of + // the worker's cursor. The quiescence barrier must pass at + // once: the in-flight pass is A's, not B's — this is the state + // in which an engine owner frees B's resources. + manager.deregister(rings[1]); + Assert.assertTrue("no pass for B is in flight — the barrier must pass immediately", + manager.awaitRingQuiescence(rings[1])); + + releaseWorker.countDown(); + + // Positive marker that the worker walked PAST B's snapshot + // slot: C sits after B in the same snapshot, so once C has + // been serviced, B's claim-or-skip decision has been made. + long deadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (!serviced.contains(rings[2])) { + if (System.nanoTime() > deadlineNs) { + throw new AssertionError("worker never serviced ring C after release"); + } + Thread.sleep(1); + } + + Assert.assertTrue("ring A must have been serviced", serviced.contains(rings[0])); + Assert.assertFalse( + "worker claimed and serviced a snapshot entry that was deregistered " + + "before its pass started — the deregistering thread may already " + + "be releasing the ring/watermark/slot lock, so a stale snapshot " + + "entry must be skipped at claim time", + serviced.contains(rings[1])); + // Defense in depth: nothing may have been installed into the + // deregistered ring either. + Assert.assertNull("no hot spare may be installed into a deregistered ring", + readHotSpare(rings[1])); + + manager.close(); + managerClosed = true; + if (hookErr.get() != null) { + throw new AssertionError("worker-side hook failed", hookErr.get()); + } + } finally { + manager.setBeforeInstallSyncHook(null); + manager.setBeforeTrimSyncHook(null); + releaseWorker.countDown(); + if (!managerClosed) { + manager.close(); + } + for (SegmentRing ring : rings) { + if (ring != null) { + ring.close(); + } + } + } + }); + } + + /** + * Pins the scratch-handoff half of the timed-out-close contract in + * isolation: after {@code close()} gives up on the bounded join and hands + * {@code pathScratch} ownership to the worker, the WORKER's exit block + * alone must free the native buffer — with no retried {@code close()} + * ever running. The sibling test + * ({@link #testCloseDoesNotFreePathScratchWhenWorkerStillAlive}) retries + * {@code close()} before asserting the free, so a regression that dropped + * the worker-side free (leaving reclaim to a retry nobody is required to + * make) would stay green there: production owners do NOT retry — a + * timed-out close returns to the pool and the worker is the only thread + * left that can reclaim the allocation. + */ + @Test(timeout = 15_000L) + public void testWorkerAloneFreesPathScratchAfterTimedOutClose() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long segSize = MmapSegment.HEADER_SIZE + (MmapSegment.FRAME_HEADER_SIZE + 32); + String slot = tmpDir + "/worker-frees-scratch-slot"; + Assert.assertEquals(0, Files.mkdir(slot, Files.DIR_MODE_DEFAULT)); + MmapSegment initial = MmapSegment.create(slot + "/sf-initial.sfa", 0L, segSize); + SegmentRing ring = new SegmentRing(initial, segSize); + SegmentManager manager = new SegmentManager(segSize, TimeUnit.SECONDS.toNanos(60)); + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(); + AtomicReference hookErr = new AtomicReference<>(); + try { + manager.register(ring, slot); + manager.setBeforeInstallSyncHook(() -> { + if (!fired.compareAndSet(false, true)) return; + workerBlocked.countDown(); + try { + if (!releaseWorker.await(10, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, + new AssertionError("timed out waiting for test to release worker")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + manager.start(); + Assert.assertTrue("worker did not reach install hook", + workerBlocked.await(5, TimeUnit.SECONDS)); + + // Timed-out close: hands scratch ownership to the worker and + // returns. This is the last close() call this test makes. + manager.setWorkerJoinTimeoutMillis(50L); + manager.close(); + Thread worker = readWorkerThread(manager); + Assert.assertTrue("worker must still be live after the timed-out close", + worker != null && worker.isAlive()); + Assert.assertTrue("scratch must still be allocated while the worker may use it", + readPathScratchImpl(manager) != 0L); + + // Release the pass. running=false already, so the worker + // finishes the pass and exits — its exit block must free the + // handed-over scratch buffer without ANY further caller action. + releaseWorker.countDown(); + worker.join(TimeUnit.SECONDS.toMillis(10)); + Assert.assertFalse("worker never exited after release", worker.isAlive()); + Assert.assertEquals( + "worker exit block must free the handed-over path scratch — no retried " + + "close() runs in production after a timed-out close, so leaving " + + "the free to a retry leaks the native buffer for the process " + + "lifetime", + 0L, readPathScratchImpl(manager)); + + // Reap the dead thread for tidiness; must not double-free. + manager.close(); + Assert.assertNull("retried close must reap the exited worker", + readWorkerThread(manager)); + if (hookErr.get() != null) { + throw new AssertionError("install hook failed", hookErr.get()); + } + } finally { + manager.setBeforeInstallSyncHook(null); + releaseWorker.countDown(); + manager.close(); + ring.close(); + } + }); + } + + /** + * Pins the {@link SegmentManager#deferUntilWorkerExit} handoff contract + * that {@code CursorSendEngine.close()}'s slot-ownership transfer depends + * on: + *
        + *
      • rejects the handoff ({@code false}) when no worker ever started — + * the caller must clean up inline;
        • + *
      • accepts it ({@code true}) while the worker is live-but-slow + * mid service pass after a timed-out close(), and runs the cleanup + * exactly when the worker exits — never while the pass is still in + * flight;
        • + *
      • rejects it again once the worker loop has exited/been reaped.
        • + *
      + */ + @Test(timeout = 15_000L) + public void testDeferUntilWorkerExitRunsCleanupAfterFinalPass() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long segSize = MmapSegment.HEADER_SIZE + (MmapSegment.FRAME_HEADER_SIZE + 32); + String slot = tmpDir + "/defer-slot"; + Assert.assertEquals(0, Files.mkdir(slot, Files.DIR_MODE_DEFAULT)); + MmapSegment initial = MmapSegment.create(slot + "/sf-initial.sfa", 0L, segSize); + SegmentRing ring = new SegmentRing(initial, segSize); + SegmentManager manager = new SegmentManager(segSize, TimeUnit.SECONDS.toNanos(60)); + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + CountDownLatch cleanupRan = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(); + AtomicReference hookErr = new AtomicReference<>(); + boolean managerClosed = false; + try { + // Never-started manager: no worker will ever run the cleanup, + // and none can touch a slot — the caller must do it inline. + Assert.assertFalse("never-started manager must reject the handoff", + manager.deferUntilWorkerExit(cleanupRan::countDown)); + + manager.register(ring, slot); + manager.setBeforeInstallSyncHook(() -> { + if (!fired.compareAndSet(false, true)) return; + workerBlocked.countDown(); + try { + if (!releaseWorker.await(10, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, + new AssertionError("timed out waiting for test to release worker")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + manager.start(); + Assert.assertTrue("worker did not reach install hook", + workerBlocked.await(5, TimeUnit.SECONDS)); + + // Timed-out close: running=false, worker parked mid-pass — the + // exact state CursorSendEngine.close() hands ownership over in. + manager.setWorkerJoinTimeoutMillis(50L); + manager.close(); + Assert.assertFalse("worker must not be reaped while parked mid-pass", + manager.isWorkerReaped()); + + Assert.assertTrue("live-but-slow worker must accept the handoff", + manager.deferUntilWorkerExit(cleanupRan::countDown)); + Assert.assertEquals("cleanup must not run while the pass is still in flight", + 1, cleanupRan.getCount()); + + // Release the pass; the worker loop observes running=false, + // exits, and must run the deferred cleanup on its way out. + releaseWorker.countDown(); + Assert.assertTrue("cleanup never ran on worker exit", + cleanupRan.await(10, TimeUnit.SECONDS)); + + // Reap the exited worker, then: no live worker, no handoff. + manager.setWorkerJoinTimeoutMillis(TimeUnit.SECONDS.toMillis(60)); + manager.close(); + managerClosed = true; + Assert.assertFalse("reaped manager must reject the handoff", + manager.deferUntilWorkerExit(() -> { + })); + if (hookErr.get() != null) { + throw new AssertionError("install hook failed", hookErr.get()); + } + } finally { + manager.setBeforeInstallSyncHook(null); + releaseWorker.countDown(); + if (!managerClosed) { + Thread.interrupted(); + manager.close(); + } + ring.close(); + } + }); + } + + /** + * Pins the {@link SegmentManager#awaitRingQuiescence} contract that + * {@code CursorSendEngine.close()} depends on: + *
        + *
      • returns {@code false} (never {@code true}) while a service pass + * for the ring is provably in flight and the timeout elapses;
        • + *
      • preserves a pending caller interrupt without aborting;
        • + *
      • returns {@code true} once the in-flight pass has finished.
        • + *
      + */ + @Test(timeout = 15_000L) + public void testAwaitRingQuiescenceBlocksWhileServicePassInFlight() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long segSize = MmapSegment.HEADER_SIZE + (MmapSegment.FRAME_HEADER_SIZE + 32); + String slot = tmpDir + "/quiesce-slot"; + Assert.assertEquals(0, Files.mkdir(slot, Files.DIR_MODE_DEFAULT)); + MmapSegment initial = MmapSegment.create(slot + "/sf-initial.sfa", 0L, segSize); + SegmentRing ring = new SegmentRing(initial, segSize); + SegmentManager manager = new SegmentManager(segSize, TimeUnit.SECONDS.toNanos(60)); + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(); + AtomicReference hookErr = new AtomicReference<>(); + boolean managerClosed = false; + try { + manager.register(ring, slot); + manager.setBeforeInstallSyncHook(() -> { + if (!fired.compareAndSet(false, true)) return; + workerBlocked.countDown(); + try { + if (!releaseWorker.await(10, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, + new AssertionError("timed out waiting for test to release worker")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + manager.start(); + Assert.assertTrue("worker did not reach install hook", + workerBlocked.await(5, TimeUnit.SECONDS)); + + manager.deregister(ring); + manager.setWorkerJoinTimeoutMillis(50L); + Thread.currentThread().interrupt(); + Assert.assertFalse( + "awaitRingQuiescence returned true while the worker was parked " + + "inside the service pass for this ring", + manager.awaitRingQuiescence(ring)); + Assert.assertTrue("awaitRingQuiescence must preserve the caller's interrupt", + Thread.interrupted()); + + releaseWorker.countDown(); + manager.setWorkerJoinTimeoutMillis(TimeUnit.SECONDS.toMillis(60)); + Assert.assertTrue( + "awaitRingQuiescence must return true once the in-flight pass finished", + manager.awaitRingQuiescence(ring)); + + manager.close(); + managerClosed = true; + if (hookErr.get() != null) { + throw new AssertionError("install hook failed", hookErr.get()); + } + } finally { + manager.setBeforeInstallSyncHook(null); + releaseWorker.countDown(); + if (!managerClosed) { + Thread.interrupted(); + manager.close(); + } + ring.close(); + } + }); + } + @Test(timeout = 15_000L) public void testInterruptedCallerDoesNotAbandonReapableWorker() throws Exception { TestUtils.assertMemoryLeak(() -> { @@ -306,21 +685,134 @@ private static void cleanupRecursively(String dir) { } } - private static long readPathScratchImpl(SegmentManager manager) throws Exception { - Field pathScratchF = SegmentManager.class.getDeclaredField("pathScratch"); - pathScratchF.setAccessible(true); - DirectUtf8Sink pathScratch = (DirectUtf8Sink) pathScratchF.get(manager); - Field sinkF = DirectUtf8Sink.class.getDeclaredField("sink"); - sinkF.setAccessible(true); - DirectByteSink sink = (DirectByteSink) sinkF.get(pathScratch); - Field implF = DirectByteSink.class.getDeclaredField("impl"); - implF.setAccessible(true); - return implF.getLong(sink); + private static Object readHotSpare(SegmentRing ring) { + return ring.getHotSpareForTesting(); + } + + private static Object readInServiceRing(SegmentManager manager) { + return manager.getInServiceRingForTesting(); + } + + /** + * Pins the SECOND bounded join in {@link SegmentManager#close()}: when + * the first (tunable) join times out but the worker has already left its + * service loop ({@code workerLoopExited}) and is running only its finite + * exit cleanups, close() must NOT hand off and walk away -- it gives the + * cleanups a second, fixed-budget join and reaps the worker once they + * finish. Without it, a bounded-join timeout landing mid-cleanup reaps + * the worker while an engine's flock release is still in flight, so + * callers observe a stale not-yet-closed state and schedule spurious + * flock-release retries. Every existing close-race test parks the worker + * MID-PASS (first-join territory); none reached this branch. + */ + @Test(timeout = 20_000L) + public void testSecondBoundedJoinReapsWorkerFinishingExitCleanups() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long segSize = MmapSegment.HEADER_SIZE + (MmapSegment.FRAME_HEADER_SIZE + 32); + String slot = tmpDir + "/second-join-slot"; + Assert.assertEquals(0, Files.mkdir(slot, Files.DIR_MODE_DEFAULT)); + MmapSegment initial = MmapSegment.create(slot + "/sf-initial.sfa", 0L, segSize); + SegmentRing ring = new SegmentRing(initial, segSize); + SegmentManager manager = new SegmentManager(segSize, TimeUnit.SECONDS.toNanos(60)); + CountDownLatch cleanupEntered = new CountDownLatch(1); + CountDownLatch releaseCleanup = new CountDownLatch(1); + AtomicReference err = new AtomicReference<>(); + Thread releaser = null; + try { + manager.register(ring, slot); + manager.start(); + // Deferred exit cleanup that parks: once the worker runs it, + // workerLoopExited is already true (set before cleanups) and + // the thread is alive in its finite exit phase -- exactly the + // second-join state. + Assert.assertTrue("live worker must accept the exit-cleanup handoff", + manager.deferUntilWorkerExit(() -> { + cleanupEntered.countDown(); + try { + if (!releaseCleanup.await(10, TimeUnit.SECONDS)) { + err.compareAndSet(null, new AssertionError( + "timed out waiting for test to release the exit cleanup")); + } + } catch (Throwable t) { + err.compareAndSet(null, t); + } + })); + Thread worker = readWorkerThread(manager); + Assert.assertNotNull(worker); + + // Hold the cleanup past the first join (200ms) and release it + // well inside the second join's fixed 5s budget. + releaser = new Thread(() -> { + try { + if (!cleanupEntered.await(10, TimeUnit.SECONDS)) { + err.compareAndSet(null, new AssertionError( + "worker never reached its exit cleanups")); + return; + } + Thread.sleep(400L); + } catch (Throwable t) { + err.compareAndSet(null, t); + } finally { + releaseCleanup.countDown(); + } + }, "second-join-releaser"); + releaser.start(); + + // close(): running=false, worker exits its loop promptly (it + // is idle), sets workerLoopExited, parks in the cleanup. The + // first join (200ms) expires against the parked cleanup; the + // fall-through must take the second join, which reaps once + // the releaser lets the cleanup finish. + manager.setWorkerJoinTimeoutMillis(200L); + manager.close(); + + // The discriminating observable: with the second bounded join, + // close() blocks until the parked cleanup finishes (released at + // +400ms) and reaps a DEAD worker; the pre-fix code reaped on + // observing workerLoopExited and returned at ~200ms with the + // worker still alive in its cleanup. Assert liveness at the + // moment close() returns -- before any join in this test can + // mask it. (isWorkerReaped() alone cannot discriminate: both + // variants null workerThread.) + Assert.assertFalse( + "close() must not return while the worker is still alive in its exit " + + "cleanups -- the second bounded join has to hold close() until " + + "they finish", + worker.isAlive()); + + Assert.assertEquals("worker must have been parked in its exit cleanups", + 0, cleanupEntered.getCount()); + Assert.assertTrue( + "second bounded join must reap a worker that finishes its exit " + + "cleanups inside the fixed budget -- close() returned unreaped", + manager.isWorkerReaped()); + worker.join(TimeUnit.SECONDS.toMillis(5)); + Assert.assertFalse("worker must be dead after the second join reaped it", + worker.isAlive()); + Assert.assertEquals( + "close() must free the path scratch itself after the second join " + + "confirmed termination (no handoff on this branch)", + 0L, readPathScratchImpl(manager)); + if (err.get() != null) { + throw new AssertionError("async participant failed", err.get()); + } + } finally { + releaseCleanup.countDown(); + if (releaser != null) { + releaser.join(TimeUnit.SECONDS.toMillis(10)); + } + Thread.interrupted(); + manager.close(); + ring.close(); + } + }); + } + + private static long readPathScratchImpl(SegmentManager manager) { + return manager.isPathScratchAllocatedForTesting() ? 1L : 0L; } - private static Thread readWorkerThread(SegmentManager manager) throws Exception { - Field workerThreadF = SegmentManager.class.getDeclaredField("workerThread"); - workerThreadF.setAccessible(true); - return (Thread) workerThreadF.get(manager); + private static Thread readWorkerThread(SegmentManager manager) { + return manager.getWorkerThreadForTesting(); } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCrashConsistencyTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCrashConsistencyTest.java new file mode 100644 index 00000000..126c79c8 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerCrashConsistencyTest.java @@ -0,0 +1,534 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + *******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.qwp.client.sf.cursor.AckWatermark; +import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentRing; +import io.questdb.client.std.Files; +import io.questdb.client.std.FilesFacade; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +public class SegmentManagerCrashConsistencyTest { + + private static void awaitTrimmed(SegmentRing ring) { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (ring.firstSealed() != null || ring.getPendingTrimCount() != 0) { + if (System.nanoTime() > deadline) { + throw new AssertionError("manager did not trim acknowledged segments"); + } + io.questdb.client.std.Compat.onSpinWait(); + } + } + + private static void awaitValue(AtomicInteger value, int expected, String message) { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (value.get() < expected) { + if (System.nanoTime() > deadline) { + throw new AssertionError(message + " [expected=" + expected + ", actual=" + value.get() + ']'); + } + io.questdb.client.std.Compat.onSpinWait(); + } + } + + private static SegmentRing createRing(String root, long segmentSize, long payload, int sealedCount) { + SegmentRing ring = null; + boolean success = false; + try { + ring = new SegmentRing(MmapSegment.create(root + "/sf-0.sfa", 0, segmentSize), segmentSize); + Assert.assertEquals(0L, ring.appendOrFsn(payload, 1)); + for (int i = 1; i <= sealedCount; i++) { + ring.installHotSpare(MmapSegment.create(root + "/sf-" + i + ".sfa", i, segmentSize)); + Assert.assertEquals(i, ring.appendOrFsn(payload, 1)); + } + ring.installHotSpare(MmapSegment.create(root + "/sf-" + (sealedCount + 1) + ".sfa", + sealedCount + 1L, segmentSize)); + Assert.assertTrue(ring.acknowledge(sealedCount - 1L)); + success = true; + return ring; + } finally { + if (!success && ring != null) ring.close(); + } + } + + private static AckWatermark openWatermark(FilesFacade ff, String root) { + return AckWatermark.open(ff, root); + } + + private static void removeRecursive(String dir) { + long find = Files.findFirst(dir); + if (find > 0) { + try { + int rc = 1; + while (rc > 0) { + String name = Files.utf8ToString(Files.findName(find)); + if (name != null && !".".equals(name) && !"..".equals(name)) { + Files.remove(dir + "/" + name); + } + rc = Files.findNext(find); + } + } finally { + Files.findClose(find); + } + } + Files.remove(dir); + } + + @Test(timeout = 15_000L) + public void testBackgroundTrimDurabilityOrder() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String root = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-manager-crash-order-" + System.nanoTime()).toString(); + Assert.assertEquals(0, Files.mkdir(root, Files.DIR_MODE_DEFAULT)); + long segmentSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 1L; + long payload = 0; + OrderingFilesFacade ff = new OrderingFilesFacade(root); + AckWatermark watermark = null; + SegmentRing ring = null; + try { + payload = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT); + ring = createRing(root, segmentSize, payload, 2); + watermark = openWatermark(ff, root); + Assert.assertNotNull(watermark); + try (SegmentManager manager = new SegmentManager( + segmentSize, TimeUnit.SECONDS.toNanos(60), segmentSize * 8L, ff, ff::ticks)) { + manager.register(ring, root, watermark); + ff.active = true; + manager.start(); + awaitTrimmed(ring); + } + Assert.assertEquals(Arrays.asList("watermark-msync", "watermark-fsync", "dir-fsync", + "segment-remove", "segment-remove", "dir-fsync"), ff.events); + } finally { + if (ring != null) ring.close(); + if (watermark != null) watermark.close(); + if (payload != 0) Unsafe.free(payload, 1, MemoryTag.NATIVE_DEFAULT); + removeRecursive(root); + } + }); + } + + @Test(timeout = 15_000L) + public void testBarrierFailuresPreserveCrashSafetyAndRetry() throws Exception { + TestUtils.assertMemoryLeak(() -> { + for (int failAt : new int[]{0, 1, 2, 5}) { + String root = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-manager-crash-fault-" + failAt + "-" + System.nanoTime()).toString(); + Assert.assertEquals(0, Files.mkdir(root, Files.DIR_MODE_DEFAULT)); + long segmentSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 1L; + long payload = 0; + OrderingFilesFacade ff = new OrderingFilesFacade(root, failAt); + AckWatermark watermark = null; + SegmentRing ring = null; + try { + payload = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT); + ring = createRing(root, segmentSize, payload, 2); + watermark = openWatermark(ff, root); + Assert.assertNotNull(watermark); + try (SegmentManager manager = new SegmentManager( + segmentSize, TimeUnit.SECONDS.toNanos(60), segmentSize * 8L, ff, ff::ticks)) { + manager.register(ring, root, watermark); + ff.active = true; + manager.start(); + ff.awaitFailure(); + if (failAt == 5) { + Assert.assertNull("post-unlink barrier failure exposed a closed segment", + ring.findSegmentContaining(0L)); + Assert.assertNull("post-unlink barrier failure kept a closed segment live", + ring.firstSealed()); + Assert.assertEquals("post-unlink barrier failure lost cleanup ownership", + 2, ring.getPendingTrimCount()); + ff.advance(TimeUnit.SECONDS.toNanos(2)); + manager.wakeWorker(); + awaitTrimmed(ring); + } + manager.close(); + } + if (failAt <= 2) { + Assert.assertEquals("unlink started before covering barrier failed", 0, ff.removeCalls.get()); + Assert.assertNotNull("barrier failure removed ring bookkeeping", ring.firstSealed()); + } else { + Assert.assertNull("post-unlink barrier retry did not commit ring removal", ring.firstSealed()); + Assert.assertEquals("post-unlink barrier retry retained pending ownership", + 0, ring.getPendingTrimCount()); + } + Assert.assertFalse("segment deletion began without a durable covering watermark", + ff.removeCalls.get() > 0 && !ff.durableWatermark); + } finally { + if (ring != null) ring.close(); + if (watermark != null) watermark.close(); + if (payload != 0) Unsafe.free(payload, 1, MemoryTag.NATIVE_DEFAULT); + removeRecursive(root); + } + } + }); + } + + @Test(timeout = 15_000L) + public void testDiskTrimWithoutWatermarkIsPreserved() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String root = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-manager-no-watermark-" + System.nanoTime()).toString(); + Assert.assertEquals(0, Files.mkdir(root, Files.DIR_MODE_DEFAULT)); + long segmentSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 1L; + long payload = 0; + SegmentRing ring = null; + try { + payload = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT); + ring = createRing(root, segmentSize, payload, 1); + CountDownLatch servicePass = new CountDownLatch(1); + try (SegmentManager manager = new SegmentManager( + segmentSize, TimeUnit.SECONDS.toNanos(60), segmentSize * 8L)) { + manager.setBeforeTrimSyncHook(servicePass::countDown); + manager.register(ring, root, null); + manager.start(); + Assert.assertTrue("manager did not reach the trim service pass", + servicePass.await(5, TimeUnit.SECONDS)); + } + Assert.assertNotNull(ring.firstSealed()); + Assert.assertTrue(Files.exists(root + "/sf-0.sfa")); + } finally { + if (ring != null) ring.close(); + if (payload != 0) Unsafe.free(payload, 1, MemoryTag.NATIVE_DEFAULT); + removeRecursive(root); + } + }); + } + + @Test(timeout = 15_000L) + public void testMoreThanOneQuantumBatchesBarriers() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String root = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-manager-crash-batch-" + System.nanoTime()).toString(); + Assert.assertEquals(0, Files.mkdir(root, Files.DIR_MODE_DEFAULT)); + long segmentSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 1L; + long payload = 0; + OrderingFilesFacade ff = new OrderingFilesFacade(root); + AckWatermark watermark = null; + SegmentRing ring = null; + try { + payload = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT); + ring = createRing(root, segmentSize, payload, 65); + watermark = openWatermark(ff, root); + Assert.assertNotNull(watermark); + try (SegmentManager manager = new SegmentManager( + segmentSize, TimeUnit.SECONDS.toNanos(60), segmentSize * 70L, ff, ff::ticks)) { + manager.register(ring, root, watermark); + ff.active = true; + manager.start(); + awaitTrimmed(ring); + } + Assert.assertEquals(65, ff.removeCalls.get()); + Assert.assertEquals("watermark must sync once per quantum", 2, ff.msyncCalls.get()); + Assert.assertEquals("directory barriers must be twice per quantum", 4, ff.dirSyncCalls.get()); + } finally { + if (ring != null) ring.close(); + if (watermark != null) watermark.close(); + if (payload != 0) Unsafe.free(payload, 1, MemoryTag.NATIVE_DEFAULT); + removeRecursive(root); + } + }); + } + + @Test(timeout = 15_000L) + public void testPersistentFailureDoesNotStarveSibling() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String base = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-manager-retry-sibling-" + System.nanoTime()).toString(); + String badRoot = base + "/bad"; + String goodRoot = base + "/good"; + Assert.assertEquals(0, Files.mkdir(base, Files.DIR_MODE_DEFAULT)); + Assert.assertEquals(0, Files.mkdir(badRoot, Files.DIR_MODE_DEFAULT)); + Assert.assertEquals(0, Files.mkdir(goodRoot, Files.DIR_MODE_DEFAULT)); + long segmentSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 1L; + long payload = 0; + OrderingFilesFacade ff = new OrderingFilesFacade(badRoot, "dir-fsync", 0); + AckWatermark badWatermark = null; + AckWatermark goodWatermark = null; + SegmentRing badRing = null; + SegmentRing goodRing = null; + try { + payload = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT); + badRing = createRing(badRoot, segmentSize, payload, 1); + goodRing = createRing(goodRoot, segmentSize, payload, 1); + badWatermark = openWatermark(ff, badRoot); + goodWatermark = openWatermark(ff, goodRoot); + try (SegmentManager manager = new SegmentManager( + segmentSize, TimeUnit.SECONDS.toNanos(60), segmentSize * 16L, ff, ff::ticks)) { + manager.register(badRing, badRoot, badWatermark); + manager.register(goodRing, goodRoot, goodWatermark); + ff.active = true; + manager.start(); + awaitValue(ff.failureCalls, 1, "bad sibling failure was not attempted"); + awaitTrimmed(goodRing); + Assert.assertNotNull("failed sibling unexpectedly trimmed", badRing.firstSealed()); + } + } finally { + if (badRing != null) badRing.close(); + if (goodRing != null) goodRing.close(); + if (badWatermark != null) badWatermark.close(); + if (goodWatermark != null) goodWatermark.close(); + if (payload != 0) Unsafe.free(payload, 1, MemoryTag.NATIVE_DEFAULT); + removeRecursive(badRoot); + removeRecursive(goodRoot); + removeRecursive(base); + } + }); + } + + @Test(timeout = 30_000L) + public void testPersistentFailuresBackOffAndRecover() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String[] failures = {"watermark-msync", "segment-remove", "post-dir-fsync"}; + for (String failure : failures) { + String root = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-manager-retry-" + failure + '-' + System.nanoTime()).toString(); + Assert.assertEquals(0, Files.mkdir(root, Files.DIR_MODE_DEFAULT)); + long segmentSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 1L; + long payload = 0; + OrderingFilesFacade ff = new OrderingFilesFacade( + root, failure, "segment-remove".equals(failure) ? 2 : 0); + if ("watermark-msync".equals(failure)) { + ff.ticks.set(Long.MAX_VALUE - 2_000_000L); + } + AckWatermark watermark = null; + SegmentRing ring = null; + try { + payload = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT); + ring = createRing(root, segmentSize, payload, 2); + watermark = openWatermark(ff, root); + Assert.assertNotNull(watermark); + AtomicInteger logs = new AtomicInteger(); + AtomicInteger passes = new AtomicInteger(); + try (SegmentManager manager = new SegmentManager( + segmentSize, TimeUnit.SECONDS.toNanos(60), segmentSize * 8L, ff, ff::ticks)) { + manager.setBeforeTrimSyncHook(passes::incrementAndGet); + manager.setRetryLogHook(logs::incrementAndGet); + manager.register(ring, root, watermark); + ff.active = true; + manager.start(); + awaitValue(ff.failureCalls, 1, "initial persistent failure was not attempted"); + awaitValue(logs, 1, "initial failure transition was not logged"); + if ("segment-remove".equals(failure)) { + Assert.assertNull("failed unlink left a closed segment live", + ring.firstSealed()); + Assert.assertEquals("successful unlink prefix was not committed", + 1, ring.getPendingTrimCount()); + Assert.assertFalse(Files.exists(root + "/sf-0.sfa")); + Assert.assertTrue(Files.exists(root + "/sf-1.sfa")); + } + + int operations = ff.operationCalls(); + for (int i = 0; i < 4; i++) { + int nextPass = passes.get() + 1; + manager.wakeWorker(); + awaitValue(passes, nextPass, "deferred retry pass did not run"); + } + Assert.assertEquals("deferred passes performed filesystem work", + operations, ff.operationCalls()); + Assert.assertEquals("deferred passes emitted logs", 1, logs.get()); + + long delay = 4_000_000L; + for (int attempt = 2; attempt <= 11; attempt++) { + int nextPass = passes.get() + 1; + operations = ff.operationCalls(); + ff.advance(delay - 1); + manager.wakeWorker(); + awaitValue(passes, nextPass, "pre-deadline pass did not run"); + Assert.assertEquals("pre-deadline pass performed filesystem work", + operations, ff.operationCalls()); + nextPass = passes.get() + 1; + ff.advance(1); + manager.wakeWorker(); + awaitValue(ff.failureCalls, attempt, "retry deadline did not enable attempt"); + awaitValue(passes, nextPass, "retry pass did not run"); + Assert.assertEquals("persistent failure log was not throttled", 1, logs.get()); + delay = Math.min(delay * 2, 1_024_000_000L); + } + + ff.failureEnabled = false; + ff.advance(delay); + manager.wakeWorker(); + awaitTrimmed(ring); + awaitValue(logs, 2, "recovery transition was not logged"); + Assert.assertEquals("recovery transition was not logged once", 2, logs.get()); + } + Assert.assertNull(ring.firstSealed()); + } finally { + if (ring != null) ring.close(); + if (watermark != null) watermark.close(); + if (payload != 0) Unsafe.free(payload, 1, MemoryTag.NATIVE_DEFAULT); + removeRecursive(root); + } + } + }); + } + + private static final class OrderingFilesFacade implements FilesFacade { + private final AtomicInteger dirSyncCalls = new AtomicInteger(); + private final List events = new ArrayList<>(); + private final int failAt; + private final AtomicInteger failureCalls = new AtomicInteger(); + private final AtomicInteger msyncCalls = new AtomicInteger(); + private final AtomicInteger removeCalls = new AtomicInteger(); + private final String persistentEvent; + private final int persistentRemoveOrdinal; + private final String root; + private final AtomicLong ticks = new AtomicLong(); + private boolean active; + private boolean durableSegments = true; + private boolean durableWatermark; + private int eventIndex; + private boolean expectPreDirSync = true; + private volatile boolean failureEnabled; + private volatile boolean failureObserved; + private int watermarkFd = -1; + + private OrderingFilesFacade(String root) { + this(root, -1); + } + + private OrderingFilesFacade(String root, int failAt) { + this.root = root; + this.failAt = failAt; + this.persistentEvent = null; + this.persistentRemoveOrdinal = 0; + } + + private OrderingFilesFacade(String root, String persistentEvent, int persistentRemoveOrdinal) { + this.root = root; + this.failAt = -1; + this.persistentEvent = persistentEvent; + this.persistentRemoveOrdinal = persistentRemoveOrdinal; + this.failureEnabled = true; + } + + private void advance(long nanos) { + ticks.addAndGet(nanos); + } + + private void awaitFailure() { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (!failureObserved) { + if (System.nanoTime() > deadline) { + throw new AssertionError("injected barrier failure was not reached"); + } + io.questdb.client.std.Compat.onSpinWait(); + } + } + + private boolean fail(String event) { + events.add("post-dir-fsync".equals(event) ? "dir-fsync" : event); + boolean persistentMatch = failureEnabled && event.equals(persistentEvent) + && (!"segment-remove".equals(event) || persistentRemoveOrdinal <= removeCalls.get()); + boolean failed = eventIndex++ == failAt || persistentMatch; + if (failed) { + failureCalls.incrementAndGet(); + failureObserved = true; + } + return failed; + } + + private int operationCalls() { + return msyncCalls.get() + dirSyncCalls.get() + removeCalls.get(); + } + + private long ticks() { + return ticks.get(); + } + + @Override public boolean allocate(int fd, long size) { return INSTANCE.allocate(fd, size); } + @Override public long allocNativePath(String path) { return INSTANCE.allocNativePath(path); } + @Override public int close(int fd) { return INSTANCE.close(fd); } + @Override public boolean exists(String path) { return INSTANCE.exists(path); } + @Override public void findClose(long findPtr) { INSTANCE.findClose(findPtr); } + @Override public long findFirst(String dir) { return INSTANCE.findFirst(dir); } + @Override public long findName(long findPtr) { return INSTANCE.findName(findPtr); } + @Override public int findNext(long findPtr) { return INSTANCE.findNext(findPtr); } + @Override public int findType(long findPtr) { return INSTANCE.findType(findPtr); } + @Override public void freeNativePath(long pathPtr) { INSTANCE.freeNativePath(pathPtr); } + @Override public int fsync(int fd) { + if (active && fd == watermarkFd && fail("watermark-fsync")) return -1; + return INSTANCE.fsync(fd); + } + @Override public int fsyncDir(String dir) { + if (active && root.equals(dir)) { + dirSyncCalls.incrementAndGet(); + String event = expectPreDirSync ? "dir-fsync" : "post-dir-fsync"; + boolean failed = fail(event); + if (expectPreDirSync) { + if (!failed) { + durableWatermark = true; + expectPreDirSync = false; + } + } else { + expectPreDirSync = true; + if (!failed) durableSegments = false; + } + if (failed) return -1; + } + return INSTANCE.fsyncDir(dir); + } + @Override public long length(int fd) { return INSTANCE.length(fd); } + @Override public long length(String path) { return INSTANCE.length(path); } + @Override public long length(long pathPtr) { return INSTANCE.length(pathPtr); } + @Override public int lock(int fd) { return INSTANCE.lock(fd); } + @Override public int mkdir(String path, int mode) { return INSTANCE.mkdir(path, mode); } + @Override public int msync(long addr, long len, boolean async) { + if (active) { + msyncCalls.incrementAndGet(); + if (fail("watermark-msync")) return -1; + } + return INSTANCE.msync(addr, len, async); + } + @Override public int openCleanRW(String path) { + int fd = INSTANCE.openCleanRW(path); + if (path.equals(root + "/" + AckWatermark.FILE_NAME)) watermarkFd = fd; + return fd; + } + @Override public int openCleanRW(long pathPtr) { return INSTANCE.openCleanRW(pathPtr); } + @Override public int openRW(String path) { + int fd = INSTANCE.openRW(path); + if (path.equals(root + "/" + AckWatermark.FILE_NAME)) watermarkFd = fd; + return fd; + } + @Override public int openRW(long pathPtr) { return INSTANCE.openRW(pathPtr); } + @Override public long read(int fd, long addr, long len, long offset) { return INSTANCE.read(fd, addr, len, offset); } + @Override public boolean remove(String path) { + if (active && path.endsWith(".sfa")) { + removeCalls.incrementAndGet(); + if (fail("segment-remove")) return false; + } + return INSTANCE.remove(path); + } + @Override public boolean remove(long pathPtr) { return INSTANCE.remove(pathPtr); } + @Override public int rename(String oldPath, String newPath) { return INSTANCE.rename(oldPath, newPath); } + @Override public boolean truncate(int fd, long size) { return INSTANCE.truncate(fd, size); } + @Override public long write(int fd, long addr, long len, long offset) { return INSTANCE.write(fd, addr, len, offset); } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerManifestFsyncTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerManifestFsyncTest.java new file mode 100644 index 00000000..8b45567b --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerManifestFsyncTest.java @@ -0,0 +1,189 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.qwp.client.sf.cursor.AckWatermark; +import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentRing; +import io.questdb.client.std.FilesFacade; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Regression test for manifest fsync amplification on the disk-trim path. + * A single trim pass covering N fully-acked sealed segments must commit the + * manifest head advance exactly once (one write + one fsync past the LAST + * batch member before any unlink), not once per trimmed segment. Recovery + * discards files "stale below head", so the single commit is byte-identical + * crash recovery — the per-segment commits are pure IO amplification. + */ +public class SegmentManagerManifestFsyncTest { + private static final String MANIFEST_NAME = "sf-manifest.bin"; + private static final long SEGMENT_SIZE = 64 * 1024; + private String tmpDir; + + @Before + public void setUp() { + tmpDir = TestUtils.createTmpDir("qdb-sf-manifest-fsync-"); + } + + @After + public void tearDown() { + TestUtils.removeTmpDir(tmpDir); + } + + @Test(timeout = 15_000L) + public void testTrimPassCommitsManifestHeadOncePerBatch() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // A contiguous chain of four 1-frame segments: three sealed + // (seqs 0..2) plus the active tail (seq 3). + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 1); + writeSegmentWithFrames(tmpDir + "/sf-0000000000000001.sfa", 1, 1); + writeSegmentWithFrames(tmpDir + "/sf-0000000000000002.sfa", 2, 1); + writeSegmentWithFrames(tmpDir + "/sf-0000000000000003.sfa", 3, 1); + ManifestFsyncCountingFacade ff = new ManifestFsyncCountingFacade(tmpDir); + AckWatermark watermark = null; + SegmentRing ring = null; + try { + ring = SegmentRing.openExisting(ff, tmpDir, SEGMENT_SIZE); + Assert.assertNotNull("legacy chain must recover", ring); + Assert.assertNotNull("recovered ring must expose sealed segments", ring.firstSealed()); + Assert.assertTrue(ring.acknowledge(2)); + watermark = openWatermark(ff, tmpDir); + Assert.assertNotNull(watermark); + try (SegmentManager manager = new SegmentManager( + SEGMENT_SIZE, TimeUnit.SECONDS.toNanos(60), SEGMENT_SIZE * 8L, ff)) { + manager.register(ring, tmpDir, watermark); + ff.active = true; + manager.start(); + awaitTrimmed(ring); + } + Assert.assertEquals( + "a trim pass must commit the manifest head once per batch, not once per segment", + 1, ff.manifestFsyncCalls.get()); + } finally { + if (ring != null) ring.close(); + if (watermark != null) watermark.close(); + } + }); + } + + private static void awaitTrimmed(SegmentRing ring) { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (ring.firstSealed() != null || ring.getPendingTrimCount() != 0) { + if (System.nanoTime() > deadline) { + throw new AssertionError("manager did not trim acknowledged segments"); + } + io.questdb.client.std.Compat.onSpinWait(); + } + } + + private static AckWatermark openWatermark(FilesFacade ff, String root) { + return AckWatermark.open(ff, root); + } + + private static void writeSegmentWithFrames(String path, long baseSeq, int frames) { + long buf = Unsafe.malloc(64, MemoryTag.NATIVE_DEFAULT); + try { + MmapSegment segment = MmapSegment.create(path, baseSeq, SEGMENT_SIZE); + try { + for (int i = 0; i < frames; i++) { + for (int b = 0; b < 64; b++) { + Unsafe.getUnsafe().putByte(buf + b, (byte) (i * 31 + b)); + } + Assert.assertTrue("test frame must fit", segment.tryAppend(buf, 64) >= 0); + } + } finally { + segment.close(); + } + } finally { + Unsafe.free(buf, 64, MemoryTag.NATIVE_DEFAULT); + } + } + + private static final class ManifestFsyncCountingFacade implements FilesFacade { + private final AtomicInteger manifestFsyncCalls = new AtomicInteger(); + private final String manifestPath; + private volatile boolean active; + private volatile int manifestFd = -1; + + private ManifestFsyncCountingFacade(String root) { + this.manifestPath = root + "/" + MANIFEST_NAME; + } + + @Override public boolean allocate(int fd, long size) { return INSTANCE.allocate(fd, size); } + @Override public long allocNativePath(String path) { return INSTANCE.allocNativePath(path); } + @Override public int close(int fd) { return INSTANCE.close(fd); } + @Override public boolean exists(String path) { return INSTANCE.exists(path); } + @Override public void findClose(long findPtr) { INSTANCE.findClose(findPtr); } + @Override public long findFirst(String dir) { return INSTANCE.findFirst(dir); } + @Override public long findName(long findPtr) { return INSTANCE.findName(findPtr); } + @Override public int findNext(long findPtr) { return INSTANCE.findNext(findPtr); } + @Override public int findType(long findPtr) { return INSTANCE.findType(findPtr); } + @Override public void freeNativePath(long pathPtr) { INSTANCE.freeNativePath(pathPtr); } + @Override public int fsync(int fd) { + if (active && fd == manifestFd) { + manifestFsyncCalls.incrementAndGet(); + } + return INSTANCE.fsync(fd); + } + @Override public int fsyncDir(String dir) { return INSTANCE.fsyncDir(dir); } + @Override public long length(int fd) { return INSTANCE.length(fd); } + @Override public long length(String path) { return INSTANCE.length(path); } + @Override public long length(long pathPtr) { return INSTANCE.length(pathPtr); } + @Override public int lock(int fd) { return INSTANCE.lock(fd); } + @Override public int mkdir(String path, int mode) { return INSTANCE.mkdir(path, mode); } + @Override public int msync(long addr, long len, boolean async) { return INSTANCE.msync(addr, len, async); } + @Override public int openCleanRW(String path) { return INSTANCE.openCleanRW(path); } + @Override public int openCleanRW(long pathPtr) { return INSTANCE.openCleanRW(pathPtr); } + @Override public int openRW(String path) { + int fd = INSTANCE.openRW(path); + if (manifestPath.equals(path)) manifestFd = fd; + return fd; + } + @Override public int openRW(long pathPtr) { return INSTANCE.openRW(pathPtr); } + @Override public int openRWExclusive(String path) { + int fd = INSTANCE.openRWExclusive(path); + if (manifestPath.equals(path)) manifestFd = fd; + return fd; + } + @Override public int openRWExclusive(long pathPtr) { return INSTANCE.openRWExclusive(pathPtr); } + @Override public long read(int fd, long addr, long len, long offset) { return INSTANCE.read(fd, addr, len, offset); } + @Override public boolean remove(String path) { return INSTANCE.remove(path); } + @Override public boolean remove(long pathPtr) { return INSTANCE.remove(pathPtr); } + @Override public int rename(String oldPath, String newPath) { return INSTANCE.rename(oldPath, newPath); } + @Override public boolean truncate(int fd, long size) { return INSTANCE.truncate(fd, size); } + @Override public long write(int fd, long addr, long len, long offset) { return INSTANCE.write(fd, addr, len, offset); } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerPassBarrierBenchmark.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerPassBarrierBenchmark.java new file mode 100644 index 00000000..1bb6b899 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerPassBarrierBenchmark.java @@ -0,0 +1,131 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; + +/** + * Standalone comparison of the service-pass ownership barriers used by + * SegmentManager before and after its per-entry state change. This deliberately + * has no pass/fail threshold: it reports target-JVM costs while the production + * state machine and lifecycle tests establish correctness. + * + * 
      + * mvn -pl core test-compile
+ * mvn -pl core exec:java -Dexec.classpathScope=test \
+ *   -Dexec.mainClass=io.questdb.client.test.cutlass.qwp.client.sf.cursor.SegmentManagerPassBarrierBenchmark \
+ *   -Dexec.args="--passes=10000000"
+ *
      + */ +public final class SegmentManagerPassBarrierBenchmark { + + private static final Object MONITOR = new Object(); + private static final AtomicIntegerFieldUpdater STATE_UPDATER = + AtomicIntegerFieldUpdater.newUpdater(Entry.class, "state"); + private static volatile long checksum; + private static volatile int serviceProbe; + + public static void main(String[] args) { + int passes = 10_000_000; + for (String arg : args) { + if (arg.startsWith("--passes=")) { + passes = Integer.parseInt(arg.substring("--passes=".length())); + } else { + throw new IllegalArgumentException("unknown argument: " + arg); + } + } + if (passes < 1) { + throw new IllegalArgumentException("passes must be positive"); + } + + int warmup = Math.max(100_000, passes / 10); + for (int rings : new int[]{1, 32, 256}) { + measureMonitor(rings, warmup); + measureAtomic(rings, warmup); + long monitorNanos = measureMonitor(rings, passes); + long atomicNanos = measureAtomic(rings, passes); + System.out.printf( + "rings=%d passes=%d monitor(two enters)=%.2f ns/pass atomic(two CAS)=%.2f ns/pass ratio=%.2f%n", + rings, + passes, + (double) monitorNanos / passes, + (double) atomicNanos / passes, + (double) monitorNanos / atomicNanos); + } + System.out.println("checksum=" + checksum); + } + + private static Entry[] entries(int count) { + Entry[] entries = new Entry[count]; + for (int i = 0; i < count; i++) { + entries[i] = new Entry(); + } + return entries; + } + + private static long measureAtomic(int rings, int passes) { + Entry[] entries = entries(rings); + long start = System.nanoTime(); + for (int i = 0; i < passes; i++) { + Entry entry = entries[i % rings]; + if (!STATE_UPDATER.compareAndSet(entry, 0, 1)) { + throw new AssertionError("claim failed"); + } + service(entry); + if (!STATE_UPDATER.compareAndSet(entry, 1, 0)) { + throw new AssertionError("completion failed"); + } + } + checksum = checksum * 31 + entries[passes % rings].state; + return System.nanoTime() - start; + } + + private static long measureMonitor(int rings, int passes) { + Entry[] entries = entries(rings); + long start = System.nanoTime(); + for (int i = 0; i < passes; i++) { + Entry entry = entries[i % rings]; + synchronized (MONITOR) { + entry.state = 1; + } + service(entry); + synchronized (MONITOR) { + entry.state = 0; + } + } + checksum = checksum * 31 + entries[passes % rings].state; + return System.nanoTime() - start; + } + + private static void service(Entry entry) { + // Models the non-trivial serviceRing0 call between claim and complete + // and prevents the JVM from coarsening both monitor regions into one. + serviceProbe = entry.state; + } + + private static final class Entry { + volatile int state; + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerPeriodicSyncTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerPeriodicSyncTest.java new file mode 100644 index 00000000..83604657 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerPeriodicSyncTest.java @@ -0,0 +1,748 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; +import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegmentException; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentRing; +import io.questdb.client.std.FilesFacade; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Test; + +import java.util.concurrent.atomic.AtomicLong; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class SegmentManagerPeriodicSyncTest { + + @Test + public void testDeadlineAndFailurePropagation() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final long intervalNanos = 100L; + final long segmentSize = 4096L; + AtomicLong ticks = new AtomicLong(); + CountingFilesFacade filesFacade = new CountingFilesFacade(); + String dir = TestUtils.createTmpDir("qdb-periodic-manager-"); + long payload = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + SegmentManager manager = null; + SegmentRing ring = null; + try { + MmapSegment active = MmapSegment.create( + filesFacade, dir + "/active.sfa", 0L, segmentSize); + ring = new SegmentRing(active, segmentSize); + ring.installHotSpare(MmapSegment.create( + filesFacade, dir + "/spare.sfa", 1L, segmentSize)); + assertEquals(0L, ring.appendOrFsn(payload, 16)); + + manager = new SegmentManager( + segmentSize, + SegmentManager.DEFAULT_POLL_NANOS, + segmentSize * 4L, + filesFacade, + ticks::get); + manager.register(ring, dir, null, intervalNanos); + + manager.serviceRingForTesting(ring); + assertTrue(active.isPublishedDurable()); + assertEquals(1, filesFacade.msyncCalls); + assertEquals(1, filesFacade.fsyncCalls); + + assertEquals(1L, ring.appendOrFsn(payload, 16)); + ticks.set(intervalNanos - 1L); + manager.serviceRingForTesting(ring); + assertEquals("checkpoint ran before its deadline", 1, filesFacade.msyncCalls); + assertEquals("checkpoint ran before its deadline", 1, filesFacade.fsyncCalls); + + ticks.set(intervalNanos); + manager.serviceRingForTesting(ring); + assertEquals(2, filesFacade.msyncCalls); + assertEquals(2, filesFacade.fsyncCalls); + + assertEquals(2L, ring.appendOrFsn(payload, 16)); + filesFacade.isFsyncFailureEnabled = true; + ticks.set(intervalNanos * 2L); + manager.serviceRingForTesting(ring); + assertEquals(3, filesFacade.msyncCalls); + assertEquals(3, filesFacade.fsyncCalls); + try { + ring.appendOrFsn(payload, 16); + fail("expected manager data-sync failure to reach the producer"); + } catch (MmapSegmentException expected) { + assertTrue(expected.getMessage().contains("sync segment file")); + } + assertEquals("failed append must not enter the ring", 2L, ring.publishedFsn()); + } finally { + if (manager != null && ring != null) { + manager.deregister(ring); + } + if (ring != null) { + ring.close(); + } + if (manager != null) { + manager.close(); + } + Unsafe.free(payload, 16, MemoryTag.NATIVE_DEFAULT); + TestUtils.removeTmpDir(dir); + } + }); + } + + @Test + public void testRotationGateDefersRotationUntilPredecessorDurable() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // The PERIODIC-mode rotation gate (requestSyncBeforeRotation): + // rotation must NOT seal a predecessor whose published range is + // not yet durable. This is the exact gate commit 88d6b792 + // accidentally shipped neutralized (`return false; // MUTANT`) + // with the whole suite staying green -- this test is the mutant + // killer. Three independent kill points: + // 1. the rotating append must return BACKPRESSURE_NO_SPARE + // while the predecessor is non-durable (a neutralized gate + // rotates immediately and returns the FSN); + // 2. the gate's sync request must run the barrier on the very + // next service pass BEFORE the interval deadline (a gate + // that fails to set syncRequested leaves the pass idle); + // 3. after the barrier, the retried append must rotate. + final long intervalNanos = 100L; + // Exactly two 16-byte frames fit: header 24 + 2 * (8 + 16) = 72. + final long segmentSize = MmapSegment.HEADER_SIZE + + 2 * (MmapSegment.FRAME_HEADER_SIZE + 16); + AtomicLong ticks = new AtomicLong(); + CountingFilesFacade filesFacade = new CountingFilesFacade(); + String dir = TestUtils.createTmpDir("qdb-periodic-gate-"); + long payload = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + SegmentManager manager = null; + SegmentRing ring = null; + try { + MmapSegment active = MmapSegment.create( + filesFacade, dir + "/active.sfa", 0L, segmentSize); + ring = new SegmentRing(active, segmentSize); + ring.installHotSpare(MmapSegment.create( + filesFacade, dir + "/spare.sfa", 2L, segmentSize)); + manager = new SegmentManager( + segmentSize, + SegmentManager.DEFAULT_POLL_NANOS, + segmentSize * 4L, + filesFacade, + ticks::get); + manager.register(ring, dir, null, intervalNanos); + + assertEquals(0L, ring.appendOrFsn(payload, 16)); + manager.serviceRingForTesting(ring); + assertTrue("first pass must leave the active durable", active.isPublishedDurable()); + assertEquals(1, filesFacade.msyncCalls); + assertEquals(1, filesFacade.fsyncCalls); + + // Fill the segment; the second frame is published but NOT + // yet durable. + assertEquals(1L, ring.appendOrFsn(payload, 16)); + assertTrue("segment must be full", active.isFull()); + assertTrue("published range must be ahead of the durable cursor", + !active.isPublishedDurable()); + + // Kill point 1: the gate must refuse to rotate and + // backpressure the producer instead. A spare IS installed, + // so this return can only come from the durability gate. + assertEquals("rotation must be deferred while the predecessor's " + + "published range is not durable", + SegmentRing.BACKPRESSURE_NO_SPARE, ring.appendOrFsn(payload, 16)); + + // Kill point 2: the deadline (tick 100) has NOT been + // reached, so this pass runs the barrier only because the + // gate requested it. + manager.serviceRingForTesting(ring); + assertEquals("gate-requested barrier must run before the deadline", + 2, filesFacade.msyncCalls); + assertEquals("gate-requested barrier must run before the deadline", + 2, filesFacade.fsyncCalls); + assertTrue(active.isPublishedDurable()); + + // Kill point 3: with the predecessor durable the retried + // append rotates into the spare. + assertEquals(2L, ring.appendOrFsn(payload, 16)); + assertEquals(1, ring.getSealedSegments().size()); + assertEquals(2L, ring.getActive().baseSeq()); + } finally { + if (manager != null && ring != null) { + manager.deregister(ring); + } + if (ring != null) { + ring.close(); + } + if (manager != null) { + manager.close(); + } + Unsafe.free(payload, 16, MemoryTag.NATIVE_DEFAULT); + TestUtils.removeTmpDir(dir); + } + }); + } + + @Test + public void testSyncPassStopsAtFirstFailureThenRetryCoversAllSegments() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // servicePeriodicSync barriers every LIVE segment (sealed first, + // active last) and aborts at the first failure. With >= 2 + // non-durable segments a mid-pass failure must skip the later + // segments, latch the producer, and the healed retry must cover + // EVERY segment before the latch clears. Recovered segments are + // the deterministic way to hold two non-durable live segments: + // recovery constructs them with durableCursor at the header, and + // syncPublished() skips already-durable segments. + final long intervalNanos = 100L; + final long segmentSize = MmapSegment.HEADER_SIZE + + 2 * (MmapSegment.FRAME_HEADER_SIZE + 16); + AtomicLong ticks = new AtomicLong(); + CountingFilesFacade filesFacade = new CountingFilesFacade(); + String dir = TestUtils.createTmpDir("qdb-periodic-multiseg-"); + long payload = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + SegmentManager manager = null; + SegmentRing ring = null; + try { + // Sealed chain member (full: FSNs 0..1) + active (FSN 2). + try (MmapSegment s0 = MmapSegment.create( + filesFacade, dir + "/r0.sfa", 0L, segmentSize)) { + s0.tryAppend(payload, 16); + s0.tryAppend(payload, 16); + s0.msync(); + } + try (MmapSegment s1 = MmapSegment.create( + filesFacade, dir + "/r1.sfa", 2L, segmentSize)) { + s1.tryAppend(payload, 16); + s1.msync(); + } + ring = SegmentRing.openExisting(filesFacade, dir, segmentSize); + assertTrue(ring != null); + assertEquals(1, ring.getSealedSegments().size()); + // Pre-install the hot spare so the service pass does not + // provision one mid-test: its header write would pollute the + // exact barrier-call accounting below. + ring.installHotSpare(MmapSegment.create( + filesFacade, dir + "/spare.sfa", 3L, segmentSize)); + + manager = new SegmentManager( + segmentSize, + SegmentManager.DEFAULT_POLL_NANOS, + segmentSize * 8L, + filesFacade, + ticks::get); + manager.register(ring, dir, null, intervalNanos); + + // First pass with the disk failing: the sealed member's + // barrier is attempted first (msync succeeds, fsync fails) + // and the pass must STOP there -- the active's barrier must + // not run after a failure. + int msyncBefore = filesFacade.msyncCalls; + int fsyncBefore = filesFacade.fsyncCalls; + filesFacade.isFsyncFailureEnabled = true; + manager.serviceRingForTesting(ring); + assertEquals("only the first (sealed) segment's barrier may be attempted", + msyncBefore + 1, filesFacade.msyncCalls); + assertEquals("the pass must abort at the first fsync failure", + fsyncBefore + 1, filesFacade.fsyncCalls); + try { + ring.appendOrFsn(payload, 16); + fail("mid-pass barrier failure must latch the producer"); + } catch (MmapSegmentException expected) { + assertTrue(expected.getMessage().contains("sync segment file")); + } + + // Heal. The retry (scheduled at now + min(interval, 1s)) + // must cover BOTH segments -- the one that failed and the + // one the aborted pass never reached -- before unlatching. + filesFacade.isFsyncFailureEnabled = false; + msyncBefore = filesFacade.msyncCalls; + fsyncBefore = filesFacade.fsyncCalls; + ticks.set(intervalNanos); + manager.serviceRingForTesting(ring); + assertEquals("the healed retry must barrier every live segment", + msyncBefore + 2, filesFacade.msyncCalls); + assertEquals("the healed retry must barrier every live segment", + fsyncBefore + 2, filesFacade.fsyncCalls); + assertEquals("producer must resume once the pass covered all segments", + 3L, ring.appendOrFsn(payload, 16)); + } finally { + if (manager != null && ring != null) { + manager.deregister(ring); + } + if (ring != null) { + ring.close(); + } + if (manager != null) { + manager.close(); + } + Unsafe.free(payload, 16, MemoryTag.NATIVE_DEFAULT); + TestUtils.removeTmpDir(dir); + } + }); + } + + @Test + public void testTransientSyncFailureClearsOnNextSuccess() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final long intervalNanos = 100L; + final long segmentSize = 4096L; + AtomicLong ticks = new AtomicLong(); + CountingFilesFacade filesFacade = new CountingFilesFacade(); + String dir = TestUtils.createTmpDir("qdb-periodic-recovery-"); + long payload = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + SegmentManager manager = null; + SegmentRing ring = null; + try { + MmapSegment active = MmapSegment.create( + filesFacade, dir + "/active.sfa", 0L, segmentSize); + ring = new SegmentRing(active, segmentSize); + ring.installHotSpare(MmapSegment.create( + filesFacade, dir + "/spare.sfa", 1L, segmentSize)); + assertEquals(0L, ring.appendOrFsn(payload, 16)); + + manager = new SegmentManager( + segmentSize, + SegmentManager.DEFAULT_POLL_NANOS, + segmentSize * 4L, + filesFacade, + ticks::get); + manager.register(ring, dir, null, intervalNanos); + + // First tick: initial sync succeeds. + manager.serviceRingForTesting(ring); + assertTrue(active.isPublishedDurable()); + + // One transient fsync failure on the next periodic barrier. + assertEquals(1L, ring.appendOrFsn(payload, 16)); + filesFacade.isFsyncFailureEnabled = true; + ticks.set(intervalNanos); + manager.serviceRingForTesting(ring); + try { + ring.appendOrFsn(payload, 16); + fail("expected the failed data sync to reach the producer"); + } catch (MmapSegmentException expected) { + assertTrue(expected.getMessage().contains("sync segment file")); + } + assertEquals("failed append must not enter the ring", 1L, ring.publishedFsn()); + + // Disk recovers; the manager's retry (scheduled at + // now + min(interval, 1s)) succeeds on the next tick. + filesFacade.isFsyncFailureEnabled = false; + ticks.set(intervalNanos * 2L); + int msyncBefore = filesFacade.msyncCalls; + int fsyncBefore = filesFacade.fsyncCalls; + manager.serviceRingForTesting(ring); + assertTrue("retry barrier must have run msync", filesFacade.msyncCalls > msyncBefore); + assertTrue("retry barrier must have run fsync", filesFacade.fsyncCalls > fsyncBefore); + + // A transient failure must not brick the producer: the retry + // covered the published range, so appends resume. + assertEquals(2L, ring.appendOrFsn(payload, 16)); + assertEquals(2L, ring.publishedFsn()); + } finally { + if (manager != null && ring != null) { + manager.deregister(ring); + } + if (ring != null) { + ring.close(); + } + if (manager != null) { + manager.close(); + } + Unsafe.free(payload, 16, MemoryTag.NATIVE_DEFAULT); + TestUtils.removeTmpDir(dir); + } + }); + } + + @Test + public void testBarrierPinsPublishedRangeAndUnpinsOnSuccess() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final long segmentSize = 4096L; + CountingFilesFacade filesFacade = new CountingFilesFacade(); + String dir = TestUtils.createTmpDir("qdb-periodic-pin-"); + long payload = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + SegmentRing ring = null; + try { + MmapSegment active = MmapSegment.create( + filesFacade, dir + "/active.sfa", 0L, segmentSize); + ring = new SegmentRing(active, segmentSize); + assertEquals(0L, ring.appendOrFsn(payload, 16)); + + active.syncPublished(); + + assertEquals("barrier must pin the not-yet-durable range", 1, filesFacade.mlockCalls); + assertEquals("successful barrier must release the pin", 1, filesFacade.munlockCalls); + // durableCursor starts at HEADER_SIZE, which aligns down to + // page 0, so the pin covers [0, published). + assertEquals("pin must cover the whole not-yet-durable range", + active.publishedOffset(), filesFacade.lastMlockLen); + assertEquals("success path must not re-dirty", 0L, active.redirtyPassesForTest()); + assertTrue(active.isPublishedDurable()); + } finally { + if (ring != null) { + ring.close(); + } + Unsafe.free(payload, 16, MemoryTag.NATIVE_DEFAULT); + TestUtils.removeTmpDir(dir); + } + }); + } + + @Test + public void testFailedBarrierRedirtiesUnderPinBeforeUnlock() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final long segmentSize = 4096L; + CountingFilesFacade filesFacade = new CountingFilesFacade(); + String dir = TestUtils.createTmpDir("qdb-periodic-redirty-"); + long payload = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + SegmentRing ring = null; + try { + MmapSegment active = MmapSegment.create( + filesFacade, dir + "/active.sfa", 0L, segmentSize); + ring = new SegmentRing(active, segmentSize); + assertEquals(0L, ring.appendOrFsn(payload, 16)); + + long[] redirtyAtUnlock = new long[1]; + filesFacade.onMunlock = () -> redirtyAtUnlock[0] = active.redirtyPassesForTest(); + filesFacade.isFsyncFailureEnabled = true; + try { + active.syncPublished(); + fail("expected the fsync failure to surface"); + } catch (MmapSegmentException expected) { + assertTrue(expected.getMessage().contains("sync segment file")); + } + + assertEquals("failed barrier must re-dirty the covered range", + 1L, active.redirtyPassesForTest()); + assertEquals("failed barrier must still release the pin", 1, filesFacade.munlockCalls); + assertEquals("re-dirty must happen BEFORE the pin is released", + 1L, redirtyAtUnlock[0]); + assertFalse("failed barrier must not advance durability", active.isPublishedDurable()); + } finally { + if (ring != null) { + ring.close(); + } + Unsafe.free(payload, 16, MemoryTag.NATIVE_DEFAULT); + TestUtils.removeTmpDir(dir); + } + }); + } + + @Test + public void testConsumedErrorRetryClearsLatchOnlyOverRedirtiedPages() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final long intervalNanos = 100L; + final long segmentSize = 4096L; + AtomicLong ticks = new AtomicLong(); + CountingFilesFacade filesFacade = new CountingFilesFacade(); + String dir = TestUtils.createTmpDir("qdb-periodic-fsyncgate-"); + long payload = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + SegmentManager manager = null; + SegmentRing ring = null; + try { + MmapSegment active = MmapSegment.create( + filesFacade, dir + "/active.sfa", 0L, segmentSize); + ring = new SegmentRing(active, segmentSize); + ring.installHotSpare(MmapSegment.create( + filesFacade, dir + "/spare.sfa", 1L, segmentSize)); + assertEquals(0L, ring.appendOrFsn(payload, 16)); + + manager = new SegmentManager( + segmentSize, + SegmentManager.DEFAULT_POLL_NANOS, + segmentSize * 4L, + filesFacade, + ticks::get); + manager.register(ring, dir, null, intervalNanos); + + // First tick: initial sync genuinely succeeds. + manager.serviceRingForTesting(ring); + assertTrue(active.isPublishedDurable()); + + // fsyncgate model: the next barrier's fsync fails once; from + // then on the facade behaves like the real kernel after EIO -- + // pages clean, error consumed -- returning 0 from msync/fsync + // WITHOUT persisting anything. + assertEquals(1L, ring.appendOrFsn(payload, 16)); + filesFacade.isFsyncGateModeEnabled = true; + ticks.set(intervalNanos); + manager.serviceRingForTesting(ring); + try { + ring.appendOrFsn(payload, 16); + fail("expected the failed data sync to latch the producer"); + } catch (MmapSegmentException expected) { + } + assertEquals("the failed barrier must have re-dirtied its range before any vacuous retry", + 1L, active.redirtyPassesForTest()); + + // Retry pass: the facade's vacuous 0 is backed by genuinely + // re-dirtied pages, so unlatching is honest. Without the + // re-dirty (the C-1 mutant) this scenario is exactly the + // unsound clear: latch gone, durableCursor advanced, nothing + // persisted and no dirty page left for any future barrier. + ticks.set(intervalNanos * 2L); + manager.serviceRingForTesting(ring); + assertTrue(active.isPublishedDurable()); + assertEquals("producer must resume after the covered retry", + 2L, ring.appendOrFsn(payload, 16)); + } finally { + if (manager != null && ring != null) { + manager.deregister(ring); + } + if (ring != null) { + ring.close(); + } + if (manager != null) { + manager.close(); + } + Unsafe.free(payload, 16, MemoryTag.NATIVE_DEFAULT); + TestUtils.removeTmpDir(dir); + } + }); + } + + @Test + public void testMlockRefusalDegradesWithoutAffectingBarrier() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final long segmentSize = 4096L; + CountingFilesFacade filesFacade = new CountingFilesFacade(); + filesFacade.isMlockRefusalEnabled = true; + String dir = TestUtils.createTmpDir("qdb-periodic-mlock-refusal-"); + long payload = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + SegmentRing ring = null; + try { + MmapSegment active = MmapSegment.create( + filesFacade, dir + "/active.sfa", 0L, segmentSize); + ring = new SegmentRing(active, segmentSize); + assertEquals(0L, ring.appendOrFsn(payload, 16)); + + int msyncBefore = filesFacade.msyncCalls; + int fsyncBefore = filesFacade.fsyncCalls; + active.syncPublished(); + + assertEquals("refused pin must not skip the mapping barrier", + msyncBefore + 1, filesFacade.msyncCalls); + assertEquals("refused pin must not skip the fd barrier", + fsyncBefore + 1, filesFacade.fsyncCalls); + assertEquals(1, filesFacade.mlockCalls); + assertEquals("a refused pin must not be unlocked", 0, filesFacade.munlockCalls); + assertTrue("refusal must not affect the barrier outcome", active.isPublishedDurable()); + assertEquals(0L, active.redirtyPassesForTest()); + assertEquals("producer must remain unaffected", 1L, ring.appendOrFsn(payload, 16)); + } finally { + if (ring != null) { + ring.close(); + } + Unsafe.free(payload, 16, MemoryTag.NATIVE_DEFAULT); + TestUtils.removeTmpDir(dir); + } + }); + } + + private static final class CountingFilesFacade implements FilesFacade { + private boolean isFsyncFailureEnabled; + // fsyncgate model: the first fsync fails; afterwards every msync and + // fsync returns 0 WITHOUT delegating to the real syscall -- the + // kernel-accurate shape of a vacuous retry after a consumed EIO + // (clean pages, seen errseq cursor). + private boolean isFsyncGateModeEnabled; + private boolean fsyncGateErrorConsumed; + private boolean isMlockRefusalEnabled; + private int fsyncCalls; + private int msyncCalls; + private int mlockCalls; + private int munlockCalls; + private long lastMlockLen; + private Runnable onMunlock; + + @Override + public boolean allocate(int fd, long size) { + return INSTANCE.allocate(fd, size); + } + + @Override + public long allocNativePath(String path) { + return INSTANCE.allocNativePath(path); + } + + @Override + public int close(int fd) { + return INSTANCE.close(fd); + } + + @Override + public boolean exists(String path) { + return INSTANCE.exists(path); + } + + @Override + public void findClose(long findPtr) { + INSTANCE.findClose(findPtr); + } + + @Override + public long findFirst(String dir) { + return INSTANCE.findFirst(dir); + } + + @Override + public long findName(long findPtr) { + return INSTANCE.findName(findPtr); + } + + @Override + public int findNext(long findPtr) { + return INSTANCE.findNext(findPtr); + } + + @Override + public int findType(long findPtr) { + return INSTANCE.findType(findPtr); + } + + @Override + public void freeNativePath(long pathPtr) { + INSTANCE.freeNativePath(pathPtr); + } + + @Override + public int fsync(int fd) { + fsyncCalls++; + if (isFsyncGateModeEnabled) { + if (!fsyncGateErrorConsumed) { + fsyncGateErrorConsumed = true; + return -1; + } + return 0; + } + return isFsyncFailureEnabled ? -1 : INSTANCE.fsync(fd); + } + + @Override + public long length(int fd) { + return INSTANCE.length(fd); + } + + @Override + public long length(String path) { + return INSTANCE.length(path); + } + + @Override + public long length(long pathPtr) { + return INSTANCE.length(pathPtr); + } + + @Override + public int lock(int fd) { + return INSTANCE.lock(fd); + } + + @Override + public int mkdir(String path, int mode) { + return INSTANCE.mkdir(path, mode); + } + + @Override + public int mlock(long addr, long len) { + mlockCalls++; + lastMlockLen = len; + return isMlockRefusalEnabled ? -1 : 0; + } + + @Override + public int msync(long addr, long len, boolean async) { + msyncCalls++; + if (isFsyncGateModeEnabled && fsyncGateErrorConsumed) { + // consumed-error semantics: no dirty pages, seen errseq -> 0 + return 0; + } + return INSTANCE.msync(addr, len, async); + } + + @Override + public int munlock(long addr, long len) { + munlockCalls++; + Runnable hook = onMunlock; + if (hook != null) { + hook.run(); + } + return 0; + } + + @Override + public int openCleanRW(String path) { + return INSTANCE.openCleanRW(path); + } + + @Override + public int openCleanRW(long pathPtr) { + return INSTANCE.openCleanRW(pathPtr); + } + + @Override + public int openRW(String path) { + return INSTANCE.openRW(path); + } + + @Override + public int openRW(long pathPtr) { + return INSTANCE.openRW(pathPtr); + } + + @Override + public long read(int fd, long addr, long len, long offset) { + return INSTANCE.read(fd, addr, len, offset); + } + + @Override + public boolean remove(String path) { + return INSTANCE.remove(path); + } + + @Override + public boolean remove(long pathPtr) { + return INSTANCE.remove(pathPtr); + } + + @Override + public int rename(String oldPath, String newPath) { + return INSTANCE.rename(oldPath, newPath); + } + + @Override + public boolean truncate(int fd, long size) { + return INSTANCE.truncate(fd, size); + } + + @Override + public long write(int fd, long addr, long len, long offset) { + return INSTANCE.write(fd, addr, len, offset); + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerPinnedTrimTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerPinnedTrimTest.java new file mode 100644 index 00000000..8e8a90ab --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerPinnedTrimTest.java @@ -0,0 +1,91 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + *******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentRing; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +public class SegmentManagerPinnedTrimTest { + + @Test(timeout = 15_000L) + public void testPinnedActiveSurvivesRotationUntilIoRelease() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long segmentSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 1L; + long payload = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT); + SegmentRing ring = null; + try { + ring = new SegmentRing(MmapSegment.createInMemory(0L, segmentSize), segmentSize); + Unsafe.getUnsafe().putByte(payload, (byte) 1); + Assert.assertEquals(0L, ring.appendOrFsn(payload, 1)); + MmapSegment pinned = ring.pinSegmentContainingForTest(0L); + Assert.assertNotNull(pinned); + Assert.assertTrue(ring.acknowledge(0L)); + + ring.installHotSpare(MmapSegment.createInMemory(1L, segmentSize)); + Assert.assertEquals(1L, ring.appendOrFsn(payload, 1)); + Assert.assertSame(pinned, ring.firstSealed()); + + CountDownLatch trimPass = new CountDownLatch(1); + try (SegmentManager manager = new SegmentManager( + segmentSize, TimeUnit.SECONDS.toNanos(60), segmentSize * 8L)) { + manager.setBeforeTrimSyncHook(trimPass::countDown); + manager.register(ring, null); + manager.start(); + Assert.assertTrue("manager did not attempt trim", + trimPass.await(5, TimeUnit.SECONDS)); + } + + Assert.assertSame("manager trimmed the I/O-pinned segment", + pinned, ring.firstSealed()); + Assert.assertNotEquals("manager freed the I/O-pinned mapping", + 0L, pinned.address()); + Assert.assertEquals(0, ring.getPendingTrimCount()); + + ring.releasePinnedSegmentForTest(pinned); + try (SegmentManager manager = new SegmentManager( + segmentSize, TimeUnit.SECONDS.toNanos(60), segmentSize * 8L)) { + manager.register(ring, null); + manager.start(); + awaitTrimmed(ring); + } + Assert.assertEquals("released segment mapping was not freed", + 0L, pinned.address()); + } finally { + if (ring != null) { + ring.close(); + } + Unsafe.free(payload, 1, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + private static void awaitTrimmed(SegmentRing ring) { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (ring.firstSealed() != null || ring.getPendingTrimCount() != 0) { + if (System.nanoTime() > deadline) { + throw new AssertionError("manager did not trim the released segment"); + } + io.questdb.client.std.Compat.onSpinWait(); + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerTest.java index 0c102fce..a00faea4 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerTest.java @@ -24,6 +24,7 @@ package io.questdb.client.test.cutlass.qwp.client.sf.cursor; +import io.questdb.client.cutlass.qwp.client.sf.cursor.AckWatermark; import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentRing; @@ -36,6 +37,10 @@ import org.junit.Test; import java.nio.file.Paths; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; @@ -73,6 +78,65 @@ public void tearDown() { Files.remove(tmpDir); } + @Test + public void testAckBatchDoesNotDelaySiblingSpareProvisioning() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final int sealedCount = 128; + long segSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 1L; + long buf = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT); + CountDownLatch busyDrained = new CountDownLatch(1); + CountDownLatch siblingInstall = new CountDownLatch(1); + AtomicInteger trimPass = new AtomicInteger(); + AtomicReference failure = new AtomicReference<>(); + try (SegmentRing busyRing = new SegmentRing(MmapSegment.createInMemory(0, segSize), segSize); + SegmentRing siblingRing = new SegmentRing(MmapSegment.createInMemory(0, segSize), segSize); + SegmentManager manager = new SegmentManager(segSize, TimeUnit.SECONDS.toNanos(60))) { + for (int i = 0; i < sealedCount; i++) { + assertEquals(i, busyRing.appendOrFsn(buf, 1)); + busyRing.installHotSpare(MmapSegment.createInMemory(busyRing.nextSeqHint(), segSize)); + } + assertEquals(sealedCount, busyRing.appendOrFsn(buf, 1)); + busyRing.installHotSpare(MmapSegment.createInMemory(busyRing.nextSeqHint(), segSize)); + busyRing.acknowledge(sealedCount - 1L); + + manager.register(busyRing, null); + manager.register(siblingRing, null); + manager.setBeforeInstallSyncHook(() -> { + try { + assertEquals("the busy ring must stop at the 64-segment quantum before sibling service", + 64, busyRing.getSealedSegments().size()); + } catch (Throwable t) { + failure.compareAndSet(null, t); + } finally { + siblingInstall.countDown(); + } + }); + manager.setBeforeTrimSyncHook(() -> { + if (trimPass.incrementAndGet() == 4) { + try { + assertEquals("the next fair pass must fully drain the busy ring", + 0, busyRing.getSealedSegments().size()); + } catch (Throwable t) { + failure.compareAndSet(null, t); + } finally { + busyDrained.countDown(); + } + } + }); + manager.start(); + assertTrue("manager did not reach sibling spare provisioning", + siblingInstall.await(10, TimeUnit.SECONDS)); + assertTrue("manager parked instead of immediately rescheduling the remaining ACK batch", + busyDrained.await(10, TimeUnit.SECONDS)); + if (failure.get() != null) { + throw new AssertionError("ACK batch fairness witness failed", failure.get()); + } + } finally { + Unsafe.free(buf, 1, MemoryTag.NATIVE_DEFAULT); + } + }); + } + @Test public void testManagerProvisionsSpareWithinPollingTick() throws Exception { TestUtils.assertMemoryLeak(() -> { @@ -136,10 +200,12 @@ public void testManagerTrimsAckedSegmentFiles() throws Exception { String seg0Path = tmpDir + "/0000000000000000.sfa"; MmapSegment seg0 = MmapSegment.create(seg0Path, 0, segSize); long buf = Unsafe.malloc(32, MemoryTag.NATIVE_DEFAULT); - try (SegmentRing ring = new SegmentRing(seg0, segSize); + try (AckWatermark watermark = AckWatermark.open(tmpDir); + SegmentRing ring = new SegmentRing(seg0, segSize); SegmentManager mgr = new SegmentManager(segSize, 200_000L)) { + assertTrue("watermark must open", watermark != null); mgr.start(); - mgr.register(ring, tmpDir); + mgr.register(ring, tmpDir, watermark); // Fill seg0 (2 frames) and force rotation by appending a third. for (int i = 0; i < 2; i++) ring.appendOrFsn(buf, 32); @@ -170,11 +236,13 @@ public void testMaxTotalBytesCapBlocksProvisioningUntilTrimFrees() throws Except long cap = 3 * segSize; MmapSegment seg0 = MmapSegment.create(tmpDir + "/0000000000000000.sfa", 0, segSize); long buf = Unsafe.malloc(64, MemoryTag.NATIVE_DEFAULT); - try (SegmentRing ring = new SegmentRing(seg0, segSize); + try (AckWatermark watermark = AckWatermark.open(tmpDir); + SegmentRing ring = new SegmentRing(seg0, segSize); SegmentManager mgr = new SegmentManager(segSize, 200_000L, cap)) { + assertTrue("watermark must open", watermark != null); mgr.start(); // register seeds totalBytes = 1*segSize (initial active). - mgr.register(ring, tmpDir); + mgr.register(ring, tmpDir, watermark); // Manager provisions spare 1 → totalBytes = 2*segSize. assertTrue(waitFor(() -> !ring.needsHotSpare(), 2000)); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerTotalBytesRaceTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerTotalBytesRaceTest.java index 644b2709..15b8e8a6 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerTotalBytesRaceTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerTotalBytesRaceTest.java @@ -33,7 +33,6 @@ import org.junit.Before; import org.junit.Test; -import java.lang.reflect.Field; import java.nio.file.Paths; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -193,15 +192,8 @@ private static void awaitParked(Thread t) { } } - private static long readTotalBytes(SegmentManager mgr) throws Exception { - Field f = SegmentManager.class.getDeclaredField("totalBytes"); - f.setAccessible(true); - Field lockF = SegmentManager.class.getDeclaredField("lock"); - lockF.setAccessible(true); - Object lock = lockF.get(mgr); - synchronized (lock) { - return f.getLong(mgr); - } + private static long readTotalBytes(SegmentManager mgr) { + return mgr.getTotalBytesForTesting(); } private static void rmDirRecursive(String dir) { @@ -226,9 +218,7 @@ private static void rmDirRecursive(String dir) { Files.remove(dir); } - private static Thread workerThread(SegmentManager mgr) throws Exception { - Field f = SegmentManager.class.getDeclaredField("workerThread"); - f.setAccessible(true); - return (Thread) f.get(mgr); + private static Thread workerThread(SegmentManager mgr) { + return mgr.getWorkerThreadForTesting(); } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerTrimDeregisterRaceTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerTrimDeregisterRaceTest.java index 8cc05407..e99898af 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerTrimDeregisterRaceTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerTrimDeregisterRaceTest.java @@ -35,7 +35,6 @@ import org.junit.Before; import org.junit.Test; -import java.lang.reflect.Field; import java.nio.file.Paths; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -43,7 +42,6 @@ import java.util.concurrent.atomic.AtomicReference; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; /** @@ -186,11 +184,8 @@ public void testTrimPathDoesNotDoubleSubtractAfterDeregister() throws Exception + "`totalBytes -= sz` on a stillRegistered re-check " + "under the same lock that covers deregister.", 0L, observed); - assertFalse("stale SegmentManager snapshot skipped drainTrimmable() " - + "after deregister and left a fully-acked sealed " - + "segment on disk. The registration guard should " - + "protect watermark/accounting only; trim ownership " - + "transfer must still close and unlink " + activePath, + assertTrue("deregister before the durable barrier must preserve the segment " + + "for owner-side quiescent cleanup " + activePath, Files.exists(activePath)); } finally { mgr.setBeforeTrimSyncHook(null); @@ -227,15 +222,8 @@ private static void awaitSpare(SegmentRing ring, String where) { } } - private static long readTotalBytes(SegmentManager mgr) throws Exception { - Field f = SegmentManager.class.getDeclaredField("totalBytes"); - f.setAccessible(true); - Field lockF = SegmentManager.class.getDeclaredField("lock"); - lockF.setAccessible(true); - Object lock = lockF.get(mgr); - synchronized (lock) { - return f.getLong(mgr); - } + private static long readTotalBytes(SegmentManager mgr) { + return mgr.getTotalBytesForTesting(); } private static void rmDirRecursive(String dir) { @@ -260,9 +248,7 @@ private static void rmDirRecursive(String dir) { Files.remove(dir); } - private static Thread workerThread(SegmentManager mgr) throws Exception { - Field f = SegmentManager.class.getDeclaredField("workerThread"); - f.setAccessible(true); - return (Thread) f.get(mgr); + private static Thread workerThread(SegmentManager mgr) { + return mgr.getWorkerThreadForTesting(); } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerUnlinkFailureTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerUnlinkFailureTest.java new file mode 100644 index 00000000..b9ef46a5 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerUnlinkFailureTest.java @@ -0,0 +1,354 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.qwp.client.sf.cursor.AckWatermark; +import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentRing; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SfOperationalException; +import io.questdb.client.std.Files; +import io.questdb.client.std.FilesFacade; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +public class SegmentManagerUnlinkFailureTest { + + private String tmpDir; + + @Before + public void setUp() { + tmpDir = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-segmgr-unlink-fault-" + System.nanoTime()).toString(); + Assert.assertEquals(0, Files.mkdir(tmpDir, Files.DIR_MODE_DEFAULT)); + } + + @After + public void tearDown() { + if (tmpDir != null) { + removeRecursive(tmpDir); + } + } + + @Test + public void testEnumerationFindNextFailureRefusesGenerationAllocation() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long segmentSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 32L; + String dir = tmpDir + "/enumeration"; + Assert.assertEquals(0, Files.mkdir(dir, Files.DIR_MODE_DEFAULT)); + String lowerName = "sf-0000000000000000.sfa"; + String lowerPath = dir + "/" + lowerName; + String higherPath = dir + "/sf-0000000000000007.sfa"; + MmapSegment lower = MmapSegment.create(lowerPath, 0L, segmentSize); + lower.close(); + MmapSegment initial = MmapSegment.create(higherPath, 0L, segmentSize); + SegmentRing ring = new SegmentRing(initial, segmentSize); + byte[] originalHigher = java.nio.file.Files.readAllBytes(Paths.get(higherPath)); + FailingFilesFacade facade = new FailingFilesFacade(null, dir, lowerName); + try (SegmentManager manager = new SegmentManager( + segmentSize, TimeUnit.SECONDS.toNanos(60), segmentSize * 4, facade)) { + try { + manager.register(ring, dir); + Assert.fail("register accepted a partially enumerated SF directory"); + } catch (SfOperationalException expected) { + Assert.assertTrue(expected.getMessage().contains("could not fully enumerate")); + } + Assert.assertTrue("fault did not occur after the lower generation was observed", + facade.partialLowerObserved); + Assert.assertTrue("failed enumeration cursor was not closed", facade.partialFindClosed); + Assert.assertEquals("enumeration failure must not allocate or truncate a path", + 0, facade.openCleanCalls); + Assert.assertTrue("higher generation disappeared", Files.exists(higherPath)); + Assert.assertArrayEquals("partial enumeration changed the unseen higher generation", + originalHigher, java.nio.file.Files.readAllBytes(Paths.get(higherPath))); + } finally { + ring.close(); + } + }); + } + + @Test(timeout = 15_000L) + public void testFailedUnlinkRetainsBookkeepingAndUsesSuccessorPath() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long segmentSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 32L; + String dir = tmpDir + "/unlink"; + Assert.assertEquals(0, Files.mkdir(dir, Files.DIR_MODE_DEFAULT)); + String failedPath = dir + "/sf-0000000000000000.sfa"; + String activePath = dir + "/sf-0000000000000001.sfa"; + MmapSegment initial = MmapSegment.create(failedPath, 0L, segmentSize); + SegmentRing ring = new SegmentRing(initial, segmentSize); + AckWatermark watermark = AckWatermark.open(dir); + Assert.assertNotNull(watermark); + watermark.write(-1L); + long payload = Unsafe.malloc(32, MemoryTag.NATIVE_DEFAULT); + try { + fill(payload, 32, (byte) 0x11); + Assert.assertEquals(0L, ring.appendOrFsn(payload, 32)); + ring.installHotSpare(MmapSegment.create(activePath, 1L, segmentSize)); + Assert.assertEquals(1L, ring.appendOrFsn(payload, 32)); + ring.acknowledge(0L); + byte[] original = java.nio.file.Files.readAllBytes(Paths.get(failedPath)); + + FailingFilesFacade facade = new FailingFilesFacade(failedPath, null, null); + try (SegmentManager manager = new SegmentManager( + segmentSize, TimeUnit.SECONDS.toNanos(60), segmentSize * 8, facade)) { + manager.register(ring, dir, watermark); + manager.start(); + Assert.assertTrue("manager never attempted the injected unlink", + facade.removeAttempted.await(5, TimeUnit.SECONDS)); + Assert.assertEquals("failed unlink must retain conservative registered bytes", + ring.totalSegmentBytes(), readTotalBytes(manager)); + } + + Assert.assertTrue("failed unlink path must remain observable", Files.exists(failedPath)); + Assert.assertTrue("failed unlink changed the acknowledged segment bytes", + Arrays.equals(original, java.nio.file.Files.readAllBytes(Paths.get(failedPath)))); + Assert.assertNull("failed unlink left a closed segment in live traversal", + ring.firstSealed()); + Assert.assertEquals("failed unlink lost pending cleanup ownership", + 1, ring.getPendingTrimCount()); + Assert.assertEquals("failed unlink must remain covered by the durable cumulative watermark", + 0L, watermark.read()); + + fill(payload, 32, (byte) 0x5A); + Assert.assertEquals("non-DEDUP successor must continue at the next FSN", + 2L, ring.appendOrFsn(payload, 32)); + String successorPath = ring.getActive().path(); + Assert.assertNotEquals("successor reused the acknowledged path", + failedPath, successorPath); + Assert.assertEquals(dir + "/sf-0000000000000002.sfa", successorPath); + Assert.assertTrue("successor segment was not created", Files.exists(successorPath)); + Assert.assertTrue("distinct non-DEDUP payload was not written to the successor", + containsRun(java.nio.file.Files.readAllBytes(Paths.get(successorPath)), + (byte) 0x5A, 32)); + Assert.assertTrue("successor write overwrote the failed-unlink segment", + Arrays.equals(original, java.nio.file.Files.readAllBytes(Paths.get(failedPath)))); + + try (SegmentManager retryManager = new SegmentManager( + segmentSize, TimeUnit.SECONDS.toNanos(60), segmentSize * 8, facade)) { + retryManager.register(ring, dir, watermark); + retryManager.start(); + retryManager.wakeWorker(); + awaitRetryCommit(failedPath, watermark); + } + MmapSegment firstSealed = ring.firstSealed(); + Assert.assertTrue("successful retry retained the acknowledged segment", + firstSealed == null || !failedPath.equals(firstSealed.path())); + Assert.assertEquals("successful retry retained pending cleanup ownership", + 0, ring.getPendingTrimCount()); + } finally { + Unsafe.free(payload, 32, MemoryTag.NATIVE_DEFAULT); + ring.close(); + watermark.close(); + } + }); + } + + private static void awaitRetryCommit(String failedPath, AckWatermark watermark) { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (Files.exists(failedPath) || watermark.read() != 0L) { + if (System.nanoTime() > deadline) { + throw new AssertionError("unlink retry did not remove the file and advance the watermark"); + } + io.questdb.client.std.Compat.onSpinWait(); + } + } + + private static boolean containsRun(byte[] bytes, byte value, int length) { + int run = 0; + for (byte b : bytes) { + run = b == value ? run + 1 : 0; + if (run == length) { + return true; + } + } + return false; + } + + private static void fill(long address, int len, byte value) { + for (int i = 0; i < len; i++) { + Unsafe.getUnsafe().putByte(address + i, value); + } + } + + private static long readTotalBytes(SegmentManager manager) throws Exception { + Field field = SegmentManager.class.getDeclaredField("totalBytes"); + field.setAccessible(true); + Field lockField = SegmentManager.class.getDeclaredField("lock"); + lockField.setAccessible(true); + Object lock = lockField.get(manager); + synchronized (lock) { + return field.getLong(manager); + } + } + + private static void removeRecursive(String dir) { + long find = Files.findFirst(dir); + if (find > 0) { + try { + int rc = 1; + while (rc > 0) { + String name = Files.utf8ToString(Files.findName(find)); + if (name != null && !".".equals(name) && !"..".equals(name)) { + String child = dir + "/" + name; + if (!Files.remove(child)) { + removeRecursive(child); + } + } + rc = Files.findNext(find); + } + } finally { + Files.findClose(find); + } + } + Files.remove(dir); + } + + private static final class FailingFilesFacade implements FilesFacade { + private static final long PARTIAL_FIND_PTR = Long.MAX_VALUE; + private final String enumerationFailureDir; + private final String partialLowerName; + private final String unlinkFailurePath; + private final CountDownLatch removeAttempted = new CountDownLatch(1); + private int openCleanCalls; + private boolean partialFindClosed; + private long partialFindNamePtr; + private boolean partialLowerObserved; + private int unlinkFailuresRemaining = 1; + + private FailingFilesFacade( + String unlinkFailurePath, + String enumerationFailureDir, + String partialLowerName + ) { + this.unlinkFailurePath = unlinkFailurePath; + this.enumerationFailureDir = enumerationFailureDir; + this.partialLowerName = partialLowerName; + } + + @Override + public boolean allocate(int fd, long size) { return INSTANCE.allocate(fd, size); } + @Override + public long allocNativePath(String path) { return INSTANCE.allocNativePath(path); } + @Override + public int close(int fd) { return INSTANCE.close(fd); } + @Override + public boolean exists(String path) { return INSTANCE.exists(path); } + @Override + public void findClose(long findPtr) { + if (findPtr == PARTIAL_FIND_PTR) { + INSTANCE.freeNativePath(partialFindNamePtr); + partialFindNamePtr = 0L; + partialFindClosed = true; + } else { + INSTANCE.findClose(findPtr); + } + } + @Override + public long findFirst(String dir) { + if (dir.equals(enumerationFailureDir)) { + partialFindNamePtr = INSTANCE.allocNativePath(partialLowerName); + return PARTIAL_FIND_PTR; + } + return INSTANCE.findFirst(dir); + } + @Override + public long findName(long findPtr) { + return findPtr == PARTIAL_FIND_PTR ? partialFindNamePtr : INSTANCE.findName(findPtr); + } + @Override + public int findNext(long findPtr) { + if (findPtr == PARTIAL_FIND_PTR) { + partialLowerObserved = true; + return -1; + } + return INSTANCE.findNext(findPtr); + } + @Override + public int findType(long findPtr) { return INSTANCE.findType(findPtr); } + @Override + public void freeNativePath(long pathPtr) { INSTANCE.freeNativePath(pathPtr); } + @Override + public int fsync(int fd) { return INSTANCE.fsync(fd); } + @Override + public long length(int fd) { return INSTANCE.length(fd); } + @Override + public long length(String path) { return INSTANCE.length(path); } + @Override + public long length(long pathPtr) { return INSTANCE.length(pathPtr); } + @Override + public int lock(int fd) { return INSTANCE.lock(fd); } + @Override + public int mkdir(String path, int mode) { return INSTANCE.mkdir(path, mode); } + @Override + public int openCleanRW(String path) { + openCleanCalls++; + return INSTANCE.openCleanRW(path); + } + @Override + public int openCleanRW(long pathPtr) { + openCleanCalls++; + return INSTANCE.openCleanRW(pathPtr); + } + @Override + public int openRW(String path) { return INSTANCE.openRW(path); } + @Override + public int openRW(long pathPtr) { return INSTANCE.openRW(pathPtr); } + @Override + public long read(int fd, long addr, long len, long offset) { + return INSTANCE.read(fd, addr, len, offset); + } + @Override + public boolean remove(String path) { + if (path.equals(unlinkFailurePath) && unlinkFailuresRemaining > 0) { + unlinkFailuresRemaining--; + removeAttempted.countDown(); + return false; + } + return INSTANCE.remove(path); + } + @Override + public boolean remove(long pathPtr) { return INSTANCE.remove(pathPtr); } + @Override + public int rename(String oldPath, String newPath) { return INSTANCE.rename(oldPath, newPath); } + @Override + public boolean truncate(int fd, long size) { return INSTANCE.truncate(fd, size); } + @Override + public long write(int fd, long addr, long len, long offset) { + return INSTANCE.write(fd, addr, len, offset); + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerWatermarkDeregisterRaceTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerWatermarkDeregisterRaceTest.java index 29794b68..605b8def 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerWatermarkDeregisterRaceTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentManagerWatermarkDeregisterRaceTest.java @@ -36,7 +36,6 @@ import org.junit.Before; import org.junit.Test; -import java.lang.reflect.Field; import java.nio.file.Paths; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -133,8 +132,8 @@ public void testStaleWorkerDoesNotWriteThroughUnmappedWatermarkAfterDeregister() // closed=true, masking the original bug's plain-boolean guard. // Releasing the mmap/fd directly leaves the object in the stale // state that a racing worker is allowed to observe. - releaseWatermarkStorageButLeaveObjectWritable(watermark); - watermarkStorageReleased = true; + watermarkStorageReleased = + releaseWatermarkStorageButLeaveObjectWritable(watermark); resumeWorker.countDown(); if (hookErr.get() != null) { throw new AssertionError("install hook failed", hookErr.get()); @@ -154,32 +153,18 @@ public void testStaleWorkerDoesNotWriteThroughUnmappedWatermarkAfterDeregister() } catch (Throwable ignored) { // best-effort } - if (!watermarkStorageReleased) { - try { - watermark.close(); - } catch (Throwable ignored) { - // best-effort - } + try { + watermark.close(); + } catch (Throwable ignored) { + // best-effort } Unsafe.free(buf, 32, MemoryTag.NATIVE_DEFAULT); } }); } - private static void releaseWatermarkStorageButLeaveObjectWritable(AckWatermark watermark) throws Exception { - Field mmapAddressF = AckWatermark.class.getDeclaredField("mmapAddress"); - mmapAddressF.setAccessible(true); - long mmapAddress = mmapAddressF.getLong(watermark); - if (mmapAddress != 0L && mmapAddress != Files.FAILED_MMAP_ADDRESS) { - Files.munmap(mmapAddress, AckWatermark.FILE_SIZE, MemoryTag.MMAP_DEFAULT); - } - - Field fdF = AckWatermark.class.getDeclaredField("fd"); - fdF.setAccessible(true); - int fd = fdF.getInt(watermark); - if (fd >= 0) { - Files.close(fd); - } + private static boolean releaseWatermarkStorageButLeaveObjectWritable(AckWatermark watermark) { + return watermark.releaseStorageButKeepWritableForTesting(); } private static void rmDirRecursive(String dir) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRecoveryIntegrityTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRecoveryIntegrityTest.java new file mode 100644 index 00000000..1b6260b9 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRecoveryIntegrityTest.java @@ -0,0 +1,1055 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; +import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegmentException; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentRing; +import io.questdb.client.std.Crc32c; +import io.questdb.client.std.Files; +import io.questdb.client.std.FilesFacade; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.Map; + +/** + * Regression tests for the SF recovery fail-open findings: directory + * enumeration errors, per-file open/read/mmap errors and boundary (leading / + * trailing) segment loss must fail startup without mutating the slot, while + * positively-identified corruption is quarantined only after the surviving + * chain validates. Also pins the crash-window states of the manifest + * protocol (fresh start, rotation, clean-drain) as recoverable. + */ +public class SegmentRecoveryIntegrityTest { + private static final String MANIFEST_NAME = "sf-manifest.bin"; + private static final long SEGMENT_SIZE = 64 * 1024; + private String tmpDir; + + @Before + public void setUp() { + tmpDir = TestUtils.createTmpDir("qdb-sf-integrity-"); + } + + @After + public void tearDown() { + TestUtils.removeTmpDir(tmpDir); + } + + // ------------------------------------------------------------------ + // Operational failures must fail closed without touching a byte. + // ------------------------------------------------------------------ + + @Test + public void testFindFirstFailureFailsRecoveryAndPreservesBytes() throws Exception { + TestUtils.assertMemoryLeak(() -> { + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 2); + writeSegmentWithFrames(tmpDir + "/sf-0000000000000001.sfa", 2, 3); + Map before = snapshotDir(); + + FilesFacade facade = new DelegatingFacade() { + @Override + public long findFirst(String dir) { + return tmpDir.equals(dir) ? -1L : super.findFirst(dir); + } + }; + try { + SegmentRing.openExisting(facade, tmpDir, SEGMENT_SIZE).close(); + Assert.fail("recovery must fail when the SF directory cannot be enumerated"); + } catch (MmapSegmentException expected) { + TestUtils.assertContains(expected.getMessage(), "could not enumerate"); + } + assertDirUnchanged(before); + }); + } + + @Test + public void testPartialFindNextFailsRecoveryAndPreservesBytes() throws Exception { + TestUtils.assertMemoryLeak(() -> { + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 2); + writeSegmentWithFrames(tmpDir + "/sf-0000000000000001.sfa", 2, 3); + Map before = snapshotDir(); + + FilesFacade facade = new DelegatingFacade() { + private int calls; + + @Override + public int findNext(long findPtr) { + // First advance succeeds, then the listing "fails" the way + // a readdir I/O error does. Recovery must treat the + // partial listing as fatal, not as end-of-directory. + return ++calls >= 2 ? -1 : super.findNext(findPtr); + } + }; + try { + SegmentRing.openExisting(facade, tmpDir, SEGMENT_SIZE).close(); + Assert.fail("recovery must fail when the directory listing is incomplete"); + } catch (MmapSegmentException expected) { + TestUtils.assertContains(expected.getMessage(), "could not fully enumerate"); + } + assertDirUnchanged(before); + }); + } + + @Test + public void testOpenRWFailureOnValidSegmentFailsRecoveryAndPreservesBytes() throws Exception { + TestUtils.assertMemoryLeak(() -> { + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 2); + String victim = tmpDir + "/sf-0000000000000001.sfa"; + writeSegmentWithFrames(victim, 2, 3); + Map before = snapshotDir(); + + FilesFacade facade = new DelegatingFacade() { + @Override + public int openRW(String path) { + // EMFILE/EACCES-style failure on a perfectly valid file. + return victim.equals(path) ? -1 : super.openRW(path); + } + }; + try { + SegmentRing.openExisting(facade, tmpDir, SEGMENT_SIZE).close(); + Assert.fail("recovery must fail when a valid segment cannot be opened"); + } catch (MmapSegmentException expected) { + TestUtils.assertContains(expected.getMessage(), "recovery failed for recognized segment"); + } + assertDirUnchanged(before); + }); + } + + @Test + public void testMmapFailureOnValidSegmentFailsRecoveryAndPreservesBytes() throws Exception { + TestUtils.assertMemoryLeak(() -> { + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 2); + String victim = tmpDir + "/sf-0000000000000001.sfa"; + writeSegmentWithFrames(victim, 2, 3); + Map before = snapshotDir(); + + FilesFacade facade = new DelegatingFacade() { + private int victimFd = Integer.MIN_VALUE; + + @Override + public int openRW(String path) { + int fd = super.openRW(path); + if (victim.equals(path)) { + victimFd = fd; + } + return fd; + } + + @Override + public long mmap(int fd, long len, long offset, int flags, int memoryTag) { + if (fd == victimFd) { + return Files.FAILED_MMAP_ADDRESS; + } + return super.mmap(fd, len, offset, flags, memoryTag); + } + }; + try { + SegmentRing.openExisting(facade, tmpDir, SEGMENT_SIZE).close(); + Assert.fail("recovery must fail when a valid segment cannot be mapped"); + } catch (MmapSegmentException expected) { + TestUtils.assertContains(expected.getMessage(), "recovery failed for recognized segment"); + } + assertDirUnchanged(before); + }); + } + + @Test + public void testReadFailureOnValidSegmentFailsRecoveryAndPreservesBytes() throws Exception { + TestUtils.assertMemoryLeak(() -> { + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 2); + String victim = tmpDir + "/sf-0000000000000001.sfa"; + writeSegmentWithFrames(victim, 2, 3); + Map before = snapshotDir(); + + FilesFacade facade = new DelegatingFacade() { + private int victimFd = Integer.MIN_VALUE; + + @Override + public int openRW(String path) { + int fd = super.openRW(path); + if (victim.equals(path)) { + victimFd = fd; + } + return fd; + } + + @Override + public long read(int fd, long addr, long len, long offset) { + return fd == victimFd ? -1L : super.read(fd, addr, len, offset); + } + }; + try { + SegmentRing.openExisting(facade, tmpDir, SEGMENT_SIZE).close(); + Assert.fail("recovery must fail when a valid segment cannot be read"); + } catch (MmapSegmentException expected) { + TestUtils.assertContains(expected.getMessage(), "recovery failed for recognized segment"); + } + assertDirUnchanged(before); + }); + } + + @Test + public void testUnsupportedVersionFailsRecoveryWithoutQuarantine() throws Exception { + TestUtils.assertMemoryLeak(() -> { + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 2); + String foreign = tmpDir + "/sf-0000000000000001.sfa"; + writeRawSegmentHeader(foreign, MmapSegment.FILE_MAGIC, (byte) 99, 2L); + Map before = snapshotDir(); + + try { + SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE).close(); + Assert.fail("a segment written by a different client version must fail recovery"); + } catch (MmapSegmentException expected) { + TestUtils.assertContains(expected.getMessage(), "recovery failed for recognized segment"); + } + // NOT renamed .corrupt: the file belongs to the client build that + // can read it; recovery keeps the slot intact for that writer. + assertDirUnchanged(before); + }); + } + + // ------------------------------------------------------------------ + // Corruption is quarantined, but never before validation decides. + // ------------------------------------------------------------------ + + @Test + public void testCorruptStrayFileQuarantinedInLegacyDirAndSiblingsRecover() throws Exception { + TestUtils.assertMemoryLeak(() -> { + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 2); + writeSegmentWithFrames(tmpDir + "/sf-0000000000000001.sfa", 2, 3); + String stray = tmpDir + "/zz-stray.sfa"; + writeRawSegmentHeader(stray, 0xDEADBEEF, (byte) 1, 0L); + + SegmentRing ring = SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.assertNotNull("valid chain must recover around a corrupt stray file", ring); + try { + Assert.assertEquals("all five durable frames must be recovered", + 4, ring.publishedFsn()); + } finally { + ring.close(); + } + Assert.assertFalse("corrupt stray must be quarantined away from the .sfa scan", + Files.exists(stray)); + Assert.assertTrue("quarantine preserves the bytes as evidence", + Files.exists(stray + ".corrupt")); + Assert.assertTrue("legacy recovery must migrate the slot to the manifest", + Files.exists(tmpDir + "/" + MANIFEST_NAME)); + }); + } + + @Test + public void testCorruptChainSegmentWithManifestFailsWithoutMutation() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String seg = tmpDir + "/sf-initial.sfa"; + writeSegmentWithFrames(seg, 0, 3); + // Migrate once so the manifest exists and the segment is flagged. + SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE).close(); + Assert.assertTrue(Files.exists(tmpDir + "/" + MANIFEST_NAME)); + // Bit-rot the magic in place. + overwriteInt(seg, 0, 0xBADC0DE); + Map before = snapshotDir(); + + try { + SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.fail("losing the only chain segment to corruption must fail recovery"); + } catch (MmapSegmentException expected) { + TestUtils.assertContains(expected.getMessage(), "corrupt"); + } + // Deferred quarantine: a FAILED recovery must not have renamed, + // deleted, or otherwise mutated anything. + assertDirUnchanged(before); + }); + } + + @Test + public void testFlaggedSegmentWithDeletedManifestFails() throws Exception { + TestUtils.assertMemoryLeak(() -> { + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 3); + SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE).close(); + Assert.assertTrue(Files.remove(tmpDir + "/" + MANIFEST_NAME)); + + try { + SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.fail("a manifest-required segment without a manifest must fail recovery"); + } catch (MmapSegmentException expected) { + TestUtils.assertContains(expected.getMessage(), "missing"); + } + }); + } + + // ------------------------------------------------------------------ + // Boundary evasion: missing leading/trailing segments must be caught. + // ------------------------------------------------------------------ + + @Test + public void testMissingActiveSegmentWithManifestFails() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Chain [0..2) exists but the manifest says the active starts at 2. + // Pre-manifest recovery would silently promote the highest present + // segment and hand out overlapping FSNs. + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 2); + writeManifest(1, 0, 2); + + try { + SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.fail("a missing trailing/active segment must fail recovery"); + } catch (MmapSegmentException expected) { + TestUtils.assertContains(expected.getMessage(), "missing expected SF active"); + } + }); + } + + @Test + public void testMissingHeadSegmentWithManifestFails() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // The manifest promises a head at base 0, but only [2..5) survived. + // Pre-manifest recovery would pass contiguity on the remainder and + // silently lose the leading rows. + writeSegmentWithFrames(tmpDir + "/sf-0000000000000001.sfa", 2, 3); + writeManifest(1, 0, 2); + + try { + SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.fail("a missing leading/head segment must fail recovery"); + } catch (MmapSegmentException expected) { + TestUtils.assertContains(expected.getMessage(), "missing expected SF head"); + } + }); + } + + @Test + public void testInteriorGapStillFailsRecovery() throws Exception { + TestUtils.assertMemoryLeak(() -> { + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 2); + writeSegmentWithFrames(tmpDir + "/sf-0000000000000002.sfa", 5, 2); + writeManifest(1, 0, 5); + + try { + SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.fail("an interior FSN gap must fail recovery"); + } catch (MmapSegmentException expected) { + TestUtils.assertContains(expected.getMessage(), "FSN gap"); + } + }); + } + + // ------------------------------------------------------------------ + // Legal crash-window states must recover without operator action. + // ------------------------------------------------------------------ + + @Test + public void testFreshStartCrashWithTwoEmptyBaseZeroSegmentsRecovers() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Fresh engine start provisions sf-initial plus a hot spare, both + // empty at baseSeq 0. A process kill in that window must not brick + // the slot on "ambiguous" empties. + MmapSegment a = MmapSegment.create(tmpDir + "/sf-initial.sfa", 0, SEGMENT_SIZE); + a.close(); + SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE).close(); + MmapSegment b = MmapSegment.create(tmpDir + "/sf-0000000000000001.sfa", 0, SEGMENT_SIZE); + b.close(); + + SegmentRing ring = SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.assertNotNull("two equivalent empty segments must not brick recovery", ring); + try { + Assert.assertEquals(-1, ring.publishedFsn()); + Assert.assertEquals(0, ring.getActive().baseSeq()); + } finally { + ring.close(); + } + Assert.assertEquals("the redundant empty must have been cleaned up", + 1, countSfaFiles()); + }); + } + + @Test + public void testRotationCrashWindowEmptyActiveAtChainEndRecovers() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Crash after rotation committed (manifest fsync'd, spare header + // synced) but before any frame reached the new active: sealed + // chain [0..2) plus an empty active at base 2. + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 2); + MmapSegment spare = MmapSegment.create(tmpDir + "/sf-0000000000000001.sfa", 2, SEGMENT_SIZE); + spare.close(); + writeManifest(1, 0, 2); + + SegmentRing ring = SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.assertNotNull("freshly-rotated empty active is a legal crash state", ring); + try { + Assert.assertEquals(1, ring.publishedFsn()); + Assert.assertEquals(2, ring.getActive().baseSeq()); + Assert.assertNotNull(ring.firstSealed()); + Assert.assertEquals(0, ring.firstSealed().baseSeq()); + } finally { + ring.close(); + } + }); + } + + @Test + public void testDrainWindowManifestWithoutSegmentsRecoversEmptyAndRemovesManifest() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Clean-drain close durably collapses the boundaries to + // head == active before unlinking, then removes segments and + // finally the manifest; a crash between the last unlink and the + // manifest removal leaves collapsed boundaries with no files. + writeManifest(3, 9, 9); + + SegmentRing ring = SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.assertNull("segment-less slot must recover as EMPTY", ring); + Assert.assertFalse("the stale manifest must be discarded", + Files.exists(tmpDir + "/" + MANIFEST_NAME)); + }); + } + + @Test + public void testZeroSegmentFilesWithUncollapsedBoundariesFailsClosed() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // headBase(7) < activeBase(9): the manifest durably committed that + // segments [7..9] existed and were never declared acked. No + // in-protocol crash leaves this next to zero .sfa files -- the + // close-time drain collapses boundaries to head == active BEFORE + // its first unlink, and a fresh start writes (0,0). Uncollapsed + // boundaries with no segment files therefore prove durable data + // vanished outside the protocol: recovery must fail closed and + // keep the manifest as evidence, not silently start fresh. + writeManifest(3, 7, 9); + Map before = snapshotDir(); + + try { + SegmentRing ring = SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + if (ring != null) { + ring.close(); + } + Assert.fail("recovery must fail closed: manifest boundaries (7,9) reference " + + "durable frames but no segment file survives"); + } catch (MmapSegmentException expected) { + TestUtils.assertContains(expected.getMessage(), "references durable data"); + } + assertDirUnchanged(before); + }); + } + + @Test + public void testUnquarantinableCorruptManifestClosesFdExactlyOnce() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Correctly-sized manifest with BOTH records CRC-broken: open() + // closes the fd and then tries to quarantine the debris. Make + // rename AND remove fail (permission-degraded slot dir) so + // quarantineDebris throws after the fd is already closed. The + // propagating failure must not close the fd a second time -- the + // OS may already have handed that number to another thread, and + // a double-close would silently kill an unrelated descriptor. + writeManifestBothRecordsCrcBroken(); + Map before = snapshotDir(); + + String manifestPath = tmpDir + "/" + MANIFEST_NAME; + int[] manifestFd = {-1}; + int[] manifestFdCloses = {0}; + FilesFacade facade = new DelegatingFacade() { + @Override + public int close(int fd) { + if (fd >= 0 && fd == manifestFd[0]) { + manifestFdCloses[0]++; + } + return super.close(fd); + } + + @Override + public int openRW(String path) { + int fd = super.openRW(path); + if (manifestPath.equals(path)) { + manifestFd[0] = fd; + } + return fd; + } + + @Override + public boolean remove(String path) { + return !manifestPath.equals(path) && super.remove(path); + } + + @Override + public int rename(String oldPath, String newPath) { + return manifestPath.equals(oldPath) ? -1 : super.rename(oldPath, newPath); + } + }; + try { + SegmentRing ring = SegmentRing.openExisting(facade, tmpDir, SEGMENT_SIZE); + if (ring != null) { + ring.close(); + } + Assert.fail("recovery must fail when corrupt-manifest quarantine cannot proceed"); + } catch (MmapSegmentException expected) { + TestUtils.assertContains(expected.getMessage(), "could not quarantine"); + } + Assert.assertTrue("manifest was never opened", manifestFd[0] >= 0); + Assert.assertEquals("manifest fd must be closed exactly once (a double-close can " + + "kill an unrelated descriptor)", 1, manifestFdCloses[0]); + assertDirUnchanged(before); + }); + } + + @Test + public void testFreshStartCrashBeforeManifestCreationRecoversViaLegacyPath() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Engine crash between creating sf-initial.sfa (unflagged) and + // creating the manifest: recovers via legacy migration. + MmapSegment initial = MmapSegment.create(tmpDir + "/sf-initial.sfa", 0, SEGMENT_SIZE); + initial.close(); + + SegmentRing ring = SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.assertNotNull(ring); + try { + Assert.assertEquals(-1, ring.publishedFsn()); + } finally { + ring.close(); + } + Assert.assertTrue("legacy migration must create the manifest", + Files.exists(tmpDir + "/" + MANIFEST_NAME)); + }); + } + + @Test + public void testSingleSectorTearLeavesPriorManifestRecordRecoverable() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long segmentSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 64; + long payload = Unsafe.malloc(65, MemoryTag.NATIVE_DEFAULT); + try { + MmapSegment initial = MmapSegment.create( + tmpDir + "/sf-initial.sfa", 0, segmentSize); + try { + Assert.assertTrue(initial.tryAppend(payload, 64) >= 0); + } finally { + initial.close(); + } + + SegmentRing ring = SegmentRing.openExisting( + FilesFacade.INSTANCE, tmpDir, segmentSize); + Assert.assertNotNull(ring); + try { + MmapSegment spare = MmapSegment.create( + tmpDir + "/sf-0000000000000001.sfa", 1, segmentSize); + ring.installHotSpare(spare); + Assert.assertEquals("oversized append must rotate but leave the new active empty", + SegmentRing.PAYLOAD_TOO_LARGE, ring.appendOrFsn(payload, 65)); + Assert.assertEquals(0L, ring.publishedFsn()); + Assert.assertEquals(1L, ring.getActive().baseSeq()); + } finally { + ring.close(); + } + + String manifestPath = tmpDir + "/" + MANIFEST_NAME; + int tornBytes = (int) Math.min(512L, Files.length(manifestPath)); + overwriteRange(manifestPath, 0, tornBytes, (byte) 0xA5); + + SegmentRing recovered = SegmentRing.openExisting( + FilesFacade.INSTANCE, tmpDir, segmentSize); + Assert.assertNotNull("one aligned 512-byte tear must leave a manifest record valid", + recovered); + try { + Assert.assertEquals("recovery must fall back to the prior committed boundary", + 0L, recovered.publishedFsn()); + Assert.assertEquals(0L, recovered.getActive().baseSeq()); + } finally { + recovered.close(); + } + Assert.assertFalse("a surviving record must prevent manifest quarantine", + Files.exists(tmpDir + "/" + MANIFEST_NAME + ".corrupt")); + } finally { + Unsafe.free(payload, 65, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + // ------------------------------------------------------------------ + // Engine level: enumeration failure must not truncate the durable log. + // ------------------------------------------------------------------ + + @Test + public void testFindFirstFailureCannotCreateInitialSegment() throws Exception { + TestUtils.assertMemoryLeak(() -> { + FilesFacade facade = new DelegatingFacade() { + @Override + public long findFirst(String dir) { + return tmpDir.equals(dir) ? -1L : super.findFirst(dir); + } + }; + SegmentManager manager = new SegmentManager( + SEGMENT_SIZE, SegmentManager.DEFAULT_POLL_NANOS, + SegmentManager.UNLIMITED_TOTAL_BYTES, facade); + manager.start(); + try { + try { + new CursorSendEngine(tmpDir, SEGMENT_SIZE, manager).close(); + Assert.fail("startup must fail when SF directory enumeration fails"); + } catch (RuntimeException expected) { + Assert.assertFalse("startup failure created sf-initial.sfa", + Files.exists(tmpDir + "/sf-initial.sfa")); + } + } finally { + manager.close(); + } + }); + } + + @Test + public void testFindFirstFailureDoesNotTruncateExistingLog() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // The original failure mode: enumeration error -> treated as empty + // -> fresh start openCleanRW(O_TRUNC) destroys the durable log. + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 4); + Map before = snapshotDir(); + + FilesFacade facade = new DelegatingFacade() { + @Override + public long findFirst(String dir) { + return tmpDir.equals(dir) ? -1L : super.findFirst(dir); + } + }; + SegmentManager manager = new SegmentManager( + SEGMENT_SIZE, SegmentManager.DEFAULT_POLL_NANOS, + SegmentManager.UNLIMITED_TOTAL_BYTES, facade); + manager.start(); + try { + try { + new CursorSendEngine(tmpDir, SEGMENT_SIZE, manager).close(); + Assert.fail("startup must fail when SF directory enumeration fails"); + } catch (RuntimeException expected) { + // expected + } + } finally { + manager.close(); + } + assertDirUnchanged(before); + }); + } + + // ------------------------------------------------------------------ + // Helpers + // ------------------------------------------------------------------ + + /** Creates a real segment at {@code path} and appends {@code frames} 64-byte frames. */ + private static void writeSegmentWithFrames(String path, long baseSeq, int frames) { + long buf = Unsafe.malloc(64, MemoryTag.NATIVE_DEFAULT); + try { + MmapSegment segment = MmapSegment.create(path, baseSeq, SEGMENT_SIZE); + try { + for (int i = 0; i < frames; i++) { + for (int b = 0; b < 64; b++) { + Unsafe.getUnsafe().putByte(buf + b, (byte) (i * 31 + b)); + } + Assert.assertTrue("test frame must fit", segment.tryAppend(buf, 64) >= 0); + } + } finally { + segment.close(); + } + } finally { + Unsafe.free(buf, 64, MemoryTag.NATIVE_DEFAULT); + } + } + + /** Writes a raw 64-byte pseudo segment header (+zero padding) for corruption tests. */ + private static void writeRawSegmentHeader(String path, int magic, byte version, long baseSeq) { + int fd = Files.openCleanRW(path); + Assert.assertTrue("could not create " + path, fd >= 0); + try { + long buf = Unsafe.malloc(64, MemoryTag.NATIVE_DEFAULT); + try { + Unsafe.getUnsafe().setMemory(buf, 64, (byte) 0); + Unsafe.getUnsafe().putInt(buf, magic); + Unsafe.getUnsafe().putByte(buf + 4, version); + Unsafe.getUnsafe().putLong(buf + 8, baseSeq); + Assert.assertEquals(64, Files.write(fd, buf, 64, 0)); + } finally { + Unsafe.free(buf, 64, MemoryTag.NATIVE_DEFAULT); + } + } finally { + Files.close(fd); + } + } + + /** Overwrites a byte range in an existing file and makes the modeled tear durable. */ + private static void overwriteRange(String path, long offset, int length, byte value) { + int fd = Files.openRW(path); + Assert.assertTrue("could not open " + path, fd >= 0); + try { + long buf = Unsafe.malloc(length, MemoryTag.NATIVE_DEFAULT); + try { + Unsafe.getUnsafe().setMemory(buf, length, value); + Assert.assertEquals(length, Files.write(fd, buf, length, offset)); + Assert.assertEquals(0, Files.fsync(fd)); + } finally { + Unsafe.free(buf, length, MemoryTag.NATIVE_DEFAULT); + } + } finally { + Files.close(fd); + } + } + + /** Overwrites a single int at {@code offset} in an existing file. */ + private static void overwriteInt(String path, long offset, int value) { + int fd = Files.openRW(path); + Assert.assertTrue("could not open " + path, fd >= 0); + try { + long buf = Unsafe.malloc(4, MemoryTag.NATIVE_DEFAULT); + try { + Unsafe.getUnsafe().putInt(buf, value); + Assert.assertEquals(4, Files.write(fd, buf, 4, offset)); + } finally { + Unsafe.free(buf, 4, MemoryTag.NATIVE_DEFAULT); + } + } finally { + Files.close(fd); + } + } + + /** + * Writes a valid {@code sf-manifest.bin} with one CRC-protected record, + * mirroring SfManifest's on-disk layout (two alternating 64-byte records + * at offsets 0 and 4096 in an 8192-byte file). + */ + private void writeManifest(long generation, long headBase, long activeBase) { + String path = tmpDir + "/" + MANIFEST_NAME; + // openRW (not openCleanRW): callers may layer a second generation's + // record into the sibling slot of an existing manifest. + int fd = Files.openRW(path); + Assert.assertTrue("could not create manifest", fd >= 0); + try { + if (Files.length(path) < 8192) { + Assert.assertTrue(Files.truncate(fd, 8192)); + } + long buf = Unsafe.malloc(64, MemoryTag.NATIVE_DEFAULT); + try { + Unsafe.getUnsafe().setMemory(buf, 64, (byte) 0); + Unsafe.getUnsafe().putInt(buf, 0x314d4653); // SFM1 + Unsafe.getUnsafe().putInt(buf + 4, 1); // version + Unsafe.getUnsafe().putLong(buf + 8, generation); + Unsafe.getUnsafe().putLong(buf + 16, headBase); + Unsafe.getUnsafe().putLong(buf + 24, activeBase); + int crc = Crc32c.update(Crc32c.INIT, buf, 60); + Unsafe.getUnsafe().putInt(buf + 60, crc); + long offset = (generation & 1L) * 4096; + Assert.assertEquals(64, Files.write(fd, buf, 64, offset)); + } finally { + Unsafe.free(buf, 64, MemoryTag.NATIVE_DEFAULT); + } + } finally { + Files.close(fd); + } + } + + /** + * Writes a correctly-sized (8192-byte) manifest whose A and B records are + * BOTH structurally plausible (magic, version, boundaries) but fail their + * CRC check -- the "no valid CRC-protected record" quarantine trigger. + */ + private void writeManifestBothRecordsCrcBroken() { + String path = tmpDir + "/" + MANIFEST_NAME; + int fd = Files.openRW(path); + Assert.assertTrue("could not create manifest", fd >= 0); + try { + Assert.assertTrue(Files.truncate(fd, 8192)); + long buf = Unsafe.malloc(64, MemoryTag.NATIVE_DEFAULT); + try { + for (long generation = 1; generation <= 2; generation++) { + Unsafe.getUnsafe().setMemory(buf, 64, (byte) 0); + Unsafe.getUnsafe().putInt(buf, 0x314d4653); // SFM1 + Unsafe.getUnsafe().putInt(buf + 4, 1); // version + Unsafe.getUnsafe().putLong(buf + 8, generation); + Unsafe.getUnsafe().putLong(buf + 16, 0); // headBase + Unsafe.getUnsafe().putLong(buf + 24, 2); // activeBase + int crc = Crc32c.update(Crc32c.INIT, buf, 60); + Unsafe.getUnsafe().putInt(buf + 60, crc + 1); // broken CRC + long offset = (generation & 1L) * 4096; + Assert.assertEquals(64, Files.write(fd, buf, 64, offset)); + } + } finally { + Unsafe.free(buf, 64, MemoryTag.NATIVE_DEFAULT); + } + } finally { + Files.close(fd); + } + } + + /** + * Snapshot of the durable SF payload files (segments, quarantined + * segments, manifest): name -> content. Lifecycle noise such as the slot + * lock and ack watermark is deliberately excluded -- "recovery must not + * mutate the slot" is a statement about the durable log, not about lock + * bookkeeping. + */ + private Map snapshotDir() { + Map out = new HashMap<>(); + Path dir = Paths.get(tmpDir); + try (java.util.stream.Stream stream = java.nio.file.Files.list(dir)) { + stream.filter(java.nio.file.Files::isRegularFile) + .filter(p -> { + String name = p.getFileName().toString(); + return name.endsWith(".sfa") || name.endsWith(".corrupt") + || MANIFEST_NAME.equals(name); + }) + .forEach(p -> { + try { + out.put(p.getFileName().toString(), java.nio.file.Files.readAllBytes(p)); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } catch (IOException e) { + throw new RuntimeException(e); + } + return out; + } + + /** Asserts the directory holds exactly the snapshotted files, byte for byte. */ + private void assertDirUnchanged(Map before) { + Map after = snapshotDir(); + Assert.assertEquals("file set must be unchanged", before.keySet(), after.keySet()); + for (Map.Entry e : before.entrySet()) { + Assert.assertArrayEquals("bytes of " + e.getKey() + " must be unchanged", + e.getValue(), after.get(e.getKey())); + } + } + + private int countSfaFiles() { + int count = 0; + for (String name : snapshotDir().keySet()) { + if (name.endsWith(".sfa")) { + count++; + } + } + return count; + } + + // ------------------------------------------------------------------ + // Manifest crash windows and record selection + // ------------------------------------------------------------------ + + @Test + public void testCreationCrashZeroByteManifestSelfHeals() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // kill -9 between the manifest's O_EXCL create and its first + // durable record leaves a zero-byte file. No boundary was ever + // committed, so nothing can depend on it: startup must self-heal, + // not demand an operator delete the file. + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 3); + int fd = Files.openCleanRW(tmpDir + "/" + MANIFEST_NAME); + Assert.assertTrue(fd >= 0); + Files.close(fd); + + SegmentRing ring = SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.assertNotNull("creation-crash manifest debris must not brick startup", ring); + try { + Assert.assertEquals(2, ring.publishedFsn()); + } finally { + ring.close(); + } + Assert.assertTrue("debris must be quarantined for postmortem", + Files.exists(tmpDir + "/" + MANIFEST_NAME + ".corrupt")); + Assert.assertTrue("a fresh valid manifest must replace the debris", + Files.exists(tmpDir + "/" + MANIFEST_NAME)); + }); + } + + @Test + public void testCreationCrashRecordlessManifestSelfHeals() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Same window, later stage: allocate() completed (8192 zero bytes) + // but the first record write/fsync never landed. + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 3); + int fd = Files.openCleanRW(tmpDir + "/" + MANIFEST_NAME); + Assert.assertTrue(fd >= 0); + Assert.assertTrue(Files.truncate(fd, 8192)); + Files.close(fd); + + SegmentRing ring = SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.assertNotNull("record-less manifest debris must not brick startup", ring); + try { + Assert.assertEquals(2, ring.publishedFsn()); + } finally { + ring.close(); + } + Assert.assertTrue(Files.exists(tmpDir + "/" + MANIFEST_NAME + ".corrupt")); + Assert.assertTrue(Files.exists(tmpDir + "/" + MANIFEST_NAME)); + }); + } + + @Test + public void testRecordlessManifestWithFlaggedSegmentsFailsClosed() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Flagged segments prove a manifest create COMPLETED at some + // point, so a record-less manifest here is double-slot bit rot, + // not creation debris -- boundaries were lost. Fail closed. + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 3); + SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE).close(); + int fd = Files.openCleanRW(tmpDir + "/" + MANIFEST_NAME); // truncates + Assert.assertTrue(fd >= 0); + Assert.assertTrue(Files.truncate(fd, 8192)); + Files.close(fd); + + try { + SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.fail("boundary loss next to flagged segments must fail closed"); + } catch (MmapSegmentException expected) { + TestUtils.assertContains(expected.getMessage(), "missing"); + } + }); + } + + @Test + public void testDrainCrashSurvivingSpareRecoversEmpty() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Clean-drain crash window: boundaries collapsed to head==active, + // the active-base file already unlinked, but an empty hot spare + // (provisional base > activeBase) survived. Everything was acked; + // this must recover as EMPTY, not brick on "missing active". + MmapSegment spare = MmapSegment.create(tmpDir + "/sf-0000000000000007.sfa", 5, SEGMENT_SIZE); + spare.close(); + writeManifest(4, 3, 3); + + SegmentRing ring = SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.assertNull("drain crash window must recover as EMPTY", ring); + Assert.assertEquals("surviving spare must be cleaned up", 0, countSfaFiles()); + Assert.assertFalse("collapsed manifest must be removed", + Files.exists(tmpDir + "/" + MANIFEST_NAME)); + }); + } + + @Test + public void testCorruptActiveWithSameBaseEmptyStandInFailsClosed() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // The active (holding unacked frames) is corrupted while a + // leftover spare coincidentally carries the same base. Accepting + // the clean empty as the "rotation crash" active would quarantine + // the unacked frames and re-issue their FSNs -- recovery must + // fail closed instead. + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 2); + String corruptActive = tmpDir + "/sf-0000000000000001.sfa"; + writeSegmentWithFrames(corruptActive, 2, 2); + overwriteInt(corruptActive, 0, 0xBADC0DE); // bit-rot the magic + MmapSegment standIn = MmapSegment.create(tmpDir + "/sf-0000000000000002.sfa", 2, SEGMENT_SIZE); + standIn.close(); + writeManifest(1, 0, 2); + Map before = snapshotDir(); + + try { + SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.fail("an empty stand-in must not mask a corrupt active"); + } catch (MmapSegmentException expected) { + TestUtils.assertContains(expected.getMessage(), "missing expected SF active"); + } + assertDirUnchanged(before); + }); + } + + @Test + public void testManifestHigherGenerationRecordWins() throws Exception { + TestUtils.assertMemoryLeak(() -> { + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 2); + MmapSegment next = MmapSegment.create(tmpDir + "/sf-0000000000000001.sfa", 2, SEGMENT_SIZE); + next.close(); + // gen1 (slot 1) says active=0; gen2 (slot 0) says active=2. If + // selection picked gen1, the empty at base 2 would sit beyond the + // committed active boundary; gen2 accepts it as the active. + writeManifest(1, 0, 0); + writeManifest(2, 0, 2); + + SegmentRing ring = SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.assertNotNull(ring); + try { + Assert.assertEquals("the higher-generation record must win", + 2, ring.getActive().baseSeq()); + } finally { + ring.close(); + } + }); + } + + @Test + public void testManifestTornNewerRecordFallsBackToOlder() throws Exception { + TestUtils.assertMemoryLeak(() -> { + writeSegmentWithFrames(tmpDir + "/sf-initial.sfa", 0, 2); + MmapSegment next = MmapSegment.create(tmpDir + "/sf-0000000000000001.sfa", 2, SEGMENT_SIZE); + next.close(); + // gen1 (slot 1) is valid and matches the segments; gen2 (slot 0) + // was torn mid-write (bad CRC). Selection must fall back to gen1 + // rather than reject the manifest or trust torn boundaries. + writeManifest(1, 0, 2); + writeManifest(2, 0, 4); + overwriteInt(tmpDir + "/" + MANIFEST_NAME, 60, 0xBADC0DE); // tear gen2's CRC (slot 0) + + SegmentRing ring = SegmentRing.openExisting(FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE); + Assert.assertNotNull("a torn newer record must fall back to the older slot", ring); + try { + Assert.assertEquals(2, ring.getActive().baseSeq()); + } finally { + ring.close(); + } + }); + } + + /** FilesFacade delegating everything to the production instance. */ + private static class DelegatingFacade implements FilesFacade { + @Override public boolean allocate(int fd, long size) { return INSTANCE.allocate(fd, size); } + @Override public long allocNativePath(String path) { return INSTANCE.allocNativePath(path); } + @Override public int close(int fd) { return INSTANCE.close(fd); } + @Override public boolean exists(String path) { return INSTANCE.exists(path); } + @Override public void findClose(long findPtr) { INSTANCE.findClose(findPtr); } + @Override public long findFirst(String dir) { return INSTANCE.findFirst(dir); } + @Override public long findName(long findPtr) { return INSTANCE.findName(findPtr); } + @Override public int findNext(long findPtr) { return INSTANCE.findNext(findPtr); } + @Override public int findType(long findPtr) { return INSTANCE.findType(findPtr); } + @Override public void freeNativePath(long pathPtr) { INSTANCE.freeNativePath(pathPtr); } + @Override public int fsync(int fd) { return INSTANCE.fsync(fd); } + @Override public long length(int fd) { return INSTANCE.length(fd); } + @Override public long length(String path) { return INSTANCE.length(path); } + @Override public int lock(int fd) { return INSTANCE.lock(fd); } + @Override public int mkdir(String path, int mode) { return INSTANCE.mkdir(path, mode); } + @Override public int openCleanRW(String path) { return INSTANCE.openCleanRW(path); } + @Override public int openCleanRW(long pathPtr) { return INSTANCE.openCleanRW(pathPtr); } + @Override public int openRW(String path) { return INSTANCE.openRW(path); } + @Override public int openRW(long pathPtr) { return INSTANCE.openRW(pathPtr); } + @Override public long length(long pathPtr) { return INSTANCE.length(pathPtr); } + @Override public long read(int fd, long addr, long len, long offset) { return INSTANCE.read(fd, addr, len, offset); } + @Override public boolean remove(String path) { return INSTANCE.remove(path); } + @Override public boolean remove(long pathPtr) { return INSTANCE.remove(pathPtr); } + @Override public int rename(String oldPath, String newPath) { return INSTANCE.rename(oldPath, newPath); } + @Override public boolean truncate(int fd, long size) { return INSTANCE.truncate(fd, size); } + @Override public long write(int fd, long addr, long len, long offset) { return INSTANCE.write(fd, addr, len, offset); } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingMembershipTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingMembershipTest.java new file mode 100644 index 00000000..d8813b7f --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingMembershipTest.java @@ -0,0 +1,209 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentRing; +import io.questdb.client.std.Files; +import io.questdb.client.std.FilesFacade; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.lang.reflect.Proxy; +import java.nio.file.Paths; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class SegmentRingMembershipTest { + + private static final String CURSOR_PACKAGE = "io.questdb.client.cutlass.qwp.client.sf.cursor."; + private static final long SEGMENT_SIZE = MmapSegment.HEADER_SIZE + + MmapSegment.FRAME_HEADER_SIZE + 16; + private String tmpDir; + + @Before + public void setUp() { + tmpDir = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-ring-membership-" + System.nanoTime()).toString(); + assertEquals(0, Files.mkdir(tmpDir, Files.DIR_MODE_DEFAULT)); + } + + @After + public void tearDown() { + if (tmpDir == null) { + return; + } + long find = Files.findFirst(tmpDir); + if (find > 0) { + try { + int rc = 1; + while (rc > 0) { + String name = Files.utf8ToString(Files.findName(find)); + if (name != null && !".".equals(name) && !"..".equals(name)) { + Files.remove(tmpDir + "/" + name); + } + rc = Files.findNext(find); + } + } finally { + Files.findClose(find); + } + } + Files.remove(tmpDir); + } + + @Test + public void testLargeRecoveryCountsMembershipPrimitives() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final int n = 2048; + createChain(n); + + CountingOperations operations = new CountingOperations(); + try (SegmentRing ring = recover(newMembershipObserver(operations, false))) { + assertRecoveredChain(ring, n); + } + long linearBound = 2L * n; + assertTrue("production default took " + operations.count + + " membership operations; expected fewer than " + linearBound, + operations.count < linearBound); + assertEquals("production default identity map must do one lookup per discovered segment", + n, operations.count); + }); + } + + @Test + public void testMembershipAllocationFailureRetainsLocalOwnership() throws Exception { + TestUtils.assertMemoryLeak(() -> { + createChain(8); + String extraPath = tmpDir + "/sf-extra.sfa"; + MmapSegment.create(extraPath, 8, SEGMENT_SIZE).close(); + try { + recover(newMembershipObserver(null, true)); + fail("expected injected membership allocation failure"); + } catch (OutOfMemoryError expected) { + assertEquals("injected membership allocation failure", expected.getMessage()); + } + + try (SegmentRing ring = SegmentRing.openExisting( + FilesFacade.INSTANCE, + tmpDir, + SEGMENT_SIZE + )) { + assertRecoveredChain(ring, 8); + } + assertFalse("retry must close and remove the unselected empty extra", Files.exists(extraPath)); + }); + } + + private static void assertRecoveredChain(SegmentRing ring, int n) { + assertNotNull(ring); + assertEquals(n, ring.nextSeqHint()); + assertEquals(n - 1, ring.publishedFsn()); + assertEquals(n - 1, ring.getSealedSegments().size()); + } + + private void createChain(int n) { + long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < 16; i++) { + Unsafe.getUnsafe().putByte(buf + i, (byte) i); + } + for (int i = 0; i < n; i++) { + String name = String.format("sf-%05d.sfa", i); + try (MmapSegment segment = MmapSegment.create(tmpDir + "/" + name, i, SEGMENT_SIZE)) { + assertTrue("setup append " + i, segment.tryAppend(buf, 16) >= 0); + } + } + } finally { + Unsafe.free(buf, 16, MemoryTag.NATIVE_DEFAULT); + } + } + + private SegmentRing recover(Object membershipObserver) throws Exception { + Class observerClass = Class.forName(CURSOR_PACKAGE + "SegmentRing$MembershipObserver"); + Method recover = SegmentRing.class.getDeclaredMethod( + "recover", + FilesFacade.class, + String.class, + long.class, + observerClass + ); + recover.setAccessible(true); + Object recovery; + try { + recovery = recover.invoke(null, FilesFacade.INSTANCE, tmpDir, SEGMENT_SIZE, membershipObserver); + } catch (InvocationTargetException e) { + Throwable cause = e.getCause(); + if (cause instanceof Error) { + throw (Error) cause; + } + if (cause instanceof RuntimeException) { + throw (RuntimeException) cause; + } + throw new AssertionError(cause); + } + Method ring = recovery.getClass().getDeclaredMethod("ring"); + ring.setAccessible(true); + return (SegmentRing) ring.invoke(recovery); + } + + private static Object newMembershipObserver( + final CountingOperations operations, + final boolean failAllocation + ) throws Exception { + Class observerClass = Class.forName(CURSOR_PACKAGE + "SegmentRing$MembershipObserver"); + return Proxy.newProxyInstance( + SegmentRing.class.getClassLoader(), + new Class[]{observerClass}, + (proxy, method, args) -> { + if ("beforeMembershipAllocation".equals(method.getName())) { + if (failAllocation) { + throw new OutOfMemoryError("injected membership allocation failure"); + } + } else if ("onMembershipOperation".equals(method.getName())) { + operations.onOperation(); + } + return null; + } + ); + } + + private static final class CountingOperations { + private long count; + + private void onOperation() { + count++; + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingRecoveryUnlinkTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingRecoveryUnlinkTest.java index 9f533fc7..a226eb0f 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingRecoveryUnlinkTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingRecoveryUnlinkTest.java @@ -76,11 +76,17 @@ public void tearDown() { } @Test - public void testRecoveryUnlinksEmptyOrphanSegments() throws Exception { + public void testRecoveryReusesSoleEmptyOrphanAsInitialActive() throws Exception { TestUtils.assertMemoryLeak(() -> { // Simulate a crashed prior session that left an unrotated hot spare // (valid SF01 header, frameCount=0). MmapSegment.create stamps the // header but writes no frames. + // + // Contract (manifest-era): a clean empty leftover is REUSED as the + // fresh ring's initial active instead of being unlinked and + // re-created. The anti-leak guarantee this test was written for + // still holds -- crash cycles converge to exactly one segment + // file, they don't accumulate. String orphanPath = tmpDir + "/sf-orphan.sfa"; MmapSegment empty = MmapSegment.create(orphanPath, 0L, SEGMENT_SIZE); empty.close(); @@ -89,12 +95,17 @@ public void testRecoveryUnlinksEmptyOrphanSegments() throws Exception { SegmentRing recovered = SegmentRing.openExisting(tmpDir, SEGMENT_SIZE); - Assert.assertNull( - "recovery returned a ring even though the only segment was empty", + Assert.assertNotNull( + "recovery must reuse the clean empty leftover as the initial active", recovered); - Assert.assertFalse( - "recovery left the empty orphan .sfa on disk — disk leak grows " - + "with every crash cycle", + try { + Assert.assertEquals("reused ring must be empty", -1, recovered.publishedFsn()); + Assert.assertEquals(0, recovered.getActive().baseSeq()); + Assert.assertEquals(orphanPath, recovered.getActive().path()); + } finally { + recovered.close(); + } + Assert.assertTrue("the reused segment file must still exist", Files.exists(orphanPath)); }); } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java index f1a0fcde..dffc1b7d 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java @@ -27,6 +27,7 @@ import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegmentException; import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentRing; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SfSanitizedResidueException; import io.questdb.client.std.Files; import io.questdb.client.std.MemoryTag; import io.questdb.client.std.Misc; @@ -38,10 +39,14 @@ import org.junit.Test; import java.nio.file.Paths; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicReference; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; public class SegmentRingTest { @@ -328,6 +333,253 @@ public void testOpenExistingRecoversActivePlusSealed() throws Exception { }); } + @Test + public void testResealedTornRecoveredSegmentDoesNotBrickNextRecovery() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // End-to-end two-crash regression: + // crash #1 -> torn active with residue up to the file end; + // recovery #1 -> resume in it, fill it, rotate (reseals the + // recovered file with a tail gap the resumed appends never + // overwrote); + // crash #2 -> recovery #2 used to throw "corrupt torn tail in + // sealed SF segment" on EVERY startup, because nothing ever + // sanitized crash #1's residue and the sealed-suffix check + // correctly refuses non-zero sealed tails. Recovery #1 now + // sanitizes the segment it resumes as active before any + // append, so recovery #2 must succeed and see the full chain. + long segSize = MmapSegment.HEADER_SIZE + + 4 * (MmapSegment.FRAME_HEADER_SIZE + 16) + + 12; // reseal gap: a 5th 24-byte frame can never fit + long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + try { + // Session 1 "crashes": two good frames, then garbage from the + // torn write all the way to the file end. + try (MmapSegment s0 = MmapSegment.create(tmpDir + "/r0.sfa", 0, segSize)) { + s0.tryAppend(buf, 16); + s0.tryAppend(buf, 16); + long addr = s0.address(); + for (long off = s0.publishedOffset(); off + 4 <= segSize; off += 4) { + Unsafe.getUnsafe().putInt(addr + off, 0xCAFEBABE); + } + s0.msync(); + } + // Recovery #1 + session 2: resume in the torn segment, fill + // it, rotate into a spare. The recovered file is resealed + // with its 12-byte gap intact. + try (SegmentRing ring = SegmentRing.openExisting(tmpDir, segSize)) { + assertNotNull(ring); + assertEquals(2, ring.nextSeqHint()); + assertEquals(2, ring.appendOrFsn(buf, 16)); + assertEquals(3, ring.appendOrFsn(buf, 16)); + ring.installHotSpare(MmapSegment.create(tmpDir + "/r1.sfa", 4, segSize)); + assertEquals("append must rotate into the spare", + 4, ring.appendOrFsn(buf, 16)); + assertEquals(1, ring.getSealedSegments().size()); + } + // Recovery #2 ("crash" #2): must not brick on the resealed + // recovered segment. + try (SegmentRing ring = SegmentRing.openExisting(tmpDir, segSize)) { + assertNotNull("second recovery must survive the resealed segment", ring); + assertEquals(1, ring.getSealedSegments().size()); + assertEquals(0, ring.getSealedSegments().get(0).baseSeq()); + assertEquals(4, ring.getSealedSegments().get(0).frameCount()); + assertEquals(4, ring.getActive().baseSeq()); + assertEquals(1, ring.getActive().frameCount()); + assertEquals(5, ring.nextSeqHint()); + } + } finally { + Unsafe.free(buf, 16, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + @Test + public void testMidFileTearInSealedMemberFailsClosedWithoutMutation() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Bit-rot mid-file in a SEALED chain member: the scan stops at + // the first bad CRC, so frames past the tear -- individually + // valid CRCs, real unacked payloads, the ONLY surviving copy -- + // are classified as torn-tail residue. Recovery must fail closed + // on the FSN gap the lost frames create BEFORE any sanitization + // decision, leaving every byte on disk for operator extraction, + // and must keep doing so identically on retry. + long segSize = MmapSegment.HEADER_SIZE + + 4 * (MmapSegment.FRAME_HEADER_SIZE + 16); + String m0Path = tmpDir + "/m0.sfa"; + String m1Path = tmpDir + "/m1.sfa"; + String m2Path = tmpDir + "/m2.sfa"; + long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + try { + fillPattern(buf, 16, 3); + MmapSegment s0 = MmapSegment.create(m0Path, 0, segSize); + for (int i = 0; i < 4; i++) s0.tryAppend(buf, 16); + s0.close(); + MmapSegment s1 = MmapSegment.create(m1Path, 4, segSize); + for (int i = 0; i < 4; i++) s1.tryAppend(buf, 16); + s1.close(); + MmapSegment s2 = MmapSegment.create(m2Path, 8, segSize); + s2.tryAppend(buf, 16); + s2.close(); + // First recovery migrates the legacy chain, creates the + // manifest and validates clean: sealed m0+m1, active m2. + try (SegmentRing ring = SegmentRing.openExisting(tmpDir, segSize)) { + assertNotNull(ring); + assertEquals(2, ring.getSealedSegments().size()); + } + // Bit-rot strikes sealed m0 mid-file: clobber the CRC of its + // SECOND frame. Frames 2..3 keep valid CRCs but are + // unreachable to the sequential scan. + long frame1Offset = MmapSegment.HEADER_SIZE + + MmapSegment.FRAME_HEADER_SIZE + 16; + int fd = Files.openRW(m0Path); + assertTrue("openRW must succeed", fd >= 0); + long bad = Unsafe.malloc(4, MemoryTag.NATIVE_DEFAULT); + try { + Unsafe.getUnsafe().putInt(bad, 0xDEADBEEF); + assertEquals(4L, Files.write(fd, bad, 4, frame1Offset)); + Files.fsync(fd); + } finally { + Unsafe.free(bad, 4, MemoryTag.NATIVE_DEFAULT); + Files.close(fd); + } + byte[] m0Before = readFileBytes(m0Path); + byte[] m1Before = readFileBytes(m1Path); + byte[] m2Before = readFileBytes(m2Path); + for (int attempt = 0; attempt < 2; attempt++) { + try { + Misc.free(SegmentRing.openExisting(tmpDir, segSize)); + throw new AssertionError( + "mid-file tear in a sealed member must fail recovery"); + } catch (MmapSegmentException expected) { + assertTrue(expected.getMessage(), + expected.getMessage().contains("FSN gap")); + } + assertArrayEquals("attempt #" + attempt + + ": failed recovery must not mutate the torn sealed member", + m0Before, readFileBytes(m0Path)); + assertArrayEquals(m1Before, readFileBytes(m1Path)); + assertArrayEquals(m2Before, readFileBytes(m2Path)); + } + } finally { + Unsafe.free(buf, 16, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + @Test + public void testProvenDeadSealedResidueSanitizedThenHealsAfterOneRestart() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // Legacy pre-sanitization poison: a sealed member whose frame + // accounting is COMPLETE (contiguity + boundaries prove no frame + // lost) but whose suffix gap carries residue from a pre-fix + // client. The residue is provably dead bytes, so recovery zeroes + // it durably, still fails closed on first sight to surface the + // incident, and the restart proves the chain clean. + long segSize = MmapSegment.HEADER_SIZE + + 4 * (MmapSegment.FRAME_HEADER_SIZE + 16) + + 12; // sealed suffix gap that can never fit a frame + String m0Path = tmpDir + "/p0.sfa"; + String m1Path = tmpDir + "/p1.sfa"; + long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + try { + fillPattern(buf, 16, 5); + MmapSegment s0 = MmapSegment.create(m0Path, 0, segSize); + for (int i = 0; i < 4; i++) s0.tryAppend(buf, 16); + long gapStart = s0.publishedOffset(); + s0.close(); + MmapSegment s1 = MmapSegment.create(m1Path, 4, segSize); + s1.tryAppend(buf, 16); + s1.close(); + // First recovery creates the manifest over the clean chain. + try (SegmentRing ring = SegmentRing.openExisting(tmpDir, segSize)) { + assertNotNull(ring); + assertEquals(1, ring.getSealedSegments().size()); + } + // Poison the sealed suffix gap the way a pre-fix client's + // reseal-after-recovery did. + int fd = Files.openRW(m0Path); + assertTrue("openRW must succeed", fd >= 0); + long junk = Unsafe.malloc(12, MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < 3; i++) { + Unsafe.getUnsafe().putInt(junk + i * 4L, 0xCAFEBABE); + } + assertEquals(12L, Files.write(fd, junk, 12, gapStart)); + Files.fsync(fd); + } finally { + Unsafe.free(junk, 12, MemoryTag.NATIVE_DEFAULT); + Files.close(fd); + } + // Recovery #1: proven-dead residue is sanitized, incident + // still fails closed on first sight. + try { + Misc.free(SegmentRing.openExisting(tmpDir, segSize)); + throw new AssertionError("poisoned sealed suffix must fail closed on first sight"); + } catch (MmapSegmentException expected) { + assertTrue("first-sight throw must be the healed-residue refinement so " + + "unattended callers can retry: " + expected.getClass().getName(), + expected instanceof SfSanitizedResidueException); + assertTrue(expected.getMessage(), + expected.getMessage().contains("corrupt torn tail in sealed SF segment")); + } + // The heal: residue durably zeroed, all frames intact. + try (MmapSegment seg = MmapSegment.openExisting(m0Path)) { + assertEquals("frames must be untouched", 4L, seg.frameCount()); + assertEquals("proven-dead residue must be zeroed on disk", + 0L, seg.tornTailBytes()); + } + // Recovery #2: the restart proves the chain clean. + try (SegmentRing ring = SegmentRing.openExisting(tmpDir, segSize)) { + assertNotNull("restart after the sanitizing fail-closed throw must succeed", ring); + assertEquals(1, ring.getSealedSegments().size()); + assertEquals(4, ring.getSealedSegments().get(0).frameCount()); + assertEquals(4, ring.getActive().baseSeq()); + assertEquals(1, ring.getActive().frameCount()); + assertEquals(5, ring.nextSeqHint()); + } + } finally { + Unsafe.free(buf, 16, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + @Test + public void testRingRecoverySanitizesResumedActiveTornTail() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // The funnel guarantee: whatever segment recovery selects as the + // resumed active has its torn residue durably zeroed before the + // ring is exposed -- even if the session then crashes without a + // single append, the reseal/resurrection hazards are gone. + String path = tmpDir + "/a0.sfa"; + long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + try { + fillPattern(buf, 16, 9); + try (MmapSegment s0 = MmapSegment.create(path, 0, 4096)) { + s0.tryAppend(buf, 16); + s0.tryAppend(buf, 16); + long addr = s0.address(); + for (long off = s0.publishedOffset(); off + 4 <= 4096; off += 4) { + Unsafe.getUnsafe().putInt(addr + off, 0xCAFEBABE); + } + s0.msync(); + } + try (SegmentRing ring = SegmentRing.openExisting(tmpDir, 4096)) { + assertNotNull(ring); + assertEquals("the observation must survive for diagnostics", + true, ring.getActive().tornTailBytes() > 0); + } + // No appends happened; the zeroing must already be durable. + try (MmapSegment seg = MmapSegment.openExisting(path)) { + assertEquals(2L, seg.frameCount()); + assertEquals("ring recovery must have sanitized the resumed active", + 0L, seg.tornTailBytes()); + } + } finally { + Unsafe.free(buf, 16, MemoryTag.NATIVE_DEFAULT); + } + }); + } + @Test public void testOpenExistingDetectsFsnGap() throws Exception { TestUtils.assertMemoryLeak(() -> { @@ -428,9 +680,10 @@ public void testAcknowledgeIsMonotonic() throws Exception { } @Test - public void testNextSealedAfterWalksThousandsOfSegmentsWithoutOverflow() throws Exception { + public void testNextSealedAfterWalksThousandsOfSegmentsInLinearOperations() throws Exception { TestUtils.assertMemoryLeak(() -> { - // Regression for "sealed snapshot grew unexpectedly large". + // Regression for both "sealed snapshot grew unexpectedly large" + // and a later quadratic list traversal at every segment boundary. // The cursor I/O loop used to copy the entire sealed list into a // fixed-size array (initial 16, grown once to 32) on every advance. // Under load — producer outpacing the WS sender, no maxTotalBytes @@ -462,8 +715,18 @@ public void testNextSealedAfterWalksThousandsOfSegmentsWithoutOverflow() throws // After the loop we have `sealedCount` sealed segments and one // active (containing nothing yet — its base = sealedCount). // Now walk: oldest sealed, then nextSealedAfter() repeatedly. + SegmentRing.resetNextSealedComparisons(); MmapSegment cursor = ring.firstSealed(); assertNotNull(cursor); + for (int i = 1; i < sealedCount / 2; i++) { + cursor = ring.nextSealedAfter(cursor); + assertNotNull(cursor); + } + long halfWalkOperations = SegmentRing.getNextSealedComparisons(); + + SegmentRing.resetNextSealedComparisons(); + cursor = ring.firstSealed(); + assertNotNull(cursor); assertEquals(0, cursor.baseSeq()); int visited = 1; long prevBase = cursor.baseSeq(); @@ -478,8 +741,14 @@ public void testNextSealedAfterWalksThousandsOfSegmentsWithoutOverflow() throws visited++; } assertEquals("must visit every sealed segment", sealedCount, visited); - // Walking past the last sealed → null (caller falls through to active). - assertNull(ring.nextSealedAfter(cursor)); + // The loop terminated when walking past the last sealed returned null. + long comparisons = SegmentRing.getNextSealedComparisons(); + assertTrue("walk inspected " + comparisons + " entries for " + sealedCount + + " segments; successor traversal must remain O(N)", + comparisons <= 2L * sealedCount); + assertTrue("doubling the walk grew operations from " + halfWalkOperations + + " to " + comparisons + "; expected linear scaling", + comparisons <= 2L * halfWalkOperations + 2L); } } finally { Unsafe.free(buf, 16, MemoryTag.NATIVE_DEFAULT); @@ -528,24 +797,88 @@ public void testNextSealedAfterStillReturnsCorrectlyWhenCursorWasTrimmed() throw }); } + @Test(timeout = 30_000L) + public void testNextSealedAfterSurvivesConcurrentRotationAndHeadTrim() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long segSize = MmapSegment.HEADER_SIZE + (MmapSegment.FRAME_HEADER_SIZE + 16); + long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + try { + MmapSegment seg0 = MmapSegment.create(tmpDir + "/race-0.sfa", 0, segSize); + try (SegmentRing ring = new SegmentRing(seg0, segSize)) { + fillPattern(buf, 16, 0); + for (int i = 0; i < 3; i++) { + assertEquals(i, ring.appendOrFsn(buf, 16)); + ring.installHotSpare(MmapSegment.create( + tmpDir + "/race-" + (i + 1) + ".sfa", + ring.nextSeqHint(), segSize)); + } + MmapSegment cursor = ring.firstSealed(); + assertNotNull(cursor); + ring.acknowledge(1); + + CountDownLatch ready = new CountDownLatch(2); + CountDownLatch start = new CountDownLatch(1); + AtomicReference failure = new AtomicReference<>(); + Thread rotate = new Thread(() -> { + ready.countDown(); + try { + start.await(); + assertEquals(3L, ring.appendOrFsn(buf, 16)); + } catch (Throwable t) { + failure.compareAndSet(null, t); + } + }, "segment-rotate"); + Thread trim = new Thread(() -> { + ready.countDown(); + try { + start.await(); + ObjList drained = ring.drainTrimmable(); + assertNotNull(drained); + assertEquals(2, drained.size()); + for (int i = 0; i < drained.size(); i++) { + drained.get(i).close(); + } + } catch (Throwable t) { + failure.compareAndSet(null, t); + } + }, "segment-trim"); + + synchronized (ring) { + rotate.start(); + trim.start(); + ready.await(); + start.countDown(); + } + rotate.join(); + trim.join(); + assertNull("concurrent mutation failed: " + failure.get(), failure.get()); + + MmapSegment next = ring.nextSealedAfter(cursor); + assertNotNull(next); + assertEquals("trim fallback must skip both removed successors", 2L, next.baseSeq()); + assertEquals("rotation must leave the former active sealed", 1, ring.getSealedSegments().size()); + } + } finally { + Unsafe.free(buf, 16, MemoryTag.NATIVE_DEFAULT); + } + }); + } + /** * Open-time sort regression: at the documented {@code sf_max_total_bytes - * / sf_max_bytes} ceiling (~16K segments) an O(N²) sort over the - * recovered segments burns multi-second wall time before the I/O thread - * can start. The previous selection-sort implementation regressed an - * earlier perf fix on the legacy {@code SegmentLog} path; this test - * guards the cursor path against the same regression. + * / sf_max_bytes} ceiling (~16K segments), an O(N²) sort over the + * recovered segments delays the I/O thread. This test guards the sort + * with a deterministic comparison count. *

      * Constructs N=2048 valid one-frame segments with names assigned in * lexicographic order — the exact pattern {@code readdir} produces on * many filesystems (and the worst case for a naive first-element pivot). * Recovers, asserts contiguous baseSeq ordering and total frame count, - * and bounds wall time at 5 s. With the median-of-three quicksort the - * test completes in well under a second; an O(N²) regression at this - * scale climbs back into multi-second territory. + * and bounds sort comparisons independently of CI timing or filesystem + * throughput. */ @Test - public void testLargeSegmentCountReopensInOrder() throws Exception { + public void testLargeSegmentCountRecoveryIsLogLinear() throws Exception { TestUtils.assertMemoryLeak(() -> { final int n = 2048; long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); @@ -577,6 +910,8 @@ public void testLargeSegmentCountReopensInOrder() throws Exception { n, ring.nextSeqHint()); // publishedFsn = n - 1 (last frame visible). assertEquals(n - 1, ring.publishedFsn()); + assertEquals("full valid chain must retain every segment", + n - 1, ring.getSealedSegments().size()); // O(N log N) quicksort with good pivots does ~2-3 * N * log2(N) // comparisons; the partition-pass + median-of-three counter we // increment per recursive frame upper-bounds this at roughly @@ -600,6 +935,147 @@ public void testLargeSegmentCountReopensInOrder() throws Exception { }); } + /** + * Adversarial-order companion to + * {@link #testLargeSegmentCountRecoveryIsLogLinear}: that test covers the + * already-sorted readdir order median-of-three was chosen for; this one + * covers the orders median-of-three does NOT defend. On the + * pre-introsort quicksort, exact simulation at N=16384 measured ~22.6M + * comparisons for organ-pipe, ~134M (full N²/2) for mass-duplicate + * baseSeqs and ~50.7M for Musser's median-of-three-killer permutation. + * The heapsort fallback caps the worst of these at ~3.8·N·log₂(N) + * (~860K), so the 8·N·log₂(N) bound below keeps >2x headroom against + * harmless implementation drift while sitting ~26x under the mildest + * quadratic blow-up. Also pins unsigned key ordering: baseSeqs with the + * sign bit set must sort above {@code Long.MAX_VALUE}, not below zero. + *

      + * A healthy client cannot produce these orders (recovery validation + * rejects duplicate or non-contiguous baseSeqs moments after the sort), + * but corrupted-yet-parseable headers or operator file copies feed the + * sort BEFORE validation runs, so the sort itself must stay log-linear. + * Sorts in-memory segments directly: staging 16K adversarial header + * files on disk per pattern would dominate the test's runtime without + * adding coverage. + */ + @Test + public void testAdversarialSegmentOrdersSortLogLinear() throws Exception { + TestUtils.assertMemoryLeak(() -> { + final int n = 16384; + final long bound = 8L * n * (long) (Math.log(n) / Math.log(2)); + + // Organ pipe: 0,1,...,n/2-1,n/2-1,...,1,0. + long[] organPipe = new long[n]; + for (int i = 0; i < n / 2; i++) { + organPipe[i] = i; + organPipe[n - 1 - i] = i; + } + assertAdversarialSortWithinBound("organ-pipe", organPipe, bound); + + long[] allDuplicates = new long[n]; + for (int i = 0; i < n; i++) { + allDuplicates[i] = 42; + } + assertAdversarialSortWithinBound("all-duplicates", allDuplicates, bound); + + long[] fewDistinct = new long[n]; + for (int i = 0; i < n; i++) { + fewDistinct[i] = i % 4; + } + assertAdversarialSortWithinBound("few-distinct", fewDistinct, bound); + + // Musser's median-of-three killer permutation of 1..n. + long[] med3Killer = new long[n]; + int half = n / 2; + for (int i = 1; i <= half; i++) { + if (i % 2 == 1) { + med3Killer[i - 1] = i; + med3Killer[i] = half + i; + } + med3Killer[half + i - 1] = 2L * i; + } + assertAdversarialSortWithinBound("median-of-three-killer", med3Killer, bound); + + // High-bit keys interleaved with small ones: exercises the + // unsigned comparison contract alongside the comparison bound. + long[] unsignedMix = new long[n]; + for (int i = 0; i < n; i++) { + unsignedMix[i] = (i % 2 == 0) ? (0x8000000000000000L | i) : i; + } + assertAdversarialSortWithinBound("unsigned-mix", unsignedMix, bound); + }); + } + + private static void assertAdversarialSortWithinBound(String label, long[] baseSeqs, long bound) { + final long segSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 1; + ObjList list = new ObjList<>(); + try { + for (long baseSeq : baseSeqs) { + list.add(MmapSegment.createInMemory(baseSeq, segSize)); + } + SegmentRing.resetSortComparisons(); + SegmentRing.sortByBaseSeqForTest(list); + long comparisons = SegmentRing.getSortComparisons(); + for (int i = 1, size = list.size(); i < size; i++) { + assertTrue(label + ": unsigned baseSeq order violated at index " + i, + Long.compareUnsigned(list.get(i - 1).baseSeq(), list.get(i).baseSeq()) <= 0); + } + assertTrue(label + " sort took " + comparisons + " comparisons (expected < " + bound + + " = 8 * N * log2(N) for N=" + baseSeqs.length + + "); regression suggests the introsort heapsort fallback stopped engaging", + comparisons < bound); + } finally { + for (int i = 0, size = list.size(); i < size; i++) { + list.get(i).close(); + } + } + } + + @Test + public void testRemovingAcknowledgedPrefixMovesLinearReferences() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long segSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 1L; + long buf = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT); + try { + long previousMoves = -1; + for (int sealedCount = 64; sealedCount <= 256; sealedCount *= 2) { + MmapSegment initial = MmapSegment.createInMemory(0, segSize); + try (SegmentRing ring = new SegmentRing(initial, segSize)) { + for (int i = 0; i < 2 * sealedCount; i++) { + assertEquals(i, ring.appendOrFsn(buf, 1)); + ring.installHotSpare(MmapSegment.createInMemory(ring.nextSeqHint(), segSize)); + } + assertEquals(2L * sealedCount, ring.appendOrFsn(buf, 1)); + ring.acknowledge(sealedCount - 1L); + SegmentRing.resetTrimMovedReferences(); + MmapSegment segment; + int removed = 0; + while ((segment = ring.firstTrimmable()) != null) { + assertTrue(ring.removeTrimmable(segment)); + segment.close(); + removed++; + } + assertEquals(sealedCount, removed); + assertEquals("unacknowledged suffix must remain live", sealedCount, + ring.getSealedSegments().size()); + assertEquals(sealedCount, ring.firstSealed().baseSeq()); + long moves = SegmentRing.getTrimMovedReferences(); + assertTrue("removing " + sealedCount + " entries moved " + moves + + " references; expected amortized linear work", + moves <= 2L * sealedCount); + if (previousMoves >= 0) { + assertTrue("doubling the acknowledged prefix grew moved references from " + + previousMoves + " to " + moves, + moves <= 2L * previousMoves + sealedCount); + } + previousMoves = moves; + } + } + } finally { + Unsafe.free(buf, 1, MemoryTag.NATIVE_DEFAULT); + } + }); + } + @Test public void testMaxBytesPerSegmentReturnsConfiguredValue() throws Exception { // Direct constructor path: the value passed in must round-trip through @@ -643,6 +1119,133 @@ public void testMaxBytesPerSegmentSurvivesOpenExisting() throws Exception { }); } + /** + * Pins findSegmentContaining across the full boundary matrix so the + * O(log N) lookup rewrite is provably semantics-preserving: empty ring, + * miss below the head segment, exact segment base, mid-segment, last FSN + * in a segment, tail sealed segment, active-segment hits, miss above the + * published range, and misses below a trimmed live head (sealedHead > 0). + * Three frames per segment give every sealed segment a distinct base, + * mid and last FSN. + */ + @Test + public void testFindSegmentContainingBoundaryMatrix() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long segSize = MmapSegment.HEADER_SIZE + + 3 * (MmapSegment.FRAME_HEADER_SIZE + 8); + long buf = Unsafe.malloc(8, MemoryTag.NATIVE_DEFAULT); + try { + fillPattern(buf, 8, 0); + MmapSegment initial = MmapSegment.createInMemory(0, segSize); + try (SegmentRing ring = new SegmentRing(initial, segSize)) { + // Empty ring: no frame published anywhere, every lookup misses. + assertNull(ring.findSegmentContaining(-1)); + assertNull(ring.findSegmentContaining(0)); + + for (int i = 0; i < 12; i++) { + if (ring.needsHotSpare()) { + ring.installHotSpare( + MmapSegment.createInMemory(ring.nextSeqHint(), segSize)); + } + assertEquals(i, ring.appendOrFsn(buf, 8)); + } + // Sealed: [0-2], [3-5], [6-8]; active: [9-11]. + for (long fsn = 0; fsn < 12; fsn++) { + MmapSegment seg = ring.findSegmentContaining(fsn); + assertNotNull("fsn " + fsn, seg); + assertEquals("fsn " + fsn, fsn / 3 * 3, seg.baseSeq()); + } + assertSame(ring.getActive(), ring.findSegmentContaining(9)); + assertSame(ring.getActive(), ring.findSegmentContaining(11)); + assertNull(ring.findSegmentContaining(-1)); + assertNull(ring.findSegmentContaining(12)); + assertNull(ring.findSegmentContaining(Long.MIN_VALUE)); + assertNull(ring.findSegmentContaining(Long.MAX_VALUE)); + + // Trim the two fully-ACK'd oldest sealed segments: their + // FSNs must now miss while the surviving window (with a + // non-zero sealedHead) still resolves every live FSN. + ring.acknowledge(5); + MmapSegment trimmable; + int removed = 0; + while ((trimmable = ring.firstTrimmable()) != null) { + assertTrue(ring.removeTrimmable(trimmable)); + trimmable.close(); + removed++; + } + assertEquals(2, removed); + for (long fsn = 0; fsn < 6; fsn++) { + assertNull("trimmed fsn " + fsn, ring.findSegmentContaining(fsn)); + } + for (long fsn = 6; fsn < 12; fsn++) { + MmapSegment seg = ring.findSegmentContaining(fsn); + assertNotNull("fsn " + fsn, seg); + assertEquals("fsn " + fsn, fsn / 3 * 3, seg.baseSeq()); + } + } + } finally { + Unsafe.free(buf, 8, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + /** + * Same lookup pinned over one-frame segments, where every FSN is both a + * segment base and a segment's last FSN: 33 sealed segments (odd live + * count), then a 17-segment trim leaving an even 16-segment window with + * sealedHead well inside the backing list. Complements the three-frame + * matrix by exercising both live-window parities and a larger list. + */ + @Test + public void testFindSegmentContainingSingleFrameSegments() throws Exception { + TestUtils.assertMemoryLeak(() -> { + long segSize = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 1; + long buf = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT); + try { + MmapSegment initial = MmapSegment.createInMemory(0, segSize); + try (SegmentRing ring = new SegmentRing(initial, segSize)) { + for (int i = 0; i < 34; i++) { + assertEquals(i, ring.appendOrFsn(buf, 1)); + ring.installHotSpare( + MmapSegment.createInMemory(ring.nextSeqHint(), segSize)); + } + // Sealed: FSNs 0..32, one frame each; active: [33]. + for (long fsn = 0; fsn <= 33; fsn++) { + MmapSegment seg = ring.findSegmentContaining(fsn); + assertNotNull("fsn " + fsn, seg); + assertEquals("fsn " + fsn, fsn, seg.baseSeq()); + } + assertNull(ring.findSegmentContaining(-1)); + assertNull(ring.findSegmentContaining(34)); + + ring.acknowledge(16); + MmapSegment trimmable; + int removed = 0; + while ((trimmable = ring.firstTrimmable()) != null) { + assertTrue(ring.removeTrimmable(trimmable)); + trimmable.close(); + removed++; + } + assertEquals(17, removed); + for (long fsn = 0; fsn < 17; fsn++) { + assertNull("trimmed fsn " + fsn, ring.findSegmentContaining(fsn)); + } + for (long fsn = 17; fsn <= 33; fsn++) { + MmapSegment seg = ring.findSegmentContaining(fsn); + assertNotNull("fsn " + fsn, seg); + assertEquals("fsn " + fsn, fsn, seg.baseSeq()); + } + } + } finally { + Unsafe.free(buf, 1, MemoryTag.NATIVE_DEFAULT); + } + }); + } + + private static byte[] readFileBytes(String path) throws java.io.IOException { + return java.nio.file.Files.readAllBytes(Paths.get(path)); + } + private static void fillPattern(long addr, int len, int seed) { for (int i = 0; i < len; i++) { Unsafe.getUnsafe().putByte(addr + i, (byte) (seed * 31 + i + 17)); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SfManifestClampTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SfManifestClampTest.java new file mode 100644 index 00000000..426c6d1f --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SfManifestClampTest.java @@ -0,0 +1,109 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.qwp.client.sf.cursor.SfManifest; + +import io.questdb.client.std.FilesFacade; +import io.questdb.client.test.tools.TestUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +/** + * Direct pin on {@link SfManifest#update}'s monotonic clamp. Committed + * boundaries only ever move forward (head advances on trim, active on + * rotation); the two writers are serialized on the ring monitor but may each + * compute arguments from a snapshot the other has already moved past. + * Regressing a durable boundary would let a later crash-recovery demand a + * segment file the trim path already unlinked (permanent "missing head + * segment" startup failure) or re-expose stale files below a committed head. + *

      + * Before this test existed, deleting the clamp passed the entire sf suite + * (verified by live mutation run) -- the same blindness class as the + * neutralized rotation gate that commit 88d6b792 accidentally shipped. This + * test lives in the production package deliberately: the manifest API is + * package-private and the clamp deserves a direct unit pin, not an + * integration-distance one. + */ +public class SfManifestClampTest { + + private String tmpDir; + + @Before + public void setUp() { + tmpDir = TestUtils.createTmpDir("qdb-sf-manifest-clamp-"); + } + + @After + public void tearDown() { + TestUtils.removeTmpDir(tmpDir); + } + + @Test + public void testUpdateClampsRegressingBoundariesAndPersistsForward() throws Exception { + TestUtils.assertMemoryLeak(() -> { + FilesFacade ff = FilesFacade.INSTANCE; + SfManifest manifest = SfManifest.create(ff, tmpDir, 10L, 20L); + try { + assertEquals(10L, manifest.headBase()); + assertEquals(20L, manifest.activeBase()); + + // A fully regressing update must be clamped on both fields. + manifest.update(5L, 15L); + assertEquals("regressing headBase must be clamped", 10L, manifest.headBase()); + assertEquals("regressing activeBase must be clamped", 20L, manifest.activeBase()); + + // Fields clamp independently: head may advance while a stale + // active snapshot is clamped in the same call. + manifest.update(12L, 18L); + assertEquals(12L, manifest.headBase()); + assertEquals("stale activeBase snapshot must be clamped independently", + 20L, manifest.activeBase()); + + // Forward motion is untouched. + manifest.update(12L, 25L); + assertEquals(12L, manifest.headBase()); + assertEquals(25L, manifest.activeBase()); + } finally { + manifest.close(); + } + + // The clamp must hold in the durable record, not just in memory: + // reopen and verify the forward-only boundaries survived. + SfManifest reopened = SfManifest.open(ff, tmpDir); + assertNotNull("manifest must reopen", reopened); + try { + assertEquals(12L, reopened.headBase()); + assertEquals(25L, reopened.activeBase()); + } finally { + reopened.close(); + } + }); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java index 644a7463..3529a5d9 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java @@ -25,6 +25,7 @@ package io.questdb.client.test.cutlass.qwp.client.sf.cursor; import io.questdb.client.cutlass.qwp.client.sf.cursor.SlotLock; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SlotLockContentionException; import io.questdb.client.std.Files; import io.questdb.client.test.tools.TestUtils; import org.junit.After; @@ -34,6 +35,7 @@ import java.nio.file.Paths; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -75,6 +77,8 @@ public void testSecondAcquireFailsOnLockContention() throws Exception { try (SlotLock ignored = SlotLock.acquire(slot)) { fail("expected slot contention to throw"); } catch (IllegalStateException expected) { + assertTrue("contention must have a typed signal", + expected instanceof SlotLockContentionException); String msg = expected.getMessage(); assertTrue("error must mention contention: " + msg, msg.contains("already in use")); @@ -103,6 +107,161 @@ public void testCloseReleasesLock() throws Exception { }); } + @Test + public void testReleaseConfirmsAndIsIdempotent() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String slot = parentDir + "/verified-release"; + SlotLock lock = SlotLock.acquire(slot); + assertTrue("first release must confirm success", lock.release()); + // Idempotent: an already-released lock keeps reporting true — + // callers gating a "slot reusable" signal on it must never see + // a spurious false after a confirmed release. + assertTrue("repeat release must stay true", lock.release()); + // close() after release() is a safe no-op (QuietCloseable path). + lock.close(); + // Confirmed release means the slot is genuinely acquirable. + try (SlotLock again = SlotLock.acquire(slot)) { + assertEquals(slot, again.slotDir()); + } + }); + } + + /** + * The {@code release() == false} branch: when the OS reports an explicit + * unlock failure, release must (a) return {@code false} so owners gating a + * "slot reusable" signal never see a lie, (b) retain the fd because the + * non-consuming unlock can safely be retried, and (c) keep returning + * {@code false} while the failure persists. Once unlock succeeds, release + * confirms and stays confirmed. Swapping in a known-bad descriptor gives + * the slot-specific native primitive a deterministic unlock failure. + */ + @Test + public void testFailedCloseRetainsRetryOwnerUntilNextAcquire() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String slot = parentDir + "/failed-close"; + SlotLock lock = SlotLock.acquire(slot); + SlotLock.ReleaseFailureForTesting releaseFailure = + lock.injectReleaseFailureForTesting(); + try { + // CursorSendEngine construction cleanup uses QuietCloseable.close(). + // A failed close must retain an owner that a later acquire can + // drive, rather than dropping the sole reference and flock fd. + lock.close(); + assertTrue("failed close must retain the injected descriptor", + lock.isReleaseFailureInjectedForTesting()); + try (SlotLock ignored = SlotLock.acquire(slot)) { + fail("slot must stay locked while the release failure persists"); + } catch (SlotLockContentionException expected) { + // The retained real flock still protects the slot. + } + + releaseFailure.close(); + try (SlotLock again = SlotLock.acquire(slot)) { + assertEquals("the next acquire must retry the retained release owner", + slot, again.slotDir()); + } catch (SlotLockContentionException stillHeld) { + fail("failed construction cleanup dropped its retry owner: " + + stillHeld.getMessage()); + } + } finally { + releaseFailure.close(); + lock.release(); + } + }); + } + + @Test + public void testFailedCloseRetainsRetryOwnerWithEquivalentPathAlias() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String slot = parentDir + "/failed-close-alias"; + SlotLock lock = SlotLock.acquire(slot); + SlotLock.ReleaseFailureForTesting releaseFailure = + lock.injectReleaseFailureForTesting(); + try { + lock.close(); + assertTrue("failed close must retain the injected descriptor", + lock.isReleaseFailureInjectedForTesting()); + + // Restore the real descriptor so the retained owner can make + // progress. The trailing separator preserves the caller's + // spelling but names the same physical .lock file. + releaseFailure.close(); + String slotAlias = slot + "/"; + try (SlotLock again = SlotLock.acquire(slotAlias)) { + assertEquals("an equivalent path must drive the retained release owner", + slotAlias, again.slotDir()); + } catch (SlotLockContentionException stillHeld) { + fail("equivalent path spelling could not find its retry owner: " + + stillHeld.getMessage()); + } + } finally { + releaseFailure.close(); + lock.release(); + } + }); + } + + @Test + public void testPersistentFailedCloseDoesNotBlockDifferentSlot() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String failedSlot = parentDir + "/persistent-failed-close"; + SlotLock failedLock = SlotLock.acquire(failedSlot); + SlotLock.ReleaseFailureForTesting releaseFailure = + failedLock.injectReleaseFailureForTesting(); + try { + failedLock.close(); + // A repeat close must not enqueue the same intrusive node twice. + failedLock.close(); + + String independentSlot = parentDir + "/independent"; + try (SlotLock independent = SlotLock.acquire(independentSlot)) { + assertEquals("a persistent failure on one slot must not block another", + independentSlot, independent.slotDir()); + } + + releaseFailure.close(); + String progressSlot = parentDir + "/progress"; + try (SlotLock ignored = SlotLock.acquire(progressSlot)) { + // Any cold acquisition drives every failed-close owner. + } + try (SlotLock reacquired = SlotLock.acquire(failedSlot)) { + assertEquals("successful retry must remove the pending list entry", + failedSlot, reacquired.slotDir()); + } + } finally { + releaseFailure.close(); + failedLock.release(); + } + }); + } + + @Test + public void testFailedUnlockRetainsFdAndReportsFalse() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String slot = parentDir + "/failed-release"; + SlotLock lock = SlotLock.acquire(slot); + try (SlotLock.ReleaseFailureForTesting ignored = lock.injectReleaseFailureForTesting()) { + assertFalse("release must report false when explicit unlock fails", + lock.release()); + assertTrue("failed unlock must retain the injected fd for a safe retry", + lock.isReleaseFailureInjectedForTesting()); + assertFalse("repeat release must stay false while unlock keeps failing", + lock.release()); + // While the release is unconfirmed the real flock remains held. + try (SlotLock ignoredLock = SlotLock.acquire(slot)) { + fail("slot must not be acquirable while the original flock fd is still open"); + } catch (IllegalStateException expected) { + // good - unconfirmed release really means "still locked". + } + } + assertTrue("release must confirm once explicit unlock succeeds", lock.release()); + assertTrue("confirmed release must stay confirmed", lock.release()); + try (SlotLock again = SlotLock.acquire(slot)) { + assertEquals(slot, again.slotDir()); + } + }); + } + @Test public void testTwoDifferentSlotsCoexist() throws Exception { TestUtils.assertMemoryLeak(() -> { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java index 6d4c5ef0..139a445f 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java @@ -64,6 +64,8 @@ public class TestWebSocketServer implements Closeable { // client-side pool actually closed the connections it opened. private final AtomicInteger liveConnections = new AtomicInteger(); private final int port; + private final AtomicInteger roleRejectCount = new AtomicInteger(); + private final CountDownLatch roleRejectLatch = new CountDownLatch(1); private final AtomicBoolean running = new AtomicBoolean(false); private final ServerSocket serverSocket; private final CountDownLatch startLatch = new CountDownLatch(1); @@ -165,6 +167,10 @@ public TestWebSocketServer(WebSocketServerHandler handler, this.port = serverSocket.getLocalPort(); } + public boolean awaitRoleReject(long timeout, TimeUnit unit) throws InterruptedException { + return roleRejectLatch.await(timeout, unit); + } + public boolean awaitStart(long timeout, TimeUnit unit) throws InterruptedException { return startLatch.await(timeout, unit); } @@ -221,6 +227,13 @@ public int liveConnectionCount() { return liveConnections.get(); } + /** + * Number of HTTP 421 role-reject responses sent over the server's lifetime. + */ + public int roleRejectCount() { + return roleRejectCount.get(); + } + /** * Replaces the advertised role for subsequent handshakes (live update). */ @@ -586,6 +599,8 @@ private boolean performHandshake() throws IOException { "\r\n"; out.write(sb.getBytes(StandardCharsets.US_ASCII)); out.flush(); + roleRejectCount.incrementAndGet(); + roleRejectLatch.countDown(); return false; } diff --git a/core/src/test/java/io/questdb/client/test/impl/QueryClientPoolErrorSafetyTest.java b/core/src/test/java/io/questdb/client/test/impl/QueryClientPoolErrorSafetyTest.java index bc36d1ae..2978ff46 100644 --- a/core/src/test/java/io/questdb/client/test/impl/QueryClientPoolErrorSafetyTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/QueryClientPoolErrorSafetyTest.java @@ -230,16 +230,31 @@ public void preWarmDoesNotLeakNativeScratchOnErrorFromStart() throws Exception { // the dispatch thread for up to SHUTDOWN_JOIN_MILLIS; cancelIfCurrent's // contract is that this lock is "held only briefly") -- and still reclaim // everything: the client's NATIVE_DEFAULT scratch and the creation slot. - // RED (teardown dropped in the restructure): the scratch leaks and the - // baseline assertion fails. - // GREEN: shutdown() runs after the lock is released -> no leak, accounting - // restored, acquire() surfaces the closed pool. + // RED (rendering a throwing before-close hook's failure also throws after + // close ownership is won): QueryWorker suppresses that second failure, + // the pool releases its creation reservation, and the client's native + // scratch remains live. + // GREEN: close() suppresses the test-witness failure without rendering it, + // completes production teardown, and only then lets the pool release + // the reservation. @Test(timeout = 30_000) public void closedMidCreationTearsDownFreshWorkerWithoutLeak() throws Exception { TestUtils.assertMemoryLeak(() -> { CountDownLatch inConnect = new CountDownLatch(1); CountDownLatch releaseConnect = new CountDownLatch(1); + AtomicInteger closeHookCalls = new AtomicInteger(); + AtomicReference createdClient = new AtomicReference<>(); Consumer connectHook = client -> { + createdClient.set(client); + client.setBeforeCloseHookForTest(() -> { + closeHookCalls.incrementAndGet(); + throw new AssertionError("injected query close hook failure") { + @Override + public String toString() { + throw new AssertionError("injected throwable rendering failure"); + } + }; + }); inConnect.countDown(); try { if (!releaseConnect.await(10, TimeUnit.SECONDS)) { @@ -268,25 +283,35 @@ public void closedMidCreationTearsDownFreshWorkerWithoutLeak() throws Exception Assert.assertTrue("acquirer never reached connect()", inConnect.await(10, TimeUnit.SECONDS)); - // Close the pool while the worker build is in flight (the fresh - // worker never entered `all`, so close()'s snapshot skips it), - // then let the build finish: acquire() must observe `closed`, - // tear the worker down on its own thread and throw. - pool.close(); + // Close the pool while the worker build is in flight. close() + // now waits for the internal creation reservation, so run it + // concurrently and wait until it parks on that reservation + // before releasing connect(). + Thread closer = new Thread(pool::close, "mid-creation-pool-closer"); + closer.start(); + awaitCreationWaiter(pool); + + // Let the build finish: acquire() must observe `closed`, tear + // the worker down on its own thread and throw. close() returns + // only after that teardown releases the reservation. releaseConnect.countDown(); acquirer.join(TimeUnit.SECONDS.toMillis(10)); + closer.join(TimeUnit.SECONDS.toMillis(10)); Assert.assertFalse("acquirer did not finish", acquirer.isAlive()); + Assert.assertFalse("pool close did not finish", closer.isAlive()); Assert.assertTrue( "acquire() must surface the closed pool, got: " + acquireOutcome.get(), acquireOutcome.get() instanceof QueryException && String.valueOf(acquireOutcome.get().getMessage()).contains("closed")); + Assert.assertEquals("pool released the creation reservation after teardown", + 0, inFlightCreations(pool)); long after = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_DEFAULT); Assert.assertEquals( - "closed-mid-creation acquire() leaked the fresh worker's NATIVE_DEFAULT scratch", + "throwing close hook prevented fresh worker teardown before reservation release", baseline, after); - Assert.assertEquals("in-flight creation accounting must be restored", - 0, inFlightCreations(pool)); + createdClient.get().close(); + Assert.assertEquals("close hook must be one-shot", 1, closeHookCalls.get()); } finally { pool.close(); } @@ -299,6 +324,17 @@ private static Consumer alwaysThrow() { }; } + private static void awaitCreationWaiter(QueryClientPool pool) { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (System.nanoTime() < deadline) { + if (pool.hasCreationWaiterForTesting()) { + return; + } + Thread.yield(); + } + Assert.fail("pool close did not wait on the creation reservation"); + } + // connectHook that connects nothing: fromConfig() has already committed the // NATIVE_DEFAULT scratch, so the client is half-built (scratch, no socket) // -- exactly the state createUnlocked() returns before start() runs. diff --git a/core/src/test/java/io/questdb/client/test/impl/QuestDBImplCloseLifecycleTest.java b/core/src/test/java/io/questdb/client/test/impl/QuestDBImplCloseLifecycleTest.java new file mode 100644 index 00000000..1bbf7058 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/impl/QuestDBImplCloseLifecycleTest.java @@ -0,0 +1,600 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.impl; + +import io.questdb.client.QueryException; +import io.questdb.client.Sender; +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.QwpQueryClient; +import io.questdb.client.impl.QueryClientPool; +import io.questdb.client.impl.QuestDBImpl; +import io.questdb.client.impl.SenderPool; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.lang.reflect.Proxy; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; +import java.util.function.IntFunction; + +public class QuestDBImplCloseLifecycleTest { + + private static final String QUERY_CFG = "ws::addr=127.0.0.1:9000;"; + private static final String SENDER_CFG = "http::addr=127.0.0.1:1;protocol_version=2;auto_flush=off;"; + + @Test(timeout = 30_000) + public void facadeCloseIsBoundedByZeroAcquireTimeoutDuringQueryCreation() throws Exception { + TestUtils.assertMemoryLeak(() -> { + CountDownLatch inCreation = new CountDownLatch(1); + CountDownLatch releaseCreation = new CountDownLatch(1); + AtomicInteger teardownCount = new AtomicInteger(); + Consumer connectHook = client -> { + client.setBeforeCloseHookForTest(teardownCount::incrementAndGet); + inCreation.countDown(); + awaitOrFail(releaseCreation, "test never released query creation"); + }; + QuestDBImpl db = newQuestDB( + SENDER_CFG, 0, 0, 0, slotIndex -> fakeSender(null, null, null), connectHook); + QueryClientPool pool = db.getQueryPoolForTesting(); + AtomicReference borrowOutcome = new AtomicReference<>(); + Thread borrower = new Thread(() -> { + try { + db.borrowQuery(); + } catch (Throwable t) { + borrowOutcome.set(t); + } + }, "bounded-query-borrower"); + Thread closer = new Thread(db::close, "bounded-query-closer"); + long nativeBaseline = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_DEFAULT); + try { + borrower.start(); + Assert.assertTrue("query borrow never reached construction", + inCreation.await(10, TimeUnit.SECONDS)); + Assert.assertEquals(1, pool.inFlightCreations()); + + closer.start(); + closer.join(TimeUnit.SECONDS.toMillis(5)); + Assert.assertFalse( + "facade close exceeded its zero creation-wait budget", + closer.isAlive()); + Assert.assertEquals( + "close must retain late-completion cleanup ownership", + 1, pool.inFlightCreations()); + Assert.assertEquals("the still-constructing query client must remain live", 0, teardownCount.get()); + + releaseCreation.countDown(); + borrower.join(TimeUnit.SECONDS.toMillis(10)); + Assert.assertFalse("query borrower did not finish", borrower.isAlive()); + Assert.assertEquals("late query creation must be torn down exactly once", 1, teardownCount.get()); + Assert.assertEquals("late query creation reservation must be released", 0, pool.inFlightCreations()); + Assert.assertEquals("late query cleanup must release native scratch", + nativeBaseline, Unsafe.getMemUsedByTag(MemoryTag.NATIVE_DEFAULT)); + Assert.assertTrue("borrowQuery() must report facade closure, got: " + borrowOutcome.get(), + borrowOutcome.get() instanceof QueryException + && String.valueOf(borrowOutcome.get().getMessage()).contains("closed")); + } finally { + releaseCreation.countDown(); + db.close(); + borrower.join(TimeUnit.SECONDS.toMillis(10)); + closer.join(TimeUnit.SECONDS.toMillis(10)); + } + }); + } + + @Test(timeout = 30_000) + public void facadeCloseIsBoundedByZeroAcquireTimeoutDuringSenderCreation() throws Exception { + TestUtils.assertMemoryLeak(() -> { + CountDownLatch inCreation = new CountDownLatch(1); + CountDownLatch releaseCreation = new CountDownLatch(1); + AtomicInteger teardownCount = new AtomicInteger(); + IntFunction senderFactory = slotIndex -> { + inCreation.countDown(); + awaitOrFail(releaseCreation, "test never released sender creation"); + return fakeSender(teardownCount, null, null); + }; + String senderConfig = "ws::addr=localhost:1;sf_dir=" + + System.getProperty("java.io.tmpdir") + "/qdb-bounded-pool-" + System.nanoTime() + ";"; + QuestDBImpl db = newQuestDB(senderConfig, 0, 0, 0, senderFactory, client -> { + }); + SenderPool pool = db.getSenderPoolForTesting(); + AtomicReference borrowOutcome = new AtomicReference<>(); + Thread borrower = new Thread(() -> { + try { + db.borrowSender(); + } catch (Throwable t) { + borrowOutcome.set(t); + } + }, "bounded-sender-borrower"); + Thread closer = new Thread(db::close, "bounded-sender-closer"); + try { + borrower.start(); + Assert.assertTrue("sender borrow never reached construction", + inCreation.await(10, TimeUnit.SECONDS)); + Assert.assertEquals(1, pool.getInFlightCreationsForTesting()); + Assert.assertTrue("SF slot must stay reserved during creation", + pool.isSlotInUseForTesting(0)); + + closer.start(); + closer.join(TimeUnit.SECONDS.toMillis(5)); + Assert.assertFalse( + "facade close exceeded its zero creation-wait budget", + closer.isAlive()); + Assert.assertEquals( + "close must retain late-completion cleanup ownership", + 1, pool.getInFlightCreationsForTesting()); + Assert.assertTrue("close must not abandon the reserved SF slot", + pool.isSlotInUseForTesting(0)); + + releaseCreation.countDown(); + borrower.join(TimeUnit.SECONDS.toMillis(10)); + Assert.assertFalse("sender borrower did not finish", borrower.isAlive()); + Assert.assertEquals("late sender creation must be torn down exactly once", 1, teardownCount.get()); + Assert.assertEquals("late sender creation reservation must be released", + 0, pool.getInFlightCreationsForTesting()); + Assert.assertFalse("late sender cleanup must release the SF slot", + pool.isSlotInUseForTesting(0)); + Assert.assertTrue("borrowSender() must report facade closure, got: " + borrowOutcome.get(), + borrowOutcome.get() instanceof LineSenderException + && String.valueOf(borrowOutcome.get().getMessage()).contains("closed")); + } finally { + releaseCreation.countDown(); + db.close(); + borrower.join(TimeUnit.SECONDS.toMillis(10)); + closer.join(TimeUnit.SECONDS.toMillis(10)); + } + }); + } + + @Test(timeout = 30_000) + public void facadeCloseIsBoundedUnderRepeatedInterruptsDuringQueryCreation() throws Exception { + TestUtils.assertMemoryLeak(() -> { + CountDownLatch inCreation = new CountDownLatch(1); + CountDownLatch releaseCreation = new CountDownLatch(1); + AtomicInteger teardownCount = new AtomicInteger(); + Consumer connectHook = client -> { + client.setBeforeCloseHookForTest(teardownCount::incrementAndGet); + inCreation.countDown(); + awaitOrFail(releaseCreation, "test never released query creation"); + }; + QuestDBImpl db = newQuestDB( + SENDER_CFG, 0, 0, 100, slotIndex -> fakeSender(null, null, null), connectHook); + QueryClientPool pool = db.getQueryPoolForTesting(); + AtomicReference borrowOutcome = new AtomicReference<>(); + AtomicBoolean closeReturnedInterrupted = new AtomicBoolean(); + AtomicBoolean keepInterrupting = new AtomicBoolean(true); + AtomicInteger interruptCount = new AtomicInteger(); + Thread borrower = new Thread(() -> { + try { + db.borrowQuery(); + } catch (Throwable t) { + borrowOutcome.set(t); + } + }, "interrupted-query-borrower"); + Thread closer = new Thread(() -> { + db.close(); + closeReturnedInterrupted.set(Thread.currentThread().isInterrupted()); + }, "interrupted-query-closer"); + Thread interrupter = new Thread(() -> { + while (keepInterrupting.get()) { + interruptCount.incrementAndGet(); + closer.interrupt(); + Thread.yield(); + } + }, "query-close-interrupter"); + long nativeBaseline = Unsafe.getMemUsedByTag(MemoryTag.NATIVE_DEFAULT); + try { + borrower.start(); + Assert.assertTrue("query borrow never reached construction", + inCreation.await(10, TimeUnit.SECONDS)); + Assert.assertEquals(1, pool.inFlightCreations()); + + closer.start(); + awaitCreationWaiter(pool, + "facade close did not wait while query construction was internally owned"); + interrupter.start(); + closer.join(TimeUnit.SECONDS.toMillis(5)); + Assert.assertFalse( + "repeated interrupts restarted the query creation-wait deadline", + closer.isAlive()); + Assert.assertTrue("test did not repeatedly interrupt query close", interruptCount.get() > 1); + Assert.assertTrue("facade close must restore query closer interruption", + closeReturnedInterrupted.get()); + Assert.assertEquals( + "close must retain late-completion cleanup ownership", + 1, pool.inFlightCreations()); + Assert.assertEquals("the still-constructing query client must remain live", 0, teardownCount.get()); + + releaseCreation.countDown(); + borrower.join(TimeUnit.SECONDS.toMillis(10)); + Assert.assertFalse("query borrower did not finish", borrower.isAlive()); + Assert.assertEquals("late query creation must be torn down exactly once", 1, teardownCount.get()); + Assert.assertEquals("late query creation reservation must be released", 0, pool.inFlightCreations()); + Assert.assertEquals("late query cleanup must release native scratch", + nativeBaseline, Unsafe.getMemUsedByTag(MemoryTag.NATIVE_DEFAULT)); + Assert.assertTrue("borrowQuery() must report facade closure, got: " + borrowOutcome.get(), + borrowOutcome.get() instanceof QueryException + && String.valueOf(borrowOutcome.get().getMessage()).contains("closed")); + } finally { + keepInterrupting.set(false); + releaseCreation.countDown(); + interrupter.join(TimeUnit.SECONDS.toMillis(10)); + db.close(); + borrower.join(TimeUnit.SECONDS.toMillis(10)); + closer.join(TimeUnit.SECONDS.toMillis(10)); + } + }); + } + + @Test(timeout = 30_000) + public void facadeCloseIsBoundedUnderRepeatedInterruptsDuringSenderCreation() throws Exception { + TestUtils.assertMemoryLeak(() -> { + CountDownLatch inCreation = new CountDownLatch(1); + CountDownLatch releaseCreation = new CountDownLatch(1); + AtomicInteger teardownCount = new AtomicInteger(); + IntFunction senderFactory = slotIndex -> { + inCreation.countDown(); + awaitOrFail(releaseCreation, "test never released sender creation"); + return fakeSender(teardownCount, null, null); + }; + String senderConfig = "ws::addr=localhost:1;sf_dir=" + + System.getProperty("java.io.tmpdir") + "/qdb-interrupted-pool-" + System.nanoTime() + ";"; + QuestDBImpl db = newQuestDB(senderConfig, 0, 0, 100, senderFactory, client -> { + }); + SenderPool pool = db.getSenderPoolForTesting(); + AtomicReference borrowOutcome = new AtomicReference<>(); + AtomicBoolean closeReturnedInterrupted = new AtomicBoolean(); + AtomicBoolean keepInterrupting = new AtomicBoolean(true); + AtomicInteger interruptCount = new AtomicInteger(); + Thread borrower = new Thread(() -> { + try { + db.borrowSender(); + } catch (Throwable t) { + borrowOutcome.set(t); + } + }, "interrupted-sender-borrower"); + Thread closer = new Thread(() -> { + db.close(); + closeReturnedInterrupted.set(Thread.currentThread().isInterrupted()); + }, "interrupted-sender-closer"); + Thread interrupter = new Thread(() -> { + while (keepInterrupting.get()) { + interruptCount.incrementAndGet(); + closer.interrupt(); + Thread.yield(); + } + }, "sender-close-interrupter"); + try { + borrower.start(); + Assert.assertTrue("sender borrow never reached construction", + inCreation.await(10, TimeUnit.SECONDS)); + Assert.assertEquals(1, pool.getInFlightCreationsForTesting()); + Assert.assertTrue("SF slot must stay reserved during creation", + pool.isSlotInUseForTesting(0)); + + closer.start(); + awaitCreationWaiter(pool, + "facade close did not wait while sender construction was internally owned"); + interrupter.start(); + closer.join(TimeUnit.SECONDS.toMillis(5)); + Assert.assertFalse( + "repeated interrupts restarted the sender creation-wait deadline", + closer.isAlive()); + Assert.assertTrue("test did not repeatedly interrupt sender close", interruptCount.get() > 1); + Assert.assertTrue("facade close must restore sender closer interruption", + closeReturnedInterrupted.get()); + Assert.assertEquals( + "close must retain late-completion cleanup ownership", + 1, pool.getInFlightCreationsForTesting()); + Assert.assertTrue("close must not abandon the reserved SF slot", + pool.isSlotInUseForTesting(0)); + + releaseCreation.countDown(); + borrower.join(TimeUnit.SECONDS.toMillis(10)); + Assert.assertFalse("sender borrower did not finish", borrower.isAlive()); + Assert.assertEquals("late sender creation must be torn down exactly once", 1, teardownCount.get()); + Assert.assertEquals("late sender creation reservation must be released", + 0, pool.getInFlightCreationsForTesting()); + Assert.assertFalse("late sender cleanup must release the SF slot", + pool.isSlotInUseForTesting(0)); + Assert.assertTrue("borrowSender() must report facade closure, got: " + borrowOutcome.get(), + borrowOutcome.get() instanceof LineSenderException + && String.valueOf(borrowOutcome.get().getMessage()).contains("closed")); + } finally { + keepInterrupting.set(false); + releaseCreation.countDown(); + interrupter.join(TimeUnit.SECONDS.toMillis(10)); + db.close(); + borrower.join(TimeUnit.SECONDS.toMillis(10)); + closer.join(TimeUnit.SECONDS.toMillis(10)); + } + }); + } + + @Test(timeout = 30_000) + public void facadeCloseWaitsForQueryCreationAndTeardown() throws Exception { + TestUtils.assertMemoryLeak(() -> { + CountDownLatch inCreation = new CountDownLatch(1); + CountDownLatch releaseCreation = new CountDownLatch(1); + CountDownLatch inTeardown = new CountDownLatch(1); + CountDownLatch releaseTeardown = new CountDownLatch(1); + AtomicInteger teardownCount = new AtomicInteger(); + Consumer connectHook = client -> { + client.setBeforeCloseHookForTest(() -> { + inTeardown.countDown(); + awaitOrFail(releaseTeardown, "test never released query teardown"); + teardownCount.incrementAndGet(); + }); + inCreation.countDown(); + awaitOrFail(releaseCreation, "test never released query creation"); + }; + QuestDBImpl db = newQuestDB(0, 0, slotIndex -> fakeSender(null, null, null), connectHook); + AtomicReference borrowOutcome = new AtomicReference<>(); + AtomicBoolean closeReturnedInterrupted = new AtomicBoolean(); + Thread borrower = new Thread(() -> { + try { + db.borrowQuery(); + } catch (Throwable t) { + borrowOutcome.set(t); + } + }, "facade-query-borrower"); + Thread closer = new Thread(() -> { + db.close(); + closeReturnedInterrupted.set(Thread.currentThread().isInterrupted()); + }, "facade-query-closer"); + try { + borrower.start(); + Assert.assertTrue("query borrow never reached construction", + inCreation.await(10, TimeUnit.SECONDS)); + + closer.start(); + QueryClientPool pool = db.getQueryPoolForTesting(); + awaitClosed(pool); + awaitCreationWaiter(pool, + "facade close did not wait while query construction was internally owned"); + closer.interrupt(); + awaitCreationWaiter(pool, + "interrupt allowed facade close to abandon query creation ownership"); + + releaseCreation.countDown(); + Assert.assertTrue("query borrow never reached closed-mid-creation teardown", + inTeardown.await(10, TimeUnit.SECONDS)); + awaitCreationWaiter(pool, + "facade close abandoned query ownership during internal teardown"); + + releaseTeardown.countDown(); + borrower.join(TimeUnit.SECONDS.toMillis(10)); + closer.join(TimeUnit.SECONDS.toMillis(10)); + Assert.assertFalse("query borrower did not finish", borrower.isAlive()); + Assert.assertFalse("facade close did not finish", closer.isAlive()); + Assert.assertEquals("internally owned query client must be torn down exactly once", + 1, teardownCount.get()); + Assert.assertTrue("facade close must preserve interruption after internal teardown", + closeReturnedInterrupted.get()); + Assert.assertTrue("borrowQuery() must report facade closure, got: " + borrowOutcome.get(), + borrowOutcome.get() instanceof QueryException + && String.valueOf(borrowOutcome.get().getMessage()).contains("closed")); + } finally { + releaseCreation.countDown(); + releaseTeardown.countDown(); + db.close(); + borrower.join(TimeUnit.SECONDS.toMillis(10)); + closer.join(TimeUnit.SECONDS.toMillis(10)); + } + }); + } + + @Test(timeout = 30_000) + public void facadeCloseWaitsForSenderCreationAndTeardown() throws Exception { + TestUtils.assertMemoryLeak(() -> { + CountDownLatch inCreation = new CountDownLatch(1); + CountDownLatch releaseCreation = new CountDownLatch(1); + CountDownLatch inTeardown = new CountDownLatch(1); + CountDownLatch releaseTeardown = new CountDownLatch(1); + AtomicInteger teardownCount = new AtomicInteger(); + IntFunction senderFactory = slotIndex -> { + inCreation.countDown(); + awaitOrFail(releaseCreation, "test never released sender creation"); + return fakeSender(teardownCount, inTeardown, releaseTeardown); + }; + QuestDBImpl db = newQuestDB(0, 0, senderFactory, client -> { + }); + AtomicReference borrowOutcome = new AtomicReference<>(); + Thread borrower = new Thread(() -> { + try { + db.borrowSender(); + } catch (Throwable t) { + borrowOutcome.set(t); + } + }, "facade-sender-borrower"); + Thread closer = new Thread(db::close, "facade-sender-closer"); + try { + borrower.start(); + Assert.assertTrue("sender borrow never reached construction", + inCreation.await(10, TimeUnit.SECONDS)); + + closer.start(); + SenderPool pool = db.getSenderPoolForTesting(); + awaitCloseStarted(pool); + awaitCreationWaiter(pool, + "facade close did not wait while sender construction was internally owned"); + + releaseCreation.countDown(); + Assert.assertTrue("sender borrow never reached closed-mid-creation teardown", + inTeardown.await(10, TimeUnit.SECONDS)); + awaitCreationWaiter(pool, + "facade close abandoned sender ownership during internal teardown"); + + releaseTeardown.countDown(); + borrower.join(TimeUnit.SECONDS.toMillis(10)); + closer.join(TimeUnit.SECONDS.toMillis(10)); + Assert.assertFalse("sender borrower did not finish", borrower.isAlive()); + Assert.assertFalse("facade close did not finish", closer.isAlive()); + Assert.assertEquals("internally owned sender must be torn down exactly once", + 1, teardownCount.get()); + Assert.assertTrue("borrowSender() must report facade closure, got: " + borrowOutcome.get(), + borrowOutcome.get() instanceof LineSenderException + && String.valueOf(borrowOutcome.get().getMessage()).contains("closed")); + } finally { + releaseCreation.countDown(); + releaseTeardown.countDown(); + db.close(); + borrower.join(TimeUnit.SECONDS.toMillis(10)); + closer.join(TimeUnit.SECONDS.toMillis(10)); + } + }); + } + + private static void awaitCloseStarted(SenderPool pool) { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (System.nanoTime() < deadline) { + if (pool.isCloseStartedForTesting()) { + return; + } + Thread.yield(); + } + Assert.fail("sender pool close did not start"); + } + + private static void awaitClosed(QueryClientPool pool) { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (System.nanoTime() < deadline) { + if (pool.isClosedForTesting()) { + return; + } + Thread.yield(); + } + Assert.fail("query pool did not close"); + } + + private static void awaitCreationWaiter(QueryClientPool pool, String message) { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (System.nanoTime() < deadline) { + if (pool.hasCreationWaiterForTesting()) { + return; + } + Thread.yield(); + } + Assert.fail(message); + } + + private static void awaitCreationWaiter(SenderPool pool, String message) { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (System.nanoTime() < deadline) { + if (pool.hasCreationWaiterForTesting()) { + return; + } + Thread.yield(); + } + Assert.fail(message); + } + + private static void awaitOrFail(CountDownLatch latch, String message) { + try { + if (!latch.await(10, TimeUnit.SECONDS)) { + throw new IllegalStateException(message); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(message, e); + } + } + + private static Sender fakeSender( + AtomicInteger teardownCount, + CountDownLatch inTeardown, + CountDownLatch releaseTeardown + ) { + return (Sender) Proxy.newProxyInstance( + Sender.class.getClassLoader(), + new Class[]{Sender.class}, + (proxy, method, args) -> { + switch (method.getName()) { + case "close": + if (inTeardown != null) { + inTeardown.countDown(); + awaitOrFail(releaseTeardown, "test never released sender teardown"); + } + if (teardownCount != null) { + teardownCount.incrementAndGet(); + } + return null; + case "toString": + return "FacadeCloseFakeSender"; + case "hashCode": + return System.identityHashCode(proxy); + case "equals": + return proxy == args[0]; + default: + Class returnType = method.getReturnType(); + if (returnType == boolean.class) return false; + if (returnType == byte.class) return (byte) 0; + if (returnType == short.class) return (short) 0; + if (returnType == int.class) return 0; + if (returnType == long.class) return 0L; + if (returnType == float.class) return 0f; + if (returnType == double.class) return 0d; + if (returnType == char.class) return (char) 0; + if (returnType == void.class) return null; + if (returnType.isInstance(proxy)) return proxy; + return null; + } + }); + } + + private static QuestDBImpl newQuestDB( + int senderMin, + int queryMin, + IntFunction senderFactory, + Consumer connectHook + ) { + return newQuestDB(SENDER_CFG, senderMin, queryMin, 10_000L, senderFactory, connectHook); + } + + private static QuestDBImpl newQuestDB( + String senderConfig, + int senderMin, + int queryMin, + long acquireTimeoutMillis, + IntFunction senderFactory, + Consumer connectHook + ) { + return new QuestDBImpl( + senderConfig, QUERY_CFG, + senderMin, 1, + queryMin, 1, + acquireTimeoutMillis, + Long.MAX_VALUE, + Long.MAX_VALUE, + Long.MAX_VALUE, + senderFactory, connectHook); + } +} diff --git a/core/src/test/java/io/questdb/client/test/impl/QuestDBImplCloseTest.java b/core/src/test/java/io/questdb/client/test/impl/QuestDBImplCloseTest.java new file mode 100644 index 00000000..c4a33e9c --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/impl/QuestDBImplCloseTest.java @@ -0,0 +1,191 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.impl; + +import io.questdb.client.Sender; +import io.questdb.client.cutlass.qwp.client.QwpQueryClient; +import io.questdb.client.impl.QuestDBImpl; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.lang.reflect.Proxy; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; +import java.util.function.IntFunction; + +/** + * P2 regression: {@link QuestDBImpl#close()} must not return before shutdown + * has completed, even when two threads call it concurrently. {@code closed} is + * volatile and set BEFORE the pool teardown chain runs, so a naive guard lets a + * second concurrent caller observe {@code closed == true} and RETURN while the + * first caller is still inside {@code closeQuietly(senderPool)} tearing down the + * flock/mmap/I/O-thread resources. After ANY {@code close()} returns -- the + * losing concurrent caller included -- callers must be able to assume shutdown + * has completed. + *

      + * The window is opened deterministically by injecting (via the {@code @TestOnly} + * senderFactory seam) a fake delegate whose {@code close()} parks on a latch. + * The last teardown step, {@code senderPool.close()}, closes the prewarmed idle + * delegate on the closing thread OUTSIDE the pool lock, so thread A parks there + * with {@code closed} already raised. Thread B then calls {@code close()}: + *

        + *
      • pre-fix -- B reads {@code closed == true} and returns immediately while A + * is still tearing down (premature return);
        • + *
      • fixed -- B blocks until A finishes the teardown, then returns.
        • + *
      + * Latch-coordinated (no {@code Thread.sleep} for correctness) with a JUnit + * timeout on the two-thread interleaving. + */ +public class QuestDBImplCloseTest { + + // Non-SF http config: the injected senderFactory replaces the native build, + // but the constructor's eager config probe must still parse it. + private static final String QUERY_CFG = "ws::addr=127.0.0.1:9000;"; + private static final String SENDER_CFG = "http::addr=127.0.0.1:1;protocol_version=2;auto_flush=off;"; + + // RED (closed set before teardown, no serialization): while thread A is + // parked inside the delegate teardown of the final senderPool.close() step, + // thread B's close() sees closed==true and RETURNS -- closeReturnedEarly is + // true and the first assertion fails. GREEN (close() serialized through + // completion): B blocks on A until the teardown finishes, so its close() + // does not return until delegateCloses == 1. + @Test(timeout = 30_000) + public void concurrentCloseSecondCallerBlocksUntilShutdownCompletes() throws Exception { + TestUtils.assertMemoryLeak(() -> { + AtomicInteger delegateCloses = new AtomicInteger(); + CountDownLatch inDelegateClose = new CountDownLatch(1); + CountDownLatch releaseDelegateClose = new CountDownLatch(1); + IntFunction senderFactory = slotIndex -> + parkingCloseSender(delegateCloses, inDelegateClose, releaseDelegateClose); + // queryMin = 0 -> QueryClientPool prewarms nothing, so the connect + // hook is never reached and its teardown is a no-op; the parking + // delegate is the only blocking teardown step. + Consumer connectHook = client -> { + }; + + QuestDBImpl questDB = newQuestDB(senderFactory, connectHook); + + // Thread A: enter close() and park inside the final teardown step + // (senderPool.close() -> idle delegate close()), with closed + // already raised. + Thread closerA = new Thread(questDB::close, "questdb-closer-A"); + closerA.start(); + Assert.assertTrue("closer A never reached the delegate teardown", + inDelegateClose.await(10, TimeUnit.SECONDS)); + + // Thread B: a concurrent close(). It must NOT return while A is + // still tearing down. + Thread closerB = new Thread(questDB::close, "questdb-closer-B"); + closerB.start(); + closerB.join(300); + boolean closeReturnedEarly = !closerB.isAlive(); + int closesWhenBReturned = delegateCloses.get(); + + // Always unpark the teardown so the test fails on the assertion, not + // its own timeout. + releaseDelegateClose.countDown(); + + Assert.assertFalse( + "concurrent close() returned while the first caller was still tearing down " + + "(delegateCloses=" + closesWhenBReturned + " when B returned): " + + "close() must serialize through shutdown completion", + closeReturnedEarly); + + // Once the teardown completes, B must return promptly and the + // delegate must have been torn down exactly once. + closerB.join(TimeUnit.SECONDS.toMillis(10)); + Assert.assertFalse("concurrent close() did not return after the teardown completed", + closerB.isAlive()); + closerA.join(TimeUnit.SECONDS.toMillis(10)); + Assert.assertFalse("first close() did not return after the teardown completed", + closerA.isAlive()); + Assert.assertEquals("the prewarmed delegate must be torn down exactly once", + 1, delegateCloses.get()); + }); + } + + private static QuestDBImpl newQuestDB( + IntFunction senderFactory, Consumer connectHook + ) { + return new QuestDBImpl( + SENDER_CFG, QUERY_CFG, + /*senderMin*/ 1, /*senderMax*/ 1, + /*queryMin*/ 0, /*queryMax*/ 1, + /*acquireTimeoutMillis*/ 250L, + /*idleTimeoutMillis*/ Long.MAX_VALUE, + /*maxLifetimeMillis*/ Long.MAX_VALUE, + /*housekeeperIntervalMillis*/ Long.MAX_VALUE, + senderFactory, connectHook); + } + + /** + * Proxy-backed fake Sender whose {@code close()} signals {@code inClose}, + * parks on {@code releaseClose}, then bumps {@code closes} -- a delegate + * teardown frozen mid-close so the test can probe what a concurrent + * close() does while it runs. + */ + private static Sender parkingCloseSender( + AtomicInteger closes, + CountDownLatch inClose, + CountDownLatch releaseClose + ) { + return (Sender) Proxy.newProxyInstance( + Sender.class.getClassLoader(), + new Class[]{Sender.class}, + (proxy, method, args) -> { + switch (method.getName()) { + case "close": + inClose.countDown(); + if (!releaseClose.await(10, TimeUnit.SECONDS)) { + throw new IllegalStateException("test never released the parked close"); + } + closes.incrementAndGet(); + return null; + case "toString": + return "ParkingCloseFakeSender"; + case "hashCode": + return System.identityHashCode(proxy); + case "equals": + return proxy == args[0]; + default: + Class rt = method.getReturnType(); + if (rt == boolean.class) return false; + if (rt == byte.class) return (byte) 0; + if (rt == short.class) return (short) 0; + if (rt == int.class) return 0; + if (rt == long.class) return 0L; + if (rt == float.class) return 0f; + if (rt == double.class) return 0d; + if (rt == char.class) return (char) 0; + if (rt == void.class) return null; + if (rt.isInstance(proxy)) return proxy; + return null; + } + }); + } +} diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolCloseLifecycleTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolCloseLifecycleTest.java index b9b0136f..188d9eb5 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolCloseLifecycleTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolCloseLifecycleTest.java @@ -286,10 +286,30 @@ public void closedMidCreationTeardownRunsOutsideThePoolLock() throws Exception { Assert.assertTrue("borrower never reached the factory", inCreate.await(10, TimeUnit.SECONDS)); - // Close the pool while the creation is in flight: nothing is - // outstanding (the new slot never entered `all`), so this - // returns promptly with `closed` raised. - pool.close(); + // Raise the same early shutdown signal used by QuestDBImpl so + // the creation deterministically takes the teardown branch. + pool.markClosingForTesting(); + + // close() now owns the in-flight creation reservation until + // its closed-mid-creation teardown completes, so run it on a + // separate thread while the creation remains parked. + Thread poolCloser = new Thread(pool::close, "pool-closer"); + poolCloser.start(); + + // Make poolCloser the primary closer DETERMINISTICALLY before + // anything else moves: wait until it has claimed closeStarted + // and parked in the bounded in-flight-creation wait (which + // releases the lock). Without this the lock-probe close() + // below can win the closeStarted race, become the primary + // closer itself and legitimately block in that same 5s wait + // (the parked teardown holds inFlightCreations at 1) -- a + // false "lock held" failure under slow thread scheduling. + long closerParkedDeadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (!pool.hasCreationWaiterForTesting()) { + Assert.assertTrue("pool-closer never parked in the in-flight-creation wait", + System.nanoTime() < closerParkedDeadline); + Thread.sleep(1); + } // Let the creation finish: the borrower re-locks, observes the // closed pool and starts the delegate teardown, which parks. @@ -313,6 +333,8 @@ public void closedMidCreationTeardownRunsOutsideThePoolLock() throws Exception { lockFree); borrower.join(TimeUnit.SECONDS.toMillis(10)); + poolCloser.join(TimeUnit.SECONDS.toMillis(10)); + Assert.assertFalse("pool close did not finish", poolCloser.isAlive()); Assert.assertFalse("borrower did not finish", borrower.isAlive()); Assert.assertTrue( "borrow() must surface the closed pool, got: " + borrowOutcome.get(), diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolErrorSafetyTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolErrorSafetyTest.java index 81055bb6..38561c46 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolErrorSafetyTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolErrorSafetyTest.java @@ -32,6 +32,8 @@ import org.junit.Test; import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.lang.reflect.Proxy; import java.nio.file.Paths; import java.util.concurrent.atomic.AtomicBoolean; @@ -83,6 +85,75 @@ public void preWarmClosesBuiltDelegatesWhenBuildThrowsError() throws Exception { }); } + @Test(timeout = 30_000) + public void elasticBorrowClosesDelegateWhenPostFactoryInitializationFails() throws Exception { + TestUtils.assertMemoryLeak(() -> { + AssertionError failure = new AssertionError("injected post-factory failure"); + AtomicInteger closeCalls = new AtomicInteger(); + try (SenderPool pool = new SenderPool( + CFG, 0, 1, 1_000, Long.MAX_VALUE, Long.MAX_VALUE, + slotIndex -> closeCountingSender(closeCalls, null), false, + () -> { + throw failure; + })) { + try { + pool.borrow(); + Assert.fail("borrow must propagate the post-factory failure"); + } catch (AssertionError actual) { + Assert.assertSame("borrow must preserve throwable identity", failure, actual); + } + Assert.assertEquals("failed elastic creation must close its delegate", 1, closeCalls.get()); + } + }); + } + + @Test(timeout = 30_000) + public void preWarmClosesDelegateWhenPostFactoryInitializationFails() throws Exception { + TestUtils.assertMemoryLeak(() -> { + AssertionError failure = new AssertionError("injected post-factory failure"); + AtomicInteger closeCalls = new AtomicInteger(); + try { + new SenderPool( + CFG, 1, 1, 1_000, Long.MAX_VALUE, Long.MAX_VALUE, + slotIndex -> closeCountingSender(closeCalls, null), false, + () -> { + throw failure; + }); + Assert.fail("prewarm must propagate the post-factory failure"); + } catch (AssertionError actual) { + Assert.assertSame("prewarm must preserve throwable identity", failure, actual); + } + Assert.assertEquals("failed prewarm creation must close its delegate", 1, closeCalls.get()); + }); + } + + @Test(timeout = 30_000) + public void recoveryClosesDelegateWhenPostFactoryInitializationFails() throws Exception { + TestUtils.assertMemoryLeak(() -> { + AssertionError failure = new AssertionError("injected post-factory failure"); + AssertionError closeFailure = new AssertionError("injected close failure"); + AtomicInteger closeCalls = new AtomicInteger(); + try (SenderPool pool = new SenderPool( + CFG, 0, 1, 1_000, Long.MAX_VALUE, Long.MAX_VALUE, + slotIndex -> closeCountingSender(closeCalls, closeFailure), false, + () -> { + throw failure; + })) { + Method createRecoverer = SenderPool.class.getDeclaredMethod("createRecoverer", int.class); + createRecoverer.setAccessible(true); + try { + createRecoverer.invoke(pool, -1); + Assert.fail("recovery creation must propagate the post-factory failure"); + } catch (InvocationTargetException e) { + Assert.assertSame("recovery must preserve throwable identity", failure, e.getCause()); + Assert.assertArrayEquals("close failure must remain secondary", + new Throwable[]{closeFailure}, failure.getSuppressed()); + } + Assert.assertEquals("failed recovery creation must close its delegate", 1, closeCalls.get()); + } + }); + } + // Companion to the catch (RuntimeException) -> track-normal-completion fix in // PooledSender.close(). flush() can exit with an Error (AssertionError under // -ea, OutOfMemoryError, ...) as well as a RuntimeException; the wrapper is @@ -232,6 +303,42 @@ public void borrowReleasesSfSlotIndexWhenCreationFails() throws Exception { }); } + private static Sender closeCountingSender(AtomicInteger closeCalls, Throwable closeFailure) { + return (Sender) Proxy.newProxyInstance( + Sender.class.getClassLoader(), + new Class[]{Sender.class}, + (proxy, method, args) -> { + if ("close".equals(method.getName())) { + closeCalls.incrementAndGet(); + if (closeFailure != null) { + throw closeFailure; + } + return null; + } + if ("toString".equals(method.getName())) { + return "CloseCountingSender"; + } + if ("hashCode".equals(method.getName())) { + return System.identityHashCode(proxy); + } + if ("equals".equals(method.getName())) { + return proxy == args[0]; + } + Class rt = method.getReturnType(); + if (rt == boolean.class) return false; + if (rt == byte.class) return (byte) 0; + if (rt == short.class) return (short) 0; + if (rt == int.class) return 0; + if (rt == long.class) return 0L; + if (rt == float.class) return 0f; + if (rt == double.class) return 0d; + if (rt == char.class) return (char) 0; + if (rt == void.class) return null; + if (rt.isInstance(proxy)) return proxy; + return null; + }); + } + private static Sender fakeSender(AtomicBoolean closedFlag) { return (Sender) Proxy.newProxyInstance( Sender.class.getClassLoader(), diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfFuzzTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfFuzzTest.java new file mode 100644 index 00000000..3220d80e --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfFuzzTest.java @@ -0,0 +1,577 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.impl; + +import io.questdb.client.Sender; +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; +import io.questdb.client.impl.PooledSender; +import io.questdb.client.impl.SenderPool; +import io.questdb.client.std.Files; +import io.questdb.client.std.Rnd; +import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.IntFunction; + +/** + * Randomised drain-liveness fuzz for the {@link SenderPool} SF recovery scan, + * the retire/re-probe machinery, and their interleaving with ordinary + * borrow/return traffic. + *

      + * Every iteration builds the same fault family the deterministic tests pin + * one shape of (see {@code SenderPoolSfTest} + * {@code testRecoveryScanStaysAliveWhileRetiredSlotHoldsStrandedData}): + *

        + *
      1. a previous run strands unacked durable data under a random subset of + * the in-range slot dirs (silent server, close without acks);
        2. + *
      2. a new pool over the same {@code sf_dir} faces an ack-ing server, but a + * random subset of its recovery builds are forged into the wedged-close + * shape: {@code close()} returns with the slot flock still held, so the + * scan retires the slot ({@code leakedSlots++}, index stays reserved) + * with its data still on disk;
        3. + *
      3. a random single-threaded schedule of recovery steps, housekeeper + * ticks, borrows (returned or discarded) and late flock releases runs + * the scan, the retire bookkeeping and the borrow cap math against each + * other in randomized order.
        4. + *
      + * After the schedule every wedge is healed (the "worker" exits, the flock + * genuinely drops) and the pool is driven quiescently -- housekeeper tick + + * recovery step, no new load. The oracle is the store-and-forward delivery + * contract at the heart of the drain-liveness bug class: once faults heal and + * the system goes quiet, EVERY durably-accepted row must reach the server IN + * THIS PROCESS through nothing but the pool's ordinary lifecycle -- no + * restart, no lucky borrow required. Concretely: {@code recoveryComplete} + * latches and {@code leakedSlots} returns to zero under quiescent driving, + * and after {@code pool.close()} (which drains live-owned delegates the + * ordinary way -- a slot ADOPTED by a borrow mid-schedule is its owner's to + * deliver, not the scan's) no slot dir may still hold a segment file. Pre-fix, + * any iteration that retires a stranded slot mid-scan latches + * {@code recoveryComplete} past the stranded data (the retired index was + * skipped as "reserved" without counting a deferral); nothing ever drains that + * dir -- close() never owned it -- and the post-close audit fails. + *

      + * The schedule is single-threaded on purpose: with one driver the whole + * iteration is a pure function of the seed, so any failure replays exactly + * with {@code TestUtils.generateRandom(null, s0, s1)} (seeds are printed by + * the harness and repeated in the failure message). Two iterations are pinned + * so the suite cannot go green on an unlucky seed while either bug shape is + * present; the rest randomize freely: + *

        + *
      • Iteration 0 -- scan abandonment. Every slot stranded, slot 0 + * wedged, and a deterministic two-step prologue drives the scan into + * the wedged-retire shape BEFORE any random traffic (a random schedule + * could otherwise draw a borrow first, adopt slot 0 and never + * materialize the wedge). Pre-fix, the scan skipped the retired index + * without a deferral and latched {@code recoveryComplete} past the + * stranded data.
        • + *
      • Iteration 1 -- post-latch runtime retire (the residual). The + * scan completes LEGITIMATELY over clean dirs first; only then does a + * borrowed sender -- built against the silent sink so its rows stay + * durably unacked -- take the wedged-close discard path + * ({@code reclaimSlot} retires it). Pre-fix, the late flock release + * restored capacity but nothing re-armed the latched scan, so the + * data waited for a restart or a lucky borrow of that index.
        • + *
      + * Random iterations (2+) additionally draw the runtime-wedge op freely, so + * runtime retires also interleave with a still-running scan. + */ +public class SenderPoolSfFuzzTest { + + private static final int ITERATIONS = 5; + private static final long CONVERGE_BUDGET_MILLIS = 20_000; + + @Test + public void testRetiredSlotDrainLivenessFuzz() throws Exception { + long s0 = System.nanoTime(); + long s1 = System.currentTimeMillis(); + Rnd rnd = TestUtils.generateRandom(null, s0, s1); + try { + for (int iter = 0; iter < ITERATIONS; iter++) { + runOneIteration(rnd, iter); + } + } catch (Throwable t) { + throw new AssertionError("fuzz failure with seeds=" + s0 + "L," + s1 + "L", t); + } + } + + private void runOneIteration(Rnd rnd, int iter) throws Exception { + String sfDir = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-sf-pool-fuzz-" + System.nanoTime() + "-" + iter).toString(); + try { + TestUtils.assertMemoryLeak(() -> { + int maxSize = 1 + rnd.nextInt(3); // 1..3 + // Iteration 0 pins the guaranteed-red scan-abandonment shape: + // every slot stranded so the wedged recovery build below has + // data behind it. Iteration 1 pins the residual and needs a + // CLEAN start so the scan latches legitimately first. Later + // iterations may strand any subset (0 = a plain clean-scan + // iteration, still a valid latch/borrow interplay). + int stranded = iter == 0 ? maxSize : iter == 1 ? 0 : rnd.nextInt(maxSize + 1); + + // Phase 1: strand unacked data under default-0..(stranded-1). + if (stranded > 0) { + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int silentPort = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String seedCfg = "ws::addr=localhost:" + silentPort + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=0;"; + try (SenderPool seed = new SenderPool(seedCfg, stranded, stranded, + 5_000, Long.MAX_VALUE, Long.MAX_VALUE)) { + PooledSender[] s = new PooledSender[stranded]; + for (int i = 0; i < stranded; i++) { + s[i] = seed.borrow(); + } + for (int i = 0; i < stranded; i++) { + int nRows = 1 + rnd.nextInt(3); + for (int r = 0; r < nRows; r++) { + s[i].table("fuzz").longColumn("v", r).atNow(); + s[i].flush(); + } + } + for (int i = stranded - 1; i >= 0; i--) { + s[i].close(); + } + } + } + for (int i = 0; i < stranded; i++) { + Assert.assertTrue("iter " + iter + ": default-" + i + " must hold unacked data", + hasSegmentFile(sfDir + "/default-" + i)); + } + } + + // Phase 2: ack-ing server; wedge a random subset of RECOVERY + // builds (iteration 0 always wedges slot 0). The forge is + // scoped by inRecoveryStep so a borrow can never receive a + // forged delegate -- exactly like a real wedged close(), which + // only the recovery/reclaim paths ever observe. + boolean[] wedge = new boolean[maxSize]; + for (int i = 0; i < maxSize; i++) { + wedge[i] = i < stranded && (iter == 0 ? i == 0 : rnd.nextBoolean()); + } + // Iteration 1's fault is a RUNTIME wedge, not an in-scan one. + boolean forceRuntimeWedge = iter == 1; + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer ack = new TestWebSocketServer(handler); + TestWebSocketServer wedgeSink = new TestWebSocketServer(new SilentHandler())) { + int ackPort = ack.getPort(); + ack.start(); + Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS)); + wedgeSink.start(); + Assert.assertTrue(wedgeSink.awaitStart(5, TimeUnit.SECONDS)); + // close_flush_timeout bounds the ordinary-lifecycle drain + // pool.close() performs on live-owned delegates; against + // the ack-ing server it completes in milliseconds. + String cfg = "ws::addr=localhost:" + ackPort + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=10000;"; + // The forged recoverers model a WEDGED worker faithfully: + // alive (setClosedForTesting only forges the flag; the I/O + // loop keeps pumping) but delivering nothing. They are + // therefore built against a SILENT sink -- an alive worker + // streaming into a black hole -- so the adopted chain is + // never acked and never trimmed; and with close-flush OFF, + // so the heal close() drops the flock WITHOUT draining the + // chain on the data's behalf. Either kindness (an ack-ing + // endpoint or a draining close) would hand the oracle a + // delivery the production wedge semantics do not provide, + // masking scan abandonment. + String cfgWedge = "ws::addr=localhost:" + wedgeSink.getPort() + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=0;"; + + boolean[] inRecoveryStep = new boolean[1]; + // Latched by the end-of-schedule heal: from that point the + // fault injector is OFF. Without this, a wedge[idx] flag + // whose first recovery build only happens DURING the + // quiescent convergence would forge a brand-new wedge + // after "every fault healed" -- a fault the schedule can + // never heal -- and the scan would (correctly!) refuse to + // complete, failing the audit for the wrong reason. + boolean[] faultsHealed = new boolean[1]; + boolean[] wedgeNextBorrow = new boolean[1]; + Sender[] lastWedgeBuild = new Sender[1]; + Sender[] forged = new Sender[maxSize]; + List unhealed = new ArrayList<>(); + IntFunction factory = idx -> { + boolean forgeNow = inRecoveryStep[0] && !faultsHealed[0] && idx < maxSize + && wedge[idx] && forged[idx] == null; + // A runtime-wedge borrow is also built against the + // silent sink (rows must stay durably unacked) but is + // NOT forged closed at build: the op writes through it + // first, then forges and discards. + boolean wedgeBorrow = !inRecoveryStep[0] && wedgeNextBorrow[0]; + Sender real = Sender.builder(forgeNow || wedgeBorrow ? cfgWedge : cfg) + .senderId("default-" + idx).build(); + if (forgeNow) { + try { + ((QwpWebSocketSender) real).setClosedForTesting(true); + } catch (Exception e) { + throw new RuntimeException(e); + } + forged[idx] = real; + unhealed.add(real); + } else if (wedgeBorrow) { + lastWedgeBuild[0] = real; + } + return real; + }; + + try (SenderPool pool = new SenderPool(cfg, 0, maxSize, 100, + Long.MAX_VALUE, Long.MAX_VALUE, factory, true)) { + try { + if (iter == 0) { + // Deterministic prologue for the pinned + // iteration: step the scan until it builds the + // forged recoverer for slot 0 and RETIRES it + // with its data still on disk (step 1), then + // walk once more so the unfixed skip-without- + // deferral path is on the table (step 2). + // Without this a random schedule could borrow + // first, adopt slot 0 as a LIVE slot, and the + // bug shape would never materialize. + for (int k = 0; k < 2; k++) { + inRecoveryStep[0] = true; + try { + pool.runStartupRecoveryStepForTesting(); + } finally { + inRecoveryStep[0] = false; + } + } + Assert.assertEquals( + "iter 0 prologue must retire the wedged slot with " + + "its stranded data still on disk", + 1, pool.leakedSlotCount()); + Assert.assertTrue("iter 0 prologue must leave slot 0 stranded", + hasSegmentFile(sfDir + "/default-0")); + } + if (forceRuntimeWedge) { + // Pinned residual prologue: latch the scan + // legitimately FIRST, then retire a live + // borrow whose rows are durably unacked. No + // heal until the end of the schedule, so the + // post-heal re-arm (or its absence) is the + // only thing that decides the audit. + long latchDeadline = System.currentTimeMillis() + 10_000; + while (!pool.isRecoveryCompleteForTesting() + && System.currentTimeMillis() < latchDeadline) { + inRecoveryStep[0] = true; + try { + pool.runStartupRecoveryStepForTesting(); + } finally { + inRecoveryStep[0] = false; + } + } + Assert.assertTrue("iter 1 prologue: scan must latch over clean dirs", + pool.isRecoveryCompleteForTesting()); + wedgeNextBorrow[0] = true; + PooledSender ps = pool.borrow(); + wedgeNextBorrow[0] = false; + Assert.assertSame("iter 1 prologue: borrow must create the wedge target", + lastWedgeBuild[0], ps.getDelegateForTesting()); + lastWedgeBuild[0] = null; + for (int r = 0; r < 2; r++) { + ps.table("fuzz").longColumn("resid", r).atNow(); + ps.flush(); + } + Sender delegate = ps.getDelegateForTesting(); + ((QwpWebSocketSender) delegate).setClosedForTesting(true); + pool.discardBrokenForTesting(ps); + unhealed.add(delegate); + Assert.assertEquals("iter 1 prologue must retire the runtime wedge", + 1, pool.leakedSlotCount()); + Assert.assertTrue("iter 1 prologue must leave its slot stranded", + hasSegmentFile(sfDir + "/default-0")); + } + // The randomized schedule. Single-threaded, so the + // iteration replays exactly from the seed. + int ops = 8 + rnd.nextInt(12); + for (int op = 0; op < ops; op++) { + switch (rnd.nextInt(iter >= 2 ? 6 : 5)) { + case 0: + case 1: // bias toward driving the scan + inRecoveryStep[0] = true; + try { + pool.runStartupRecoveryStepForTesting(); + } finally { + inRecoveryStep[0] = false; + } + break; + case 2: // housekeeper tick: reap + re-probe retired + pool.reapIdle(); + break; + case 3: { // ordinary traffic, returned to the pool + // The lease ALWAYS comes home: a leaked + // lease would (correctly) block close()'s + // ordinary-lifecycle drain and turn a + // write hiccup into a bogus audit red. + PooledSender ps = null; + try { + ps = pool.borrow(); + ps.table("fuzz").longColumn("live", op).atNow(); + ps.flush(); + } catch (LineSenderException e) { + // Legal under wedged-retire capacity + // starvation: borrow timed out. + } finally { + if (ps != null) { + ps.close(); + } + } + break; + } + case 4: // late flock release for one wedged slot + // Pinned iteration 0 keeps its wedge held + // through the WHOLE schedule: a mid-schedule + // heal would let a later borrow adopt the + // freed index and deliver the data through + // its own lifecycle -- legal, but it would + // mask the scan-abandonment bug the pinned + // iteration exists to catch. With the wedge + // held, no borrow can ever own slot 0, so + // post-heal the SCAN is the only possible + // deliverer (fixed) versus nobody (bug). + if (iter >= 2 && !unhealed.isEmpty()) { + Sender s = unhealed.remove(rnd.nextInt(unhealed.size())); + ((QwpWebSocketSender) s).setClosedForTesting(false); + s.close(); + } + break; + case 5: { // runtime wedge: retire a live borrow undelivered + wedgeNextBorrow[0] = true; + PooledSender ps = null; + try { + ps = pool.borrow(); + } catch (LineSenderException e) { + // Capacity-starved: legal, move on. + break; + } finally { + // Disarm regardless: a reused borrow + // must not leave the flag armed for + // an unrelated later creation. + wedgeNextBorrow[0] = false; + } + Sender delegate = ps.getDelegateForTesting(); + if (delegate == lastWedgeBuild[0]) { + lastWedgeBuild[0] = null; + ps.table("fuzz").longColumn("wedged", op).atNow(); + ps.flush(); + ((QwpWebSocketSender) delegate).setClosedForTesting(true); + pool.discardBrokenForTesting(ps); + unhealed.add(delegate); + } else { + // Borrow reused an idle (ack-server) + // sender; nothing to wedge. + lastWedgeBuild[0] = null; + ps.close(); + } + break; + } + } + } + + // Heal every remaining wedge: the "workers" exit and + // the flocks genuinely drop. Also switch the fault + // injector off -- no new wedge may be born after + // this point (see faultsHealed). + faultsHealed[0] = true; + while (!unhealed.isEmpty()) { + Sender s = unhealed.remove(unhealed.size() - 1); + ((QwpWebSocketSender) s).setClosedForTesting(false); + s.close(); + } + + // Quiescent convergence: housekeeper tick + recovery + // step only -- NO new borrows. The delivery contract + // must not depend on future load. + long deadline = System.currentTimeMillis() + CONVERGE_BUDGET_MILLIS; + while (!pool.isRecoveryCompleteForTesting() + && System.currentTimeMillis() < deadline) { + pool.reapIdle(); + boolean more; + inRecoveryStep[0] = true; + try { + more = pool.runStartupRecoveryStepForTesting(); + } finally { + inRecoveryStep[0] = false; + } + if (!more && !pool.isRecoveryCompleteForTesting()) { + Thread.sleep(5); + } + } + + // In-pool audit: the scan itself must have + // converged -- a latched-early scan is exactly the + // drain-liveness bug shape. + Assert.assertTrue( + "iter " + iter + ": recovery scan must complete once every " + + "flock is healed -- a latched-early scan strands " + + "retired slots' data until restart", + pool.isRecoveryCompleteForTesting()); + Assert.assertEquals( + "iter " + iter + ": healed flocks must restore all retired capacity", + 0, pool.leakedSlotCount()); + } finally { + // A failed assertion must not leak forged natives: + // un-forge and close for real (idempotent). + for (Sender s : unhealed) { + try { + ((QwpWebSocketSender) s).setClosedForTesting(false); + s.close(); + } catch (Throwable ignore) { + // best-effort teardown + } + } + unhealed.clear(); + } + } + + // Post-close audit: the pool exited through its ordinary + // lifecycle (close() drains live-owned delegates; the scan + // drained everything nobody owned). NOTHING may remain + // durably-accepted-but-undelivered in this process -- no + // restart, no lucky borrow. Pre-fix, a slot retired + // mid-scan fails exactly here: the latched scan abandoned + // it and close() never owned it. + for (int i = 0; i < maxSize; i++) { + Assert.assertTrue( + "iter " + iter + ": default-" + i + " must not hold " + + "undelivered durable data after quiescent convergence " + + "and an ordinary-lifecycle close", + awaitNoSegmentFile(sfDir + "/default-" + i, 15_000)); + } + } + }); + } finally { + rmDir(sfDir); + } + } + + // ------------------------------------------------------------------ + // Local copies of the SenderPoolSfTest harness helpers (private there). + // ------------------------------------------------------------------ + + private static boolean hasSegmentFile(String slotPath) { + if (!Files.exists(slotPath)) { + return false; + } + long find = Files.findFirst(slotPath); + if (find <= 0) { + return false; + } + try { + int rc = 1; + while (rc > 0) { + String name = Files.utf8ToString(Files.findName(find)); + rc = Files.findNext(find); + if (name != null && name.endsWith(".sfa")) { + return true; + } + } + } finally { + Files.findClose(find); + } + return false; + } + + private static boolean awaitNoSegmentFile(String slotPath, long timeoutMillis) + throws InterruptedException { + long deadline = System.currentTimeMillis() + timeoutMillis; + while (System.currentTimeMillis() < deadline) { + if (!hasSegmentFile(slotPath)) { + return true; + } + Thread.sleep(10); + } + return !hasSegmentFile(slotPath); + } + + private static void rmDir(String dir) { + if (dir == null || !Files.exists(dir)) { + return; + } + long find = Files.findFirst(dir); + if (find > 0) { + try { + int rc = 1; + while (rc > 0) { + String name = Files.utf8ToString(Files.findName(find)); + if (name != null && !".".equals(name) && !"..".equals(name)) { + String child = dir + "/" + name; + if (!Files.remove(child)) { + rmDir(child); + } + } + rc = Files.findNext(find); + } + } finally { + Files.findClose(find); + } + } + Files.remove(dir); + } + + private static final class CountingAckHandler implements TestWebSocketServer.WebSocketServerHandler { + private final Map seqByClient = + new ConcurrentHashMap<>(); + + @Override + public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + AtomicLong seq = seqByClient.computeIfAbsent(client, c -> new AtomicLong(0)); + try { + client.sendBinary(buildAck(seq.getAndIncrement())); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + static byte[] buildAck(long seq) { + byte[] buf = new byte[1 + 8 + 2]; + ByteBuffer bb = ByteBuffer.wrap(buf).order(ByteOrder.LITTLE_ENDIAN); + bb.put((byte) 0x00); // STATUS_OK + bb.putLong(seq); + bb.putShort((short) 0); + return buf; + } + } + + private static final class SilentHandler implements TestWebSocketServer.WebSocketServerHandler { + @Override + public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + // No ack -- frames stay unacked on disk so recovery sees candidates. + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java index cfef2feb..e7b48aa3 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java @@ -30,7 +30,14 @@ import ch.qos.logback.core.read.ListAppender; import io.questdb.client.Sender; import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; import io.questdb.client.cutlass.qwp.client.sf.cursor.OrphanScanner; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SlotLock; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SlotLockContentionException; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; import io.questdb.client.impl.PooledSender; import io.questdb.client.impl.SenderPool; import io.questdb.client.std.Files; @@ -43,16 +50,19 @@ import org.slf4j.LoggerFactory; import java.io.IOException; -import java.lang.reflect.Field; -import java.lang.reflect.Method; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Proxy; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.file.Paths; +import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; @@ -112,8 +122,8 @@ public void testTwoConcurrentSfSendersGetDistinctSlots() throws Exception { // call, so comparing the wrappers is vacuously true. // Distinctness of the two borrowed senders lives in the // underlying slots (mirrors SenderPoolTest.slotOf usage). - Assert.assertNotSame("two borrows must hold distinct slots", - slotOf(a), slotOf(b)); + Assert.assertFalse("two borrows must hold distinct slots", + a.hasSameSlotForTesting(b)); Assert.assertTrue("slot default-0 must exist", Files.exists(slot("default-0"))); Assert.assertTrue("slot default-1 must exist", Files.exists(slot("default-1"))); Assert.assertEquals("exactly two slot dirs", 2, countSlotDirs()); @@ -213,8 +223,8 @@ public void testReturnedSenderReusesSameSlot() throws Exception { try { // borrow() now returns a fresh wrapper each time; the // recycled thing is the underlying slot. - Assert.assertSame("returned slot must be recycled", - getField(first, "slot"), getField(second, "slot")); + Assert.assertTrue("returned slot must be recycled", + first.hasSameSlotForTesting(second)); Assert.assertEquals("no new slot dir on recycle", 1, countSlotDirs()); Assert.assertTrue(Files.exists(slot("default-0"))); } finally { @@ -482,17 +492,17 @@ public void testSlotLeakedWhenDelegateCloseDoesNotReleaseFlock() throws Exceptio // below is a no-op and leaves the forged flag in place. Sender delegate = getDelegate(a); delegate.close(); - setBooleanField(delegate, "slotLockReleased", false); + ((QwpWebSocketSender) delegate).setSlotLockReleasedForTesting(false); // Route the wrapper through the pool's broken-eviction path. - invokeDiscardBroken(pool, a); + pool.discardBrokenForTesting(a); // The leaked index must NOT be returned to the free set, // and capacity must be accounted as permanently consumed. Assert.assertEquals("one slot must be retired as leaked", - 1, getIntField(pool, "leakedSlots")); - boolean[] slotInUse = (boolean[]) getField(pool, "slotInUse"); - Assert.assertTrue("leaked slot index 0 must stay reserved", slotInUse[0]); + 1, pool.leakedSlotCount()); + Assert.assertTrue("leaked slot index 0 must stay reserved", + pool.isSlotInUseForTesting(0)); // The next borrow must take a fresh index -- never reuse the // still-locked default-0 dir. @@ -555,12 +565,12 @@ public void testLeakedSlotIsObservable() throws Exception { // discardBroken's re-close leaves the forged flag set. Sender delegate = getDelegate(a); delegate.close(); - setBooleanField(delegate, "slotLockReleased", false); - invokeDiscardBroken(pool, a); + ((QwpWebSocketSender) delegate).setSlotLockReleasedForTesting(false); + pool.discardBrokenForTesting(a); // Sanity: the slot really was retired as leaked. Assert.assertEquals("precondition: one slot must leak", - 1, getIntField(pool, "leakedSlots")); + 1, pool.leakedSlotCount()); // The leak must be observable via public API (metric). Assert.assertEquals("leaked slot must be observable via leakedSlotCount()", 1, pool.leakedSlotCount()); @@ -619,7 +629,7 @@ public void testSlotLeakedWhenDelegateCloseDoesNotReleaseFlockDuringReap() throw pool.giveBack(a); Sender delegate = getDelegate(a); delegate.close(); - setBooleanField(delegate, "slotLockReleased", false); + ((QwpWebSocketSender) delegate).setSlotLockReleasedForTesting(false); // Drive the sweep: the idle timeout has elapsed. Thread.sleep(10); @@ -627,9 +637,9 @@ public void testSlotLeakedWhenDelegateCloseDoesNotReleaseFlockDuringReap() throw // The reap leaked branch must have fired. Assert.assertEquals("reapIdle must retire the still-locked slot as leaked", - 1, getIntField(pool, "leakedSlots")); - boolean[] slotInUse = (boolean[]) getField(pool, "slotInUse"); - Assert.assertTrue("leaked slot index 0 must stay reserved", slotInUse[0]); + 1, pool.leakedSlotCount()); + Assert.assertTrue("leaked slot index 0 must stay reserved", + pool.isSlotInUseForTesting(0)); Assert.assertEquals("leaked slot must be observable via leakedSlotCount()", 1, pool.leakedSlotCount()); @@ -658,99 +668,196 @@ public void testSlotLeakedWhenDelegateCloseDoesNotReleaseFlockDuringReap() throw }); } - // ---------------------------------------------------------------------- - // Recovery: stable slot ids let a re-created pool re-adopt unacked data. - // ---------------------------------------------------------------------- - @Test - public void testRecoveryReplayThroughPooledSlot() throws Exception { - // Phase 1: write rows to a slot against a silent server (no acks), so - // the data persists unacked on disk under default-0. Close. - // Phase 2: a new pool against an ack-ing server re-adopts default-0 - // (stable index) and replays the unacked frames. Stable, deterministic - // slot ids are exactly what make this recovery possible. + public void testRetiredSlotRecoveredByHousekeeperAfterLateFlockRelease() throws Exception { + // Recovery twin of testSlotLeakedWhenDelegateCloseDoesNotReleaseFlock: + // a slot retired because close() returned with the flock still held is + // NOT lost until process exit. Engine cleanup may complete later on a + // worker/I/O-thread exit path (isSlotLockReleased() re-probes the + // retained engine), and the housekeeper's reapIdle() tick must then + // return the index to the free set: leakedSlots back down, slotInUse + // cleared, full capacity restored. TestUtils.assertMemoryLeak(() -> { - // Phase 1 -- silent server. - try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { - int silentPort = silent.getPort(); - silent.start(); - Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); - String cfg1 = "ws::addr=localhost:" + silentPort + ";sf_dir=" + sfDir - + ";close_flush_timeout_millis=500;"; - try (SenderPool pool = new SenderPool(cfg1, 1, 1, 1_000, Long.MAX_VALUE, Long.MAX_VALUE)) { - PooledSender s = pool.borrow(); - for (int i = 0; i < 3; i++) { - s.table("recover").longColumn("v", i).atNow(); - s.flush(); + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try (SenderPool pool = new SenderPool(config, 1, 2, 500, Long.MAX_VALUE, Long.MAX_VALUE)) { + PooledSender a = pool.borrow(); + Assert.assertTrue(Files.exists(slot("default-0"))); + + // Forge the retire: real teardown first (no native leaks), + // then clear slotLockReleased so discardBroken retires the + // slot as leaked. + Sender delegate = getDelegate(a); + delegate.close(); + ((QwpWebSocketSender) delegate).setSlotLockReleasedForTesting(false); + pool.discardBrokenForTesting(a); + Assert.assertEquals("precondition: one slot must be retired", + 1, pool.leakedSlotCount()); + + // Forge the late release: the deferred cleanup finished and + // the delegate now reports the flock dropped (in production + // this flip comes from isSlotLockReleased() re-probing the + // retained engine after the manager worker exited). + ((QwpWebSocketSender) delegate).setSlotLockReleasedForTesting(true); + + // The housekeeper tick is the recovery driver. + pool.reapIdle(); + + Assert.assertEquals("recovered slot must leave the leaked count", + 0, pool.leakedSlotCount()); + Assert.assertFalse("recovered slot index 0 must return to the free set", + pool.isSlotInUseForTesting(0)); + + // Full capacity restored: with maxSize=2, two concurrent + // borrows must succeed again (index 0 is reusable — its + // flock is genuinely free). + PooledSender b = pool.borrow(); + PooledSender c = pool.borrow(); + try { + Assert.assertEquals(2, countSlotDirs()); + } finally { + c.close(); + b.close(); } - s.close(); } } - // Data must be on disk, unacked, under default-0. - Assert.assertTrue("unacked data must persist on disk", hasSegmentFile(slot("default-0"))); + }); + } - // Phase 2 -- ack-ing server, brand-new pool, same sf_dir. + @Test + public void testCapacityStarvedBorrowRecoversRetiredSlot() throws Exception { + // Borrow-path twin of the housekeeper recovery test: a borrow that + // would otherwise park on the cap check must re-probe retired slots + // before waiting, so a late flock release converts a guaranteed + // borrow timeout into an immediate creation on the recovered index — + // no housekeeper tick required. + TestUtils.assertMemoryLeak(() -> { CountingAckHandler handler = new CountingAckHandler(); - try (TestWebSocketServer ack = new TestWebSocketServer(handler)) { - int ackPort = ack.getPort(); - ack.start(); - Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS)); - String cfg2 = "ws::addr=localhost:" + ackPort + ";sf_dir=" + sfDir + ";"; - try (SenderPool pool = new SenderPool(cfg2, 1, 1, 5_000, Long.MAX_VALUE, Long.MAX_VALUE)) { - PooledSender s = pool.borrow(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try (SenderPool pool = new SenderPool(config, 1, 2, 500, Long.MAX_VALUE, Long.MAX_VALUE)) { + PooledSender a = pool.borrow(); + Assert.assertTrue(Files.exists(slot("default-0"))); + + Sender delegate = getDelegate(a); + delegate.close(); + ((QwpWebSocketSender) delegate).setSlotLockReleasedForTesting(false); + pool.discardBrokenForTesting(a); + Assert.assertEquals("precondition: one slot must be retired", + 1, pool.leakedSlotCount()); + + // One live borrow + one retired slot = cap reached. + PooledSender b = pool.borrow(); + Assert.assertTrue(Files.exists(slot("default-1"))); try { - // Drain replays the recovered, unacked frames. - s.drain(5_000); - Assert.assertTrue("recovered frames must be replayed to the new server", - awaitAtLeast(handler.frames, 1, 5_000)); + // Late release lands while the pool is capacity-starved. + ((QwpWebSocketSender) delegate).setSlotLockReleasedForTesting(true); + + // The next borrow hits the cap check, re-probes, frees + // index 0, and must create on it instead of timing out. + PooledSender c = pool.borrow(); + try { + Assert.assertEquals("borrow must recover the retired slot's capacity", + 0, pool.leakedSlotCount()); + Assert.assertEquals(2, countSlotDirs()); + } finally { + c.close(); + } } finally { - s.close(); + b.close(); } } } }); } - @Test - public void testRecoveryDelegateForcesOffInitialConnectMode() throws Exception { - // M1 regression: a startup-recovery delegate runs on the PoolHousekeeper - // thread, so its build() must NOT inherit the user's SYNC initial-connect - // mode (auto-enabled by any reconnect_* knob). SYNC would retry the - // connect for the whole reconnect budget inside build() -- far past - // PoolHousekeeper.STOP_TIMEOUT_MILLIS -- so a close() landing during that - // build would make the housekeeper join time out and leave the recoverer - // holding the slot flock after close() returned. The recovery factory - // forces initial_connect_mode=OFF (at most one connect attempt); the - // normal factory must still honour the user's promoted SYNC mode. + @Test(timeout = 30_000) + public void testDeferredFlockReleaseWakesParkedLongTimeoutBorrower() throws Exception { TestUtils.assertMemoryLeak(() -> { CountingAckHandler handler = new CountingAckHandler(); - try (TestWebSocketServer ack = new TestWebSocketServer(handler)) { - int ackPort = ack.getPort(); - ack.start(); - Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS)); - // reconnect_max_duration_millis set, initial_connect_mode unset - // -> the builder promotes to SYNC for ordinary senders. - String cfg = "ws::addr=localhost:" + ackPort + ";sf_dir=" + sfDir - + ";reconnect_max_duration_millis=30000;"; - // min=0 (no prewarm connect), no stranded data (recovery no-op). - try (SenderPool pool = new SenderPool(cfg, 0, 2, 1_000, Long.MAX_VALUE, Long.MAX_VALUE)) { - // Normal managed-slot delegate: inherits the promoted SYNC. - Sender normal = invokeBuildSlotDelegate(pool, "defaultSender", 0); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + String config = "ws::addr=localhost:" + server.getPort() + ";sf_dir=" + sfDir + ";"; + try (SenderPool pool = new SenderPool( + config, 1, 1, 60_000, Long.MAX_VALUE, Long.MAX_VALUE)) { + PooledSender lease = pool.borrow(); + Sender delegate = getDelegate(lease); + CursorSendEngine engine = ((QwpWebSocketSender) delegate).getCursorEngineForTesting(); + SlotLock slotLock = engine.getSlotLockForTesting(); + CountDownLatch borrowerAcquired = new CountDownLatch(1); + CountDownLatch borrowerParked = new CountDownLatch(1); + CountDownLatch releaseBorrower = new CountDownLatch(1); + AtomicReference borrowerFailure = new AtomicReference<>(); + AtomicReference recovered = new AtomicReference<>(); + Thread borrower = new Thread(() -> { + try { + recovered.set(pool.borrow()); + borrowerAcquired.countDown(); + if (!releaseBorrower.await(10, TimeUnit.SECONDS)) { + throw new AssertionError("timed out waiting to release borrower"); + } + } catch (Throwable t) { + borrowerFailure.compareAndSet(null, t); + } finally { + PooledSender sender = recovered.get(); + if (sender != null) { + sender.close(); + } + } + }, "sender-pool-deferred-release-waiter"); + + SlotLock.ReleaseFailureForTesting releaseFailure = + slotLock.injectReleaseFailureForTesting(); try { - Assert.assertEquals( - "ordinary pooled sender must honour the user's promoted SYNC mode", - Sender.InitialConnectMode.SYNC, readInitialConnectMode(normal)); + pool.discardBrokenForTesting(lease); + Assert.assertEquals("failed release must retire the only slot", + 1, pool.leakedSlotCount()); + Assert.assertFalse(engine.isCloseCompleted()); + + pool.setBeforeBorrowWaitHook(borrowerParked::countDown); + borrower.start(); + Assert.assertTrue("borrower must reach the condition wait with a long timeout", + borrowerParked.await(5, TimeUnit.SECONDS)); + // The hook runs under the pool lock immediately before awaitNanos. + // Acquiring that lock here proves awaitNanos atomically enqueued the + // borrower and released the lock before restoration can start. This + // read-only operation neither changes pool state nor signals a waiter. + pool.availableSize(); + + // This restored fd is the only source of progress: the retry driver + // confirms the release. There is no housekeeper or pool mutation. + releaseFailure.close(); + Assert.assertTrue("deferred flock release must wake the parked borrower", + borrowerAcquired.await(5, TimeUnit.SECONDS)); + Assert.assertNull("borrower must not fail", borrowerFailure.get()); + Assert.assertEquals("release wakeup must recover retired capacity", + 0, pool.leakedSlotCount()); } finally { - normal.close(); + pool.setBeforeBorrowWaitHook(null); + releaseBorrower.countDown(); + if (!engine.isCloseCompleted()) { + releaseFailure.close(); + } + borrower.join(TimeUnit.SECONDS.toMillis(1)); + if (borrower.isAlive()) { + borrower.interrupt(); + borrower.join(TimeUnit.SECONDS.toMillis(5)); + } + Assert.assertFalse("borrower thread must finish", borrower.isAlive()); } - // Recovery delegate on a different slot: forced OFF. - Sender recoverer = invokeBuildSlotDelegate(pool, "defaultRecoverySender", 1); - try { - Assert.assertEquals( - "recovery delegate must force OFF so build() makes at most one connect attempt", - Sender.InitialConnectMode.OFF, readInitialConnectMode(recoverer)); - } finally { - recoverer.close(); + if (borrowerFailure.get() != null) { + throw new AssertionError("borrower failed", borrowerFailure.get()); } } } @@ -758,108 +865,1498 @@ public void testRecoveryDelegateForcesOffInitialConnectMode() throws Exception { } @Test - public void testStartupRecoveryRetiresSlotWhenRecovererCloseLeavesFlockHeld() throws Exception { - // C1 regression: the startup recovery loop MUST mirror discardBroken / - // reapIdle. When a recoverer's delegate close() returns with the SF - // flock still held (the I/O thread refused to stop), the recovered slot - // index must be retired permanently (leakedSlots++, slotInUse stays - // set) -- NOT freed. Freeing it would let a later borrow re-pick the - // still-locked dir and resurrect "sf slot already in use", the exact - // failure class this PR exists to kill. Pre-fix the recovery finally - // set slotInUse[i]=false unconditionally; this test is RED until it - // consults flockReleased() like the other two close-and-reclaim paths. + public void testDirectRetiredSlotCallbacksHaveLinearProbeCount() throws Exception { TestUtils.assertMemoryLeak(() -> { - // Phase 1: leave unacked data on disk under default-0 so startup - // recovery treats it as a candidate orphan and builds a recoverer. - try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { - int silentPort = silent.getPort(); - silent.start(); - Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); - String cfg1 = "ws::addr=localhost:" + silentPort + ";sf_dir=" + sfDir - + ";close_flush_timeout_millis=500;"; - try (SenderPool pool = new SenderPool(cfg1, 1, 1, 1_000, Long.MAX_VALUE, Long.MAX_VALUE)) { - PooledSender s = pool.borrow(); + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + final int slotCount = 32; + String config = "ws::addr=localhost:" + server.getPort() + ";sf_dir=" + sfDir + ";"; + try (SenderPool pool = new SenderPool( + config, 0, slotCount, 5_000, Long.MAX_VALUE, Long.MAX_VALUE)) { + PooledSender[] leases = new PooledSender[slotCount]; + Sender[] delegates = new Sender[slotCount]; + Runnable[] callbacks = new Runnable[slotCount]; + for (int i = 0; i < slotCount; i++) { + leases[i] = pool.borrow(); + delegates[i] = getDelegate(leases[i]); + callbacks[i] = ((QwpWebSocketSender) delegates[i]).getSlotLockReleaseListenerForTesting(); + Assert.assertNotNull(callbacks[i]); + } + for (int i = 0; i < slotCount; i++) { + delegates[i].close(); + ((QwpWebSocketSender) delegates[i]).setSlotLockReleasedForTesting(false); + pool.discardBrokenForTesting(leases[i]); + } + Assert.assertEquals(slotCount, pool.leakedSlotCount()); + pool.setRetiredSlotProbeCountForTesting(0); + + int[] geometricCheckpoints = {4, 8, 16, 32}; + int checkpoint = 0; + for (int i = 0; i < slotCount; i++) { + ((QwpWebSocketSender) delegates[i]).setSlotLockReleasedForTesting(true); + callbacks[i].run(); + if (i + 1 == geometricCheckpoints[checkpoint]) { + Assert.assertEquals("direct release probes must grow linearly", + i + 1, pool.getRetiredSlotProbeCountForTesting()); + checkpoint++; + } + } + Assert.assertEquals(0, pool.leakedSlotCount()); + Assert.assertTrue(pool.getRetiredSlotCountForTesting() == 0); + } + } + }); + } + + @Test + public void testDirectRetiredSlotCallbackFallbackAndIdempotence() throws Exception { + TestUtils.assertMemoryLeak(() -> { + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + String config = "ws::addr=localhost:" + server.getPort() + ";sf_dir=" + sfDir + ";"; + try (SenderPool pool = new SenderPool(config, 0, 3, 5_000, Long.MAX_VALUE, Long.MAX_VALUE)) { + PooledSender[] leases = new PooledSender[3]; + Sender[] delegates = new Sender[3]; + Runnable[] callbacks = new Runnable[3]; for (int i = 0; i < 3; i++) { - s.table("recover").longColumn("v", i).atNow(); - s.flush(); + leases[i] = pool.borrow(); + delegates[i] = getDelegate(leases[i]); + callbacks[i] = ((QwpWebSocketSender) delegates[i]).getSlotLockReleaseListenerForTesting(); + delegates[i].close(); + ((QwpWebSocketSender) delegates[i]).setSlotLockReleasedForTesting(false); + pool.discardBrokenForTesting(leases[i]); } - s.close(); + Assert.assertEquals(3, pool.leakedSlotCount()); + pool.setRetiredSlotProbeCountForTesting(0); + + // Simulate callback registration becoming unavailable. The + // periodic housekeeper scan must remain a complete fallback. + ((QwpWebSocketSender) delegates[0]).setSlotLockReleaseListener(null); + ((QwpWebSocketSender) delegates[0]).setSlotLockReleasedForTesting(true); + pool.reapIdle(); + Assert.assertEquals(2, pool.leakedSlotCount()); + + // A premature callback must not remove an unreleased slot. + callbacks[1].run(); + Assert.assertEquals(2, pool.leakedSlotCount()); + ((QwpWebSocketSender) delegates[1]).setSlotLockReleasedForTesting(true); + callbacks[1].run(); + Assert.assertEquals(1, pool.leakedSlotCount()); + + // Duplicate and stale callbacks are idempotent and do not + // probe or mutate the slot after its direct removal. + ((QwpWebSocketSender) delegates[2]).setSlotLockReleasedForTesting(true); + callbacks[2].run(); + long probesAfterRecovery = pool.getRetiredSlotProbeCountForTesting(); + callbacks[2].run(); + callbacks[1].run(); + Assert.assertEquals(probesAfterRecovery, + pool.getRetiredSlotProbeCountForTesting()); + Assert.assertEquals(0, pool.leakedSlotCount()); + Assert.assertTrue(pool.getRetiredSlotCountForTesting() == 0); } } - Assert.assertTrue("unacked data must persist under default-0", - hasSegmentFile(slot("default-0"))); + }); + } - // Phase 2: ack-ing server + a new pool whose injected factory forges - // the exact leak symptom for the recovery build of slot 0. The - // factory returns a real, flock-holding QwpWebSocketSender but - // pre-sets closed=true, so the recovery close() is a complete no-op - // (checkNotClosed short-circuits drain too): the flock stays held - // and slotLockReleased never flips -- precisely a refused I/O-thread - // stop. flockReleased(recoverer) must therefore report false. + @Test + public void testMixedRetiredSlotsRecoverWithoutLosingAccounting() throws Exception { + TestUtils.assertMemoryLeak(() -> { CountingAckHandler handler = new CountingAckHandler(); - try (TestWebSocketServer ack = new TestWebSocketServer(handler)) { - int ackPort = ack.getPort(); - ack.start(); - Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS)); - String cfg2 = "ws::addr=localhost:" + ackPort + ";sf_dir=" + sfDir + ";"; + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); - AtomicReference forged = new AtomicReference<>(); - IntFunction factory = idx -> { - Sender real = Sender.builder(cfg2).senderId("default-" + idx).build(); - if (idx == 0) { - try { - setBooleanField(real, "closed", true); - } catch (Exception e) { - throw new RuntimeException(e); - } - forged.set(real); + String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try (SenderPool pool = new SenderPool( + config, 0, 8, 500, Long.MAX_VALUE, Long.MAX_VALUE)) { + PooledSender[] leases = new PooledSender[8]; + Sender[] delegates = new Sender[8]; + for (int i = 0; i < leases.length; i++) { + leases[i] = pool.borrow(); + delegates[i] = getDelegate(leases[i]); } - return real; - }; + for (int i = 0; i < leases.length; i++) { + // Release the real native resources, then forge the + // delayed publication which makes the pool retire the + // slot. The idempotent second close in discardBroken() + // leaves the forged state unchanged. + delegates[i].close(); + ((QwpWebSocketSender) delegates[i]).setSlotLockReleasedForTesting(false); + pool.discardBrokenForTesting(leases[i]); + } + Assert.assertEquals(8, pool.leakedSlotCount()); + + // Mix completed and incomplete entries throughout the + // retired list. A recovery pass must remove exactly these + // four without skipping a swapped entry or corrupting the + // bitmap/count relationship. + int[] released = {0, 2, 5, 7}; + for (int i = 0; i < released.length; i++) { + ((QwpWebSocketSender) delegates[released[i]]).setSlotLockReleasedForTesting(true); + } + pool.reapIdle(); - // minSize=0 so prewarm never adopts slot 0 -- recovery is the - // only builder of slot 0. maxSize=2 so a later borrow can still - // get a fresh slot (default-1), proving capacity dropped by one. - SenderPool pool = newPoolWithFactory(cfg2, 0, 2, 500, factory); - try { - // The forge must actually have reached recovery's build. - Assert.assertNotNull("recovery must have built slot 0", forged.get()); - // The retire branch must have fired during construction. - Assert.assertEquals("recovery must retire the still-locked slot as leaked", - 1, getIntField(pool, "leakedSlots")); - boolean[] slotInUse = (boolean[]) getField(pool, "slotInUse"); - Assert.assertTrue("retired slot 0 must stay reserved", slotInUse[0]); - Assert.assertFalse("slot 1 must remain free", slotInUse[1]); + Assert.assertEquals(4, pool.leakedSlotCount()); + Assert.assertEquals(4, pool.getRetiredSlotCountForTesting()); + for (int i = 0; i < delegates.length; i++) { + boolean mustRemainRetired = i == 1 || i == 3 || i == 4 || i == 6; + Assert.assertEquals("slot reservation mismatch at index " + i, + mustRemainRetired, pool.isSlotInUseForTesting(i)); + } - // A later borrow must take the fresh slot 1, never re-pick - // the still-locked default-0 (which would throw "sf slot - // already in use"). - PooledSender b = pool.borrow(); + // Exactly the four restored reservations are reusable. + PooledSender[] recovered = new PooledSender[4]; try { - Assert.assertTrue("borrow must use a fresh slot dir", - Files.exists(slot("default-1"))); - // Capacity is permanently reduced by the leaked slot: - // max=2, one leaked + one live => the next borrow times - // out rather than colliding on the locked dir. - try { - pool.borrow(); - Assert.fail("capacity must be reduced by the leaked slot"); - } catch (LineSenderException e) { - Assert.assertTrue(e.getMessage(), e.getMessage().contains("timed out")); + for (int i = 0; i < recovered.length; i++) { + recovered[i] = pool.borrow(); + } + Assert.assertEquals(4, pool.totalSize()); + Assert.assertEquals(4, pool.leakedSlotCount()); + } finally { + for (int i = 0; i < recovered.length; i++) { + if (recovered[i] != null) { + recovered[i].close(); + } + } + } + } + } + }); + } + + @Test(timeout = 30_000) + public void testMultipleWaitingBorrowersWakeForStaggeredRetiredSlotRecovery() throws Exception { + TestUtils.assertMemoryLeak(() -> { + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try (SenderPool pool = new SenderPool( + config, 0, 6, 10_000, Long.MAX_VALUE, Long.MAX_VALUE)) { + PooledSender[] leases = new PooledSender[6]; + Sender[] delegates = new Sender[6]; + for (int i = 0; i < leases.length; i++) { + leases[i] = pool.borrow(); + delegates[i] = getDelegate(leases[i]); + } + for (int i = 0; i < leases.length; i++) { + delegates[i].close(); + ((QwpWebSocketSender) delegates[i]).setSlotLockReleasedForTesting(false); + pool.discardBrokenForTesting(leases[i]); + } + Assert.assertEquals("all capacity must start retired", + 6, pool.leakedSlotCount()); + + final int borrowerCount = 3; + CountDownLatch allAcquired = new CountDownLatch(borrowerCount); + CountDownLatch allDone = new CountDownLatch(borrowerCount); + CountDownLatch firstAcquired = new CountDownLatch(1); + CountDownLatch initialWaiters = new CountDownLatch(borrowerCount); + CountDownLatch releaseBorrowers = new CountDownLatch(1); + CountDownLatch reparkedWaiters = new CountDownLatch(borrowerCount - 1); + AtomicReference failure = new AtomicReference<>(); + ConcurrentHashMap initialWaiterThreads = new ConcurrentHashMap<>(); + ConcurrentHashMap reparkedWaiterThreads = new ConcurrentHashMap<>(); + PooledSender[] recovered = new PooledSender[borrowerCount]; + pool.setBeforeBorrowWaitHook(() -> { + if (initialWaiterThreads.putIfAbsent(Thread.currentThread(), Boolean.TRUE) == null) { + initialWaiters.countDown(); + } + }); + for (int i = 0; i < borrowerCount; i++) { + final int borrower = i; + Thread thread = new Thread(() -> { + try { + recovered[borrower] = pool.borrow(); + allAcquired.countDown(); + firstAcquired.countDown(); + if (!releaseBorrowers.await(10, TimeUnit.SECONDS)) { + throw new AssertionError("timed out waiting to release recovered borrower"); + } + } catch (Throwable e) { + failure.compareAndSet(null, e); + } finally { + try { + if (recovered[borrower] != null) { + recovered[borrower].close(); + } + } catch (Throwable e) { + failure.compareAndSet(null, e); + } finally { + allDone.countDown(); + } + } + }, "sender-pool-retired-waiter-" + i); + thread.start(); + } + + try { + Assert.assertTrue("all borrowers must reach the condition wait", + initialWaiters.await(5, TimeUnit.SECONDS)); + Assert.assertEquals("three distinct borrowers must reach the wait path", + borrowerCount, initialWaiterThreads.size()); + + // The hook runs while each borrower holds the pool lock, + // immediately before awaitNanos atomically releases that + // lock and enqueues it. Once the last latch count lands, + // the first reapIdle() cannot acquire the lock until all + // three borrowers are definitely condition waiters. + pool.setBeforeBorrowWaitHook(() -> { + if (reparkedWaiterThreads.putIfAbsent(Thread.currentThread(), Boolean.TRUE) == null) { + reparkedWaiters.countDown(); + } + }); + ((QwpWebSocketSender) delegates[2]).setSlotLockReleasedForTesting(true); + pool.reapIdle(); + + // One restored index admits exactly one borrower. The + // other two must consume signalAll(), lose the capacity + // race, and deterministically re-enter the wait path. + Assert.assertTrue("one borrower must take the first restored index", + firstAcquired.await(5, TimeUnit.SECONDS)); + Assert.assertTrue("two distinct borrowers must re-park after the first recovery", + reparkedWaiters.await(5, TimeUnit.SECONDS)); + Assert.assertEquals("exactly two distinct borrowers must re-enter the wait path", + borrowerCount - 1, reparkedWaiterThreads.size()); + Assert.assertEquals("exactly one borrower should hold restored capacity", + borrowerCount - 1, allAcquired.getCount()); + + // Recover two non-contiguous entries in one reverse / + // swap-remove pass. A single signal() here would wake + // only one of the two proven waiters; signalAll() must + // wake both and let them claim the two restored indices. + pool.setBeforeBorrowWaitHook(null); + ((QwpWebSocketSender) delegates[0]).setSlotLockReleasedForTesting(true); + ((QwpWebSocketSender) delegates[5]).setSlotLockReleasedForTesting(true); + pool.reapIdle(); + Assert.assertTrue("all waiting borrowers must receive restored capacity", + allAcquired.await(5, TimeUnit.SECONDS)); + Assert.assertEquals(3, pool.totalSize()); + Assert.assertEquals(3, pool.leakedSlotCount()); + + boolean[] seen = new boolean[6]; + for (int i = 0; i < recovered.length; i++) { + int slotIndex = recovered[i].getSlotIndexForTesting(); + Assert.assertFalse("borrowers must receive distinct restored indices", + seen[slotIndex]); + seen[slotIndex] = true; + } + Assert.assertTrue("slot 0 must be restored", seen[0]); + Assert.assertTrue("slot 2 must be restored", seen[2]); + Assert.assertTrue("slot 5 must be restored", seen[5]); + if (failure.get() != null) { + throw new AssertionError("waiting borrower failed", failure.get()); + } + } finally { + pool.setBeforeBorrowWaitHook(null); + releaseBorrowers.countDown(); + Assert.assertTrue("borrower threads must finish", + allDone.await(10, TimeUnit.SECONDS)); + } + if (failure.get() != null) { + throw new AssertionError("waiting borrower failed", failure.get()); + } + } + } + }); + } + + @Test + public void testPoolRetiresAndRecoversSlotThroughFailedFlockReleaseRetry() throws Exception { + TestUtils.assertMemoryLeak(() -> { + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try (SenderPool pool = new SenderPool(config, 1, 1, 2_000, + Long.MAX_VALUE, Long.MAX_VALUE)) { + PooledSender a = pool.borrow(); + Sender delegate = getDelegate(a); + CursorSendEngine engine = ((QwpWebSocketSender) delegate).getCursorEngineForTesting(); + SlotLock slotLock = engine.getSlotLockForTesting(); + SlotLock.ReleaseFailureForTesting releaseFailure = + slotLock.injectReleaseFailureForTesting(); + try { + // Inject one persistent explicit-unlock failure. + // Delegate close must retire the only pool slot rather + // than publish a release while the real flock remains held. + pool.discardBrokenForTesting(a); + Assert.assertEquals("failed release must retire pool capacity", + 1, pool.leakedSlotCount()); + Assert.assertFalse(engine.isCloseCompleted()); + + // Remove the fault without calling close again: the + // engine's error-path retry driver runs outside the + // pool lock and must eventually publish completion. + releaseFailure.close(); + long deadlineNs = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (!engine.isCloseCompleted()) { + if (System.nanoTime() > deadlineNs) { + throw new AssertionError("flock-release retry never completed"); + } + Thread.sleep(1L); + } + + // A capacity-starved public borrow re-probes the + // retained delegate, frees index 0, and proves the + // recovered flock is genuinely acquirable. + PooledSender recovered = pool.borrow(); + try { + Assert.assertEquals("retry must restore pool capacity", + 0, pool.leakedSlotCount()); + Assert.assertEquals(1, countSlotDirs()); + } finally { + recovered.close(); + } + } finally { + if (!engine.isCloseCompleted()) { + releaseFailure.close(); + Assert.assertTrue("restored fd must release cleanly", + slotLock.release()); + } + } + } + } + }); + } + + @Test + public void testPoolRetiresAndRecoversSlotThroughRealManagerWorkerWedge() throws Exception { + // Full-stack twin of the two forged-flag recovery tests above: no + // reflection-forged slotLockReleased anywhere. The REAL mechanism is + // driven end to end — the delegate's owned SegmentManager worker is + // wedged mid service pass (test hook), the delegate's close() takes + // the real timed-out-join → worker-exit handoff path, the pool + // retires the slot off the delegate's genuine isSlotLockReleased() + // report, the worker's deferred cleanup releases the real flock, and + // the housekeeper re-probe restores capacity — proving the recovered + // index is genuinely reusable by borrowing on it again (a forged flag + // would pass the accounting asserts but collide on the flock here). + TestUtils.assertMemoryLeak(() -> { + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try (SenderPool pool = new SenderPool(config, 1, 2, 500, Long.MAX_VALUE, Long.MAX_VALUE)) { + PooledSender a = pool.borrow(); + Assert.assertTrue(Files.exists(slot("default-0"))); + + Sender delegate = getDelegate(a); + CursorSendEngine engine = ((QwpWebSocketSender) delegate).getCursorEngineForTesting(); + Assert.assertNotNull("SF delegate must own a cursor engine", engine); + SegmentManager manager = engine.getManagerForTesting(); + + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(); + AtomicReference hookErr = new AtomicReference<>(); + try { + // Park the manager worker inside a service pass for + // the delegate's ring. The trim-sync point is reached + // on every ~1ms tick, so this wedges deterministically. + manager.setBeforeTrimSyncHook(() -> { + if (!fired.compareAndSet(false, true)) return; + workerBlocked.countDown(); + try { + if (!releaseWorker.await(20, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, new AssertionError( + "timed out waiting for test to release worker")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + Assert.assertTrue("manager worker never entered a service pass", + workerBlocked.await(5, TimeUnit.SECONDS)); + + // Real teardown against the wedged worker: the + // delegate's engine close times out its bounded join, + // hands cleanup to the worker's exit path, and reports + // the retained flock; the pool must retire the slot. + // Fault the old callback-allocation/registration path. + // C3 makes the owned-engine handoff preallocated, so + // this hook must no longer be reached by production + // teardown. + manager.setBeforeExitCleanupRegistrationHook(() -> { + throw new OutOfMemoryError("simulated callback allocation failure"); + }); + manager.setWorkerJoinTimeoutMillis(50L); + pool.discardBrokenForTesting(a); + Assert.assertEquals( + "pool must retire the slot while the delegate's manager " + + "worker holds the deferred cleanup", + 1, pool.leakedSlotCount()); + Assert.assertFalse("engine cleanup must still be pending", + engine.isCloseCompleted()); + + // Un-wedge and deterministically reap the manager. + // No sender/engine close retry is allowed: worker exit + // itself must own and finish the preallocated handoff. + releaseWorker.countDown(); + manager.close(); + Assert.assertTrue("manager worker must be reaped", manager.isWorkerReaped()); + Assert.assertTrue("worker exit must run deferred cleanup despite the " + + "old allocation fault injection", + engine.isCloseCompleted()); + + // The housekeeper tick is the recovery driver. + pool.reapIdle(); + Assert.assertEquals("recovered slot must leave the leaked count", + 0, pool.leakedSlotCount()); + Assert.assertFalse("recovered slot index 0 must return to the free set", + pool.isSlotInUseForTesting(0)); + + // The proof a forged flag cannot fake: both indices — + // including the recovered one, whose flock was really + // dropped by the worker-exit cleanup — admit live + // senders again. + PooledSender b = pool.borrow(); + PooledSender c = pool.borrow(); + try { + Assert.assertEquals(2, countSlotDirs()); + } finally { + c.close(); + b.close(); + } + if (hookErr.get() != null) { + throw new AssertionError("trim hook failed", hookErr.get()); + } + } finally { + manager.setBeforeExitCleanupRegistrationHook(null); + manager.setBeforeTrimSyncHook(null); + releaseWorker.countDown(); + } + } + } + }); + } + + @Test + public void testPreallocatedExitHandoffCleansInRangeStartupRecoverer() throws Exception { + assertPreallocatedExitHandoffCleansStartupRecoverer(0, 1); + } + + @Test + public void testPreallocatedExitHandoffCleansOutOfRangeStartupRecoverer() throws Exception { + assertPreallocatedExitHandoffCleansStartupRecoverer(1, 1); + } + + // ---------------------------------------------------------------------- + // Recovery: stable slot ids let a re-created pool re-adopt unacked data. + // ---------------------------------------------------------------------- + + @Test + public void testRecoveryReplayThroughPooledSlot() throws Exception { + // Phase 1: write rows to a slot against a silent server (no acks), so + // the data persists unacked on disk under default-0. Close. + // Phase 2: a new pool against an ack-ing server re-adopts default-0 + // (stable index) and replays the unacked frames. Stable, deterministic + // slot ids are exactly what make this recovery possible. + TestUtils.assertMemoryLeak(() -> { + // Phase 1 -- silent server. + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int silentPort = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String cfg1 = "ws::addr=localhost:" + silentPort + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=500;"; + try (SenderPool pool = new SenderPool(cfg1, 1, 1, 1_000, Long.MAX_VALUE, Long.MAX_VALUE)) { + PooledSender s = pool.borrow(); + for (int i = 0; i < 3; i++) { + s.table("recover").longColumn("v", i).atNow(); + s.flush(); + } + s.close(); + } + } + // Data must be on disk, unacked, under default-0. + Assert.assertTrue("unacked data must persist on disk", hasSegmentFile(slot("default-0"))); + + // Phase 2 -- ack-ing server, brand-new pool, same sf_dir. + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer ack = new TestWebSocketServer(handler)) { + int ackPort = ack.getPort(); + ack.start(); + Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS)); + String cfg2 = "ws::addr=localhost:" + ackPort + ";sf_dir=" + sfDir + ";"; + try (SenderPool pool = new SenderPool(cfg2, 1, 1, 5_000, Long.MAX_VALUE, Long.MAX_VALUE)) { + PooledSender s = pool.borrow(); + try { + // Drain replays the recovered, unacked frames. + s.drain(5_000); + Assert.assertTrue("recovered frames must be replayed to the new server", + awaitAtLeast(handler.frames, 1, 5_000)); + } finally { + s.close(); + } + } + } + }); + } + + @Test + public void testRecoveryDelegateForcesOffInitialConnectMode() throws Exception { + // M1 regression: a startup-recovery delegate runs on the PoolHousekeeper + // thread, so its build() must NOT inherit the user's SYNC initial-connect + // mode (auto-enabled by any reconnect_* knob). SYNC would retry the + // connect for the whole reconnect budget inside build() -- far past + // PoolHousekeeper.STOP_TIMEOUT_MILLIS -- so a close() landing during that + // build would make the housekeeper join time out and leave the recoverer + // holding the slot flock after close() returned. The recovery factory + // forces initial_connect_mode=OFF (at most one connect attempt); the + // normal factory must still honour the user's promoted SYNC mode. + TestUtils.assertMemoryLeak(() -> { + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer ack = new TestWebSocketServer(handler)) { + int ackPort = ack.getPort(); + ack.start(); + Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS)); + // reconnect_max_duration_millis set, initial_connect_mode unset + // -> the builder promotes to SYNC for ordinary senders. + String cfg = "ws::addr=localhost:" + ackPort + ";sf_dir=" + sfDir + + ";reconnect_max_duration_millis=30000;"; + // min=0 (no prewarm connect), no stranded data (recovery no-op). + try (SenderPool pool = new SenderPool(cfg, 0, 2, 1_000, Long.MAX_VALUE, Long.MAX_VALUE)) { + // Normal managed-slot delegate: inherits the promoted SYNC. + Sender normal = pool.buildSenderForTesting(0); + try { + Assert.assertEquals( + "ordinary pooled sender must honour the user's promoted SYNC mode", + Sender.InitialConnectMode.SYNC, ((QwpWebSocketSender) normal).getInitialConnectModeForTesting()); + } finally { + normal.close(); + } + // Recovery delegate on a different slot: forced OFF. + Sender recoverer = pool.buildRecoverySenderForTesting(1); + try { + Assert.assertEquals( + "recovery delegate must force OFF so build() makes at most one connect attempt", + Sender.InitialConnectMode.OFF, ((QwpWebSocketSender) recoverer).getInitialConnectModeForTesting()); + } finally { + recoverer.close(); + } + } + } + }); + } + + @Test + public void testSharedManagerPassCompletionRecoversRetiredPoolSlot() throws Exception { + TestUtils.assertMemoryLeak(() -> { + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + long segSize = io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment.HEADER_SIZE + + io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment.FRAME_HEADER_SIZE + 32L; + SegmentManager manager = new SegmentManager(segSize, TimeUnit.SECONDS.toNanos(60)); + CountDownLatch cleanupFinished = new CountDownLatch(1); + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(); + AtomicReference hookErr = new AtomicReference<>(); + AtomicReference engineRef = new AtomicReference<>(); + manager.setAfterRingCleanupHook(cleanupFinished::countDown); + manager.setBeforeInstallSyncHook(() -> { + if (!fired.compareAndSet(false, true)) { + return; + } + workerBlocked.countDown(); + try { + if (!releaseWorker.await(20, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, + new AssertionError("timed out waiting for test to release worker")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + manager.start(); + Assert.assertEquals(0, Files.mkdir(sfDir, Files.DIR_MODE_DEFAULT)); + String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + IntFunction factory = slotIndex -> { + CursorSendEngine engine = new CursorSendEngine( + slot("default-" + slotIndex), segSize, manager); + engineRef.set(engine); + QwpWebSocketSender sender = QwpWebSocketSender.createForTesting("localhost", port); + sender.setCursorEngine(engine, true); + return sender; + }; + SenderPool pool = new SenderPool( + config, 0, 1, 500, 0, Long.MAX_VALUE, factory); + try { + PooledSender lease = pool.borrow(); + Assert.assertTrue("shared manager never entered the sender ring's service pass", + workerBlocked.await(5, TimeUnit.SECONDS)); + lease.close(); + + manager.setWorkerJoinTimeoutMillis(50L); + pool.reapIdle(); + Assert.assertEquals("pool must retire the slot while its shared-manager pass is live", + 1, pool.leakedSlotCount()); + CursorSendEngine engine = engineRef.get(); + Assert.assertFalse("engine cleanup must remain pending during the live pass", + engine.isCloseCompleted()); + + releaseWorker.countDown(); + Assert.assertTrue("deferred cleanup did not finish with the ring pass", + cleanupFinished.await(5, TimeUnit.SECONDS)); + if (hookErr.get() != null) { + throw new AssertionError("install hook failed", hookErr.get()); + } + + pool.reapIdle(); + Assert.assertEquals("pass completion must drive deferred engine cleanup and restore capacity", + 0, pool.leakedSlotCount()); + Assert.assertTrue("deferred cleanup must publish the released flock", + engine.isCloseCompleted()); + try (PooledSender recovered = pool.borrow()) { + Assert.assertTrue("recovered slot must be reusable", Files.exists(slot("default-0"))); + } + } finally { + releaseWorker.countDown(); + manager.setAfterRingCleanupHook(null); + manager.setBeforeInstallSyncHook(null); + pool.close(); + manager.close(); + } + } + }); + } + + @Test + public void testStartupRecoveryRetiresSlotWhenRecovererCloseLeavesFlockHeld() throws Exception { + // C1 regression: the startup recovery loop MUST mirror discardBroken / + // reapIdle. When a recoverer's delegate close() returns with the SF + // flock still held (the I/O thread refused to stop), the recovered slot + // index must be retired (leakedSlots++, slotInUse stays + // set) -- NOT freed. Freeing it would let a later borrow re-pick the + // still-locked dir and resurrect "sf slot already in use", the exact + // failure class this PR exists to kill. Pre-fix the recovery finally + // set slotInUse[i]=false unconditionally; this test is RED until it + // consults flockReleased() like the other two close-and-reclaim paths. + TestUtils.assertMemoryLeak(() -> { + // Phase 1: leave unacked data on disk under default-0 so startup + // recovery treats it as a candidate orphan and builds a recoverer. + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int silentPort = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String cfg1 = "ws::addr=localhost:" + silentPort + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=500;"; + try (SenderPool pool = new SenderPool(cfg1, 1, 1, 1_000, Long.MAX_VALUE, Long.MAX_VALUE)) { + PooledSender s = pool.borrow(); + for (int i = 0; i < 3; i++) { + s.table("recover").longColumn("v", i).atNow(); + s.flush(); + } + s.close(); + } + } + Assert.assertTrue("unacked data must persist under default-0", + hasSegmentFile(slot("default-0"))); + + // Phase 2: ack-ing server + a new pool whose injected factory forges + // the exact leak symptom for the recovery build of slot 0. The + // factory returns a real, flock-holding QwpWebSocketSender but + // pre-sets closed=true, so the recovery close() is a complete no-op + // (checkNotClosed short-circuits drain too): the flock stays held + // and slotLockReleased never flips -- precisely a refused I/O-thread + // stop. flockReleased(recoverer) must therefore report false. + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer ack = new TestWebSocketServer(handler)) { + int ackPort = ack.getPort(); + ack.start(); + Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS)); + String cfg2 = "ws::addr=localhost:" + ackPort + ";sf_dir=" + sfDir + ";"; + + AtomicReference forged = new AtomicReference<>(); + IntFunction factory = idx -> { + Sender real = Sender.builder(cfg2).senderId("default-" + idx).build(); + if (idx == 0) { + try { + ((QwpWebSocketSender) real).setClosedForTesting(true); + } catch (Exception e) { + throw new RuntimeException(e); + } + forged.set(real); + } + return real; + }; + + // minSize=0 so prewarm never adopts slot 0 -- recovery is the + // only builder of slot 0. maxSize=2 so a later borrow can still + // get a fresh slot (default-1), proving capacity dropped by one. + SenderPool pool = newPoolWithFactory(cfg2, 0, 2, 500, factory); + try { + // The forge must actually have reached recovery's build. + Assert.assertNotNull("recovery must have built slot 0", forged.get()); + // The retire branch must have fired during construction. + Assert.assertEquals("recovery must retire the still-locked slot as leaked", + 1, pool.leakedSlotCount()); + Assert.assertTrue("retired slot 0 must stay reserved", + pool.isSlotInUseForTesting(0)); + Assert.assertFalse("slot 1 must remain free", + pool.isSlotInUseForTesting(1)); + + // A later borrow must take the fresh slot 1, never re-pick + // the still-locked default-0 (which would throw "sf slot + // already in use"). + PooledSender b = pool.borrow(); + try { + Assert.assertTrue("borrow must use a fresh slot dir", + Files.exists(slot("default-1"))); + // Capacity stays reduced while the flock is held: + // max=2, one leaked + one live => the next borrow times + // out rather than colliding on the locked dir. + try { + pool.borrow(); + Assert.fail("capacity must be reduced by the leaked slot"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage(), e.getMessage().contains("timed out")); + } + } finally { + b.close(); + } + } finally { + pool.close(); + // Release the forged recoverer's real flock + native + // resources: pool.close() never saw it (recovery never + // added the recoverer to `all`), so un-forge closed and + // close it for real, otherwise assertMemoryLeak trips. + Sender leaked = forged.get(); + if (leaked != null) { + ((QwpWebSocketSender) leaked).setClosedForTesting(false); + leaked.close(); + } + } + } + }); + } + + @Test + public void testStartupRetiredSlotRecoveredAfterLateFlockRelease() throws Exception { + // Recovery twin of + // testStartupRecoveryRetiresSlotWhenRecovererCloseLeavesFlockHeld: a + // slot retired by STARTUP recovery (recoverer close() returned with + // the flock still held) must not stay lost until process exit. + // isSlotLockReleased() is no longer a one-shot snapshot -- deferred + // engine cleanup on a worker exit path can release the flock later -- + // so the pool must keep the recoverer in retiredSlots and re-probe it. + // Pre-fix, startup recovery only ticked leakedSlots and dropped the + // recoverer: at maxSize=1 every later borrow timed out forever even + // after the flock dropped. This test is RED until the recoverer is + // retained. + TestUtils.assertMemoryLeak(() -> { + // Phase 1: strand unacked data under default-0 so startup recovery + // treats it as a candidate orphan and builds a recoverer. + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int silentPort = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String cfg1 = "ws::addr=localhost:" + silentPort + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=500;"; + try (SenderPool pool = new SenderPool(cfg1, 1, 1, 1_000, Long.MAX_VALUE, Long.MAX_VALUE)) { + PooledSender s = pool.borrow(); + for (int i = 0; i < 3; i++) { + s.table("recover").longColumn("v", i).atNow(); + s.flush(); + } + s.close(); + } + } + Assert.assertTrue("unacked data must persist under default-0", + hasSegmentFile(slot("default-0"))); + + // Phase 2: maxSize=1 -- the worst case, where the single slot's + // retirement starves the whole pool. Forge the retention exactly + // like the retire test (closed=true makes the recovery drain and + // close a no-op, so the real flock stays held and slotLockReleased + // stays false), but only for the FIRST build of slot 0: the + // post-recovery borrow below must get a working delegate on the + // recovered index. + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer ack = new TestWebSocketServer(handler)) { + int ackPort = ack.getPort(); + ack.start(); + Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS)); + String cfg2 = "ws::addr=localhost:" + ackPort + ";sf_dir=" + sfDir + ";"; + + AtomicReference forged = new AtomicReference<>(); + IntFunction factory = idx -> { + Sender real = Sender.builder(cfg2).senderId("default-" + idx).build(); + if (idx == 0 && forged.compareAndSet(null, real)) { + try { + ((QwpWebSocketSender) real).setClosedForTesting(true); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + return real; + }; + + try (SenderPool pool = newPoolWithFactory(cfg2, 0, 1, 500, factory)) { + Assert.assertNotNull("recovery must have built slot 0", forged.get()); + Assert.assertEquals("precondition: startup recovery must retire the slot", + 1, pool.leakedSlotCount()); + + // While the flock is genuinely held, borrows time out: the + // cap-check re-probe finds the flock still reported held. + try { + pool.borrow(); + Assert.fail("borrow must time out while the slot is retired"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage(), e.getMessage().contains("timed out")); + } + + // The late release: the "wedged worker" finishes and the + // flock genuinely drops (un-forge and close the recoverer + // for real; in production this flip comes from + // isSlotLockReleased() re-probing the retained engine after + // the worker exited). + Sender recoverer = forged.get(); + ((QwpWebSocketSender) recoverer).setClosedForTesting(false); + recoverer.close(); + + // The capacity-starved borrow must re-probe the startup- + // retired slot, recover its capacity, and create on the + // freed index -- no housekeeper tick required. + PooledSender b = pool.borrow(); + try { + Assert.assertEquals("borrow must recover the startup-retired slot's capacity", + 0, pool.leakedSlotCount()); + Assert.assertTrue("recovered index 0 must carry the new borrow", + pool.isSlotInUseForTesting(0)); + Assert.assertEquals(1, countSlotDirs()); + } finally { + b.close(); + } + } + } + }); + } + + @Test + public void testRecoveryScanStaysAliveWhileRetiredSlotHoldsStrandedData() throws Exception { + // Drain-liveness regression: a slot RETIRED mid-scan (recoverer close() + // returned with the flock still held) keeps slotInUse[i] set, so the + // next walk over its index took the "reserved => live" skip WITHOUT + // counting a deferral and latched recoveryComplete with the slot's + // unacked data still on disk. From there the data was stranded until a + // restart or a lucky borrow of that exact index -- which steady low + // load may never produce (testStartupRetiredSlotRecoveredAfterLate- + // FlockRelease passes only because its borrow IS that lucky borrow). + // The scan must instead treat a retired index whose dir is still a + // candidate orphan as a deferral -- the same rule as a CONTENDED park: + // the cycle keeps rewinding, and once the late flock release restores + // the index the SCAN itself drains the data, no borrow required. This + // test is RED (latches complete) until that deferral is counted. + TestUtils.assertMemoryLeak(() -> { + // Phase 1: strand unacked data under default-0. + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int silentPort = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String cfg1 = "ws::addr=localhost:" + silentPort + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=500;"; + try (SenderPool pool = new SenderPool(cfg1, 1, 1, 1_000, Long.MAX_VALUE, Long.MAX_VALUE)) { + PooledSender s = pool.borrow(); + for (int i = 0; i < 3; i++) { + s.table("recover").longColumn("v", i).atNow(); + s.flush(); + } + s.close(); + } + } + Assert.assertTrue("unacked data must persist under default-0", + hasSegmentFile(slot("default-0"))); + + // Phase 2: ack-ing server; the FIRST recovery build of slot 0 is + // forged into the wedged shape (closed=true makes the recovery + // drain throw and close() a no-op, so the real flock stays held + // and slotLockReleased stays false) so the scan's own drain FAILs + // and the slot is retired. deferStartupRecovery=true keeps the + // scan entirely under this test's control -- no background driver. + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer ack = new TestWebSocketServer(handler)) { + int ackPort = ack.getPort(); + ack.start(); + Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS)); + String cfg2 = "ws::addr=localhost:" + ackPort + ";sf_dir=" + sfDir + ";"; + + AtomicReference forged = new AtomicReference<>(); + IntFunction factory = idx -> { + Sender real = Sender.builder(cfg2).senderId("default-" + idx).build(); + if (idx == 0 && forged.compareAndSet(null, real)) { + try { + ((QwpWebSocketSender) real).setClosedForTesting(true); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + return real; + }; + + try (SenderPool pool = new SenderPool(cfg2, 0, 1, 500, + Long.MAX_VALUE, Long.MAX_VALUE, factory, true)) { + // Step 1: builds the forged recoverer, fails its drain, and + // retires the slot (leakedSlots=1, slotInUse[0] stays set). + // The FAILED outcome stops the drive without advancing. + long deadline = System.currentTimeMillis() + 10_000; + while (pool.leakedSlotCount() == 0 + && System.currentTimeMillis() < deadline) { + pool.runStartupRecoveryStepForTesting(); + } + Assert.assertNotNull("recovery must have built slot 0", forged.get()); + Assert.assertEquals("scan must retire the wedged slot", + 1, pool.leakedSlotCount()); + + // Re-drive the scan: the retired index is skipped as + // reserved. Pre-fix the first full walk latched + // recoveryComplete; post-fix the still-candidate retired + // dir counts as a deferral and the cycle keeps rewinding. + for (int k = 0; k < 8; k++) { + pool.runStartupRecoveryStepForTesting(); + } + Assert.assertTrue("retired dir must still hold the stranded data", + hasSegmentFile(slot("default-0"))); + Assert.assertFalse( + "scan must NOT latch recoveryComplete while a retired slot still " + + "holds stranded durable data -- that abandons the data " + + "until a restart or a lucky borrow of that exact index", + pool.isRecoveryCompleteForTesting()); + + // The late flock release: the "wedged worker" finishes + // (un-forge + real close). The housekeeper tick re-probes + // retiredSlots and frees the index; the still-alive scan + // must then reserve it and drain the stranded data ITSELF + // -- no borrow anywhere in this phase. + Sender recoverer = forged.get(); + ((QwpWebSocketSender) recoverer).setClosedForTesting(false); + recoverer.close(); + pool.reapIdle(); + Assert.assertEquals("late release must restore the retired capacity", + 0, pool.leakedSlotCount()); + + deadline = System.currentTimeMillis() + 10_000; + while (!pool.isRecoveryCompleteForTesting() + && System.currentTimeMillis() < deadline) { + pool.runStartupRecoveryStepForTesting(); + } + Assert.assertTrue("scan must complete once the stranded data is drained", + pool.isRecoveryCompleteForTesting()); + Assert.assertTrue("the scan itself must have drained the stranded data", + awaitNoSegmentFile(slot("default-0"), 15_000)); + Assert.assertTrue("replayed frames must reach the server", + awaitAtLeast(handler.frames, 1, 15_000)); + } + } + }); + } + + @Test + public void testRuntimeRetireAfterScanCompleteRearmsRecoveryOnFlockRelease() throws Exception { + // Residual drain-liveness gap, runtime flavour: the startup scan + // finishes LEGITIMATELY (recoveryComplete latches), and only LATER + // does a runtime reclaim (discardBroken) retire a slot whose close() + // kept the flock -- with unacked durable data still in its dir. When + // the wedged worker finally releases the flock, recoverRetiredSlotAt + // restores the CAPACITY but nothing re-drains the dir: the scan is + // latched and close() never owns the dir, so the data waits for a + // restart or a lucky borrow of that exact index. Pin the fix: the + // flock RELEASE of a retired slot whose dir is still a candidate + // orphan must re-arm the scan (un-latch + rewind) so the ordinary + // driver cadence drains it in THIS process. RED at the un-latch + // assertion until recoverRetiredSlotAt re-arms. + TestUtils.assertMemoryLeak(() -> { + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer ack = new TestWebSocketServer(handler); + TestWebSocketServer wedgeSink = new TestWebSocketServer(new SilentHandler())) { + int ackPort = ack.getPort(); + ack.start(); + Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS)); + wedgeSink.start(); + Assert.assertTrue(wedgeSink.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + ackPort + ";sf_dir=" + sfDir + ";"; + // The wedged sender: alive worker streaming into a silent sink + // (rows stay durably unacked) whose close() must never drain + // the chain on the data's behalf. + String cfgWedge = "ws::addr=localhost:" + wedgeSink.getPort() + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=0;"; + + boolean[] wedgeNextBuild = new boolean[1]; + IntFunction factory = idx -> + Sender.builder(wedgeNextBuild[0] ? cfgWedge : cfg) + .senderId("default-" + idx).build(); + + try (SenderPool pool = new SenderPool(cfg, 0, 1, 500, + Long.MAX_VALUE, Long.MAX_VALUE, factory, true)) { + // Phase 1: the scan completes legitimately (clean dirs). + pool.runStartupRecoveryToCompletionForTesting(); + Assert.assertTrue("precondition: scan must have latched complete", + pool.isRecoveryCompleteForTesting()); + + // Phase 2: runtime wedge AFTER the latch. The borrowed + // sender writes durably against the silent sink, then its + // close() is forged to keep the flock (closed=true makes + // discardBroken's close a no-op). + wedgeNextBuild[0] = true; + PooledSender ps = pool.borrow(); + wedgeNextBuild[0] = false; + for (int i = 0; i < 3; i++) { + ps.table("resid").longColumn("v", i).atNow(); + ps.flush(); + } + Sender delegate = ps.getDelegateForTesting(); + ((QwpWebSocketSender) delegate).setClosedForTesting(true); + pool.discardBrokenForTesting(ps); + Assert.assertEquals("runtime reclaim must retire the wedged slot", + 1, pool.leakedSlotCount()); + Assert.assertTrue("retired dir must hold the stranded data", + hasSegmentFile(slot("default-0"))); + Assert.assertTrue("retire alone must not disturb the latch", + pool.isRecoveryCompleteForTesting()); + + // Phase 3: the late release -- the wedged worker exits + // WITHOUT delivering (close-flush 0 against the silent + // sink). The release listener restores capacity; the fix + // must ALSO re-arm the scan because the freed dir is still + // a candidate orphan. + ((QwpWebSocketSender) delegate).setClosedForTesting(false); + delegate.close(); + Assert.assertEquals("release must restore the retired capacity", + 0, pool.leakedSlotCount()); + Assert.assertFalse( + "flock release of a retired slot with stranded data must re-arm " + + "the recovery scan -- otherwise the data waits for a restart " + + "or a lucky borrow of that exact index", + pool.isRecoveryCompleteForTesting()); + + // Phase 4: the ordinary driver cadence (housekeeper proxy) + // must now drain the dir and re-latch. + long deadline = System.currentTimeMillis() + 10_000; + while (!pool.isRecoveryCompleteForTesting() + && System.currentTimeMillis() < deadline) { + pool.runStartupRecoveryStepForTesting(); + } + Assert.assertTrue("re-armed scan must complete once the data is drained", + pool.isRecoveryCompleteForTesting()); + Assert.assertTrue("the re-armed scan must deliver the stranded data", + awaitNoSegmentFile(slot("default-0"), 15_000)); + Assert.assertTrue("replayed frames must reach the server", + awaitAtLeast(handler.frames, 1, 15_000)); + } + } + }); + } + + @Test + public void testRevivedDirectDriverDrainsRuntimeRetiredSlotAfterLateRelease() throws Exception { + // Direct-pool twin of + // testRuntimeRetireAfterScanCompleteRearmsRecoveryOnFlockRelease: a + // direct pool (deferStartupRecovery=false) finishes its inline scan at + // construction and lets its private recovery driver die. A runtime + // retire after that leaves NO driver to pick up the re-armed scan + // when the flock releases -- the un-latch alone is not enough. Pin + // the revival: the release must respawn the direct pool's private + // driver, which drains the freed dir with NO test-side stepping and + // no borrow. RED (segments never clear) until release revives the + // driver. + TestUtils.assertMemoryLeak(() -> { + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer ack = new TestWebSocketServer(handler); + TestWebSocketServer wedgeSink = new TestWebSocketServer(new SilentHandler())) { + int ackPort = ack.getPort(); + ack.start(); + Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS)); + wedgeSink.start(); + Assert.assertTrue(wedgeSink.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + ackPort + ";sf_dir=" + sfDir + ";"; + String cfgWedge = "ws::addr=localhost:" + wedgeSink.getPort() + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=0;"; + + boolean[] wedgeNextBuild = new boolean[1]; + IntFunction factory = idx -> + Sender.builder(wedgeNextBuild[0] ? cfgWedge : cfg) + .senderId("default-" + idx).build(); + + // deferStartupRecovery=false: the inline scan over the clean + // dir completes at construction; no private driver survives. + try (SenderPool pool = new SenderPool(cfg, 0, 1, 500, + Long.MAX_VALUE, Long.MAX_VALUE, factory, false)) { + Assert.assertTrue("precondition: inline scan must have latched complete", + pool.isRecoveryCompleteForTesting()); + + wedgeNextBuild[0] = true; + PooledSender ps = pool.borrow(); + wedgeNextBuild[0] = false; + for (int i = 0; i < 3; i++) { + ps.table("resid").longColumn("v", i).atNow(); + ps.flush(); + } + Sender delegate = ps.getDelegateForTesting(); + ((QwpWebSocketSender) delegate).setClosedForTesting(true); + pool.discardBrokenForTesting(ps); + Assert.assertEquals("runtime reclaim must retire the wedged slot", + 1, pool.leakedSlotCount()); + Assert.assertTrue("retired dir must hold the stranded data", + hasSegmentFile(slot("default-0"))); + + // Late release without delivery. From here the pool must + // recover the data ENTIRELY on its own: no recovery-step + // calls, no reapIdle, no borrow -- the revived private + // driver is the only possible deliverer. + ((QwpWebSocketSender) delegate).setClosedForTesting(false); + delegate.close(); + + Assert.assertTrue( + "the revived direct driver must drain the freed dir on its own -- " + + "no steps, no reap, no borrow", + awaitNoSegmentFile(slot("default-0"), 15_000)); + Assert.assertTrue("replayed frames must reach the server", + awaitAtLeast(handler.frames, 1, 15_000)); + long deadline = System.currentTimeMillis() + 10_000; + while (!pool.isRecoveryCompleteForTesting() + && System.currentTimeMillis() < deadline) { + Thread.sleep(10); + } + Assert.assertTrue("the revived driver must re-latch completion", + pool.isRecoveryCompleteForTesting()); + Assert.assertEquals(0, pool.leakedSlotCount()); + } + } + }); + } + + @Test + public void testMidCycleFlockReleaseCannotLatchScanPastStrandedData() throws Exception { + // Third drain-liveness shape (found by SenderPoolSfFuzzTest): a + // retire AND its flock release both land while the scan is ALIVE but + // after the cursor already passed that index in its current cycle -- + // the index was LIVE (borrowed) when walked, so no retired-candidate + // deferral was counted, and the release fires with recoveryComplete + // still false, so the post-latch re-arm path does not apply either. + // If the rest of the cycle finds nothing to defer, the scan latches + // with the freed dir's data stranded. Pin the fix: a release that + // frees a still-candidate dir mid-cycle must flag a rewind that the + // latch decision consumes, so the cycle cannot latch past it. RED at + // the post-latch assertion until the rearm flag exists. + TestUtils.assertMemoryLeak(() -> { + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer ack = new TestWebSocketServer(handler); + TestWebSocketServer wedgeSink = new TestWebSocketServer(new SilentHandler())) { + int ackPort = ack.getPort(); + ack.start(); + Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS)); + wedgeSink.start(); + Assert.assertTrue(wedgeSink.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + ackPort + ";sf_dir=" + sfDir + ";"; + String cfgWedge = "ws::addr=localhost:" + wedgeSink.getPort() + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=0;"; + + // Choreography state: slot 0 is the wedge target (borrowed + // BEFORE the walk so the cursor skips it as live). Slot 1's + // FIRST recovery build points at the silent sink so its drain + // times out (FAILED, flock released, cursor parked ON slot 1 + // past slot 0); its second build is real and drains. + boolean[] wedgeNextBuild = new boolean[1]; + int[] slot1Builds = new int[1]; + IntFunction factory = idx -> { + String c = cfg; + if (wedgeNextBuild[0]) { + c = cfgWedge; + } else if (idx == 1 && slot1Builds[0]++ == 0) { + c = cfgWedge; + } + return Sender.builder(c).senderId("default-" + idx).build(); + }; + + try (SenderPool pool = new SenderPool(cfg, 0, 2, 500, + Long.MAX_VALUE, Long.MAX_VALUE, factory, true)) { + // Strand slot 1 only: plant unacked rows through a + // borrowed silent-sink sender... simpler: write via a + // wedge-built borrow on slot 0 FIRST (it will stay live + // through the walk), and seed slot 1 by a second borrow + // released before recovery starts. + wedgeNextBuild[0] = true; + PooledSender live0 = pool.borrow(); // creates slot 0, silent sink + wedgeNextBuild[0] = false; + wedgeNextBuild[0] = true; + PooledSender seed1 = pool.borrow(); // creates slot 1, silent sink + wedgeNextBuild[0] = false; + for (int i = 0; i < 2; i++) { + live0.table("midcycle").longColumn("v", i).atNow(); + live0.flush(); + seed1.table("midcycle").longColumn("w", i).atNow(); + seed1.flush(); + } + // Return slot 1's seeder through the HEALTHY discard path + // (real close, flock released, data left unacked on disk). + Sender seed1Delegate = seed1.getDelegateForTesting(); + pool.discardBrokenForTesting(seed1); + Assert.assertEquals("seeding must not leak capacity", 0, pool.leakedSlotCount()); + Assert.assertTrue("slot 1 must hold unacked data", + hasSegmentFile(slot("default-1"))); + Assert.assertTrue("slot 0 must hold unacked data", + hasSegmentFile(slot("default-0"))); + + // Step A: the walk skips slot 0 (live borrow -- NO + // deferral, correctly), reaches slot 1, builds the + // silent-sink recoverer, and the drain times out: FAILED, + // cursor parked ON slot 1, scan alive, zero deferrals. + Assert.assertFalse(pool.runStartupRecoveryStepForTesting()); + Assert.assertFalse("scan must still be running", + pool.isRecoveryCompleteForTesting()); + + // Mid-cycle: retire slot 0 (wedged close) and heal it + // immediately -- the release lands with the scan alive and + // the cursor already PAST slot 0. + Sender live0Delegate = live0.getDelegateForTesting(); + ((QwpWebSocketSender) live0Delegate).setClosedForTesting(true); + pool.discardBrokenForTesting(live0); + Assert.assertEquals(1, pool.leakedSlotCount()); + ((QwpWebSocketSender) live0Delegate).setClosedForTesting(false); + live0Delegate.close(); + Assert.assertEquals("release must restore capacity", 0, pool.leakedSlotCount()); + Assert.assertTrue("slot 0 data must still be stranded", + hasSegmentFile(slot("default-0"))); + + // Step B: cursor resumes AT slot 1 (never revisits slot 0 + // this cycle), slot 1's second build is real and drains, + // the cycle ends with zero deferrals. Pre-fix this latched + // recoveryComplete past slot 0's stranded data; the fix's + // rearm flag must force a rewind instead. + long deadline = System.currentTimeMillis() + 10_000; + while (hasSegmentFile(slot("default-1")) + && System.currentTimeMillis() < deadline) { + pool.runStartupRecoveryStepForTesting(); + } + Assert.assertFalse("slot 1 must have drained", hasSegmentFile(slot("default-1"))); + // One more step reaches the end-of-cycle latch decision + // (pass 2 is empty, zero deferrals were counted). + pool.runStartupRecoveryStepForTesting(); + Assert.assertFalse( + "a mid-cycle flock release that frees a still-candidate dir must " + + "flag a rewind -- the cycle must NOT latch recoveryComplete " + + "past the stranded data it never revisited", + pool.isRecoveryCompleteForTesting()); + + // The rewound cycle must now drain slot 0 and only then + // complete. + deadline = System.currentTimeMillis() + 10_000; + while (!pool.isRecoveryCompleteForTesting() + && System.currentTimeMillis() < deadline) { + pool.runStartupRecoveryStepForTesting(); + } + Assert.assertTrue("rewound scan must complete after draining slot 0", + pool.isRecoveryCompleteForTesting()); + Assert.assertTrue("the rewound scan must deliver slot 0's stranded data", + awaitNoSegmentFile(slot("default-0"), 15_000)); + Assert.assertTrue("replayed frames must reach the server", + awaitAtLeast(handler.frames, 1, 15_000)); + // Silence the unused warning; the seeder delegate was + // closed for real by discardBroken (healthy path). + Assert.assertNotNull(seed1Delegate); + } + } + }); + } + + @Test + public void testZeroTimeoutBorrowProbesRetiredSlotBeforeThrowing() throws Exception { + // Boundary twin of testStartupRetiredSlotRecoveredAfterLateFlockRelease: + // acquireTimeoutMillis=0 is a valid try-once borrow (builder rejects only + // < 0). Pre-fix, borrow() ran the terminal timeout check BEFORE + // reprobeRetiredSlots(), so a zero-budget borrow threw "timed out" + // without its one probe -- even when the retired slot's flock had + // already dropped and a probe would have restored capacity and admitted + // a creation. Deterministic: no housekeeper runs in this test, so + // borrow() is the only reprobe driver. Recovery is driven manually via + // the deferred-pool step helper because the inline path reuses + // acquireTimeoutMillis as its recovery budget -- 0 would skip recovery + // outright. This test is RED until the probe is hoisted above the + // timeout check. + TestUtils.assertMemoryLeak(() -> { + // Phase 1: strand unacked data under default-0 so startup recovery + // treats it as a candidate orphan and builds a recoverer. + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int silentPort = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String cfg1 = "ws::addr=localhost:" + silentPort + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=500;"; + try (SenderPool pool = new SenderPool(cfg1, 1, 1, 1_000, Long.MAX_VALUE, Long.MAX_VALUE)) { + PooledSender s = pool.borrow(); + for (int i = 0; i < 3; i++) { + s.table("recover").longColumn("v", i).atNow(); + s.flush(); + } + s.close(); + } + } + Assert.assertTrue("unacked data must persist under default-0", + hasSegmentFile(slot("default-0"))); + + // Phase 2: maxSize=1, zero acquire budget, deferred recovery driven + // step-by-step (the housekeeper's per-tick unit, budgeted by + // RECOVERY_DRAIN_BUDGET_MILLIS, not the zero acquire budget). Forge + // the retirement (closed=true makes the recovery drain and close a + // no-op, so the real flock stays held), then release the flock for + // real BEFORE borrowing: the recovery is discoverable, but only via + // a probe. + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer ack = new TestWebSocketServer(handler)) { + int ackPort = ack.getPort(); + ack.start(); + Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS)); + String cfg2 = "ws::addr=localhost:" + ackPort + ";sf_dir=" + sfDir + ";"; + + AtomicReference forged = new AtomicReference<>(); + IntFunction factory = idx -> { + Sender real = Sender.builder(cfg2).senderId("default-" + idx).build(); + if (idx == 0 && forged.compareAndSet(null, real)) { + try { + ((QwpWebSocketSender) real).setClosedForTesting(true); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + return real; + }; + + try (SenderPool pool = newDeferredPoolWithFactory(cfg2, 0, 1, 0, factory)) { + //noinspection StatementWithEmptyBody + while (pool.runStartupRecoveryStepForTesting()) { + // drive the whole backlog, one housekeeper-tick unit at a time + } + Assert.assertNotNull("recovery must have built slot 0", forged.get()); + Assert.assertEquals("precondition: startup recovery must retire the slot", + 1, pool.leakedSlotCount()); + + // The late release: un-forge and close the recoverer for + // real. The flock genuinely drops, but nothing signals the + // pool -- the release happens in the delegate, volatile + // writes only. + Sender recoverer = forged.get(); + ((QwpWebSocketSender) recoverer).setClosedForTesting(false); + recoverer.close(); + + // Try-once borrow: its single pass must probe, recover the + // capacity, and create on the freed index. Pre-fix this + // threw "timed out" without ever probing. + PooledSender b = pool.borrow(); + try { + Assert.assertEquals("zero-timeout borrow must recover the retired slot's capacity", + 0, pool.leakedSlotCount()); + } finally { + b.close(); + } + } + } + }); + } + + @Test + public void testParkedBorrowerGetsFinalProbeAfterBudgetExpiry() throws Exception { + // Positive-timeout twin of the zero-timeout test. A borrower parks in + // awaitNanos while the retired slot's flock is genuinely held and + // sleeps out its full budget. A test hook releases the flock after the + // wait reports expiry but before the terminal loop pass. Pre-fix that + // pass hit the timeout check before reprobeRetiredSlots() and threw -- + // missing capacity that had already come back. Post-fix the terminal + // pass probes first, recovers the index, and admits the creation. + TestUtils.assertMemoryLeak(() -> { + // Phase 1: strand unacked data under default-0. + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int silentPort = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String cfg1 = "ws::addr=localhost:" + silentPort + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=500;"; + try (SenderPool pool = new SenderPool(cfg1, 1, 1, 1_000, Long.MAX_VALUE, Long.MAX_VALUE)) { + PooledSender s = pool.borrow(); + for (int i = 0; i < 3; i++) { + s.table("recover").longColumn("v", i).atNow(); + s.flush(); + } + s.close(); + } + } + Assert.assertTrue("unacked data must persist under default-0", + hasSegmentFile(slot("default-0"))); + + // Phase 2: maxSize=1 and a positive acquire budget. A test hook + // releases the flock only after awaitNanos() has returned with that + // budget exhausted, so the terminal wake-up pass is deterministic. + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer ack = new TestWebSocketServer(handler)) { + int ackPort = ack.getPort(); + ack.start(); + Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS)); + String cfg2 = "ws::addr=localhost:" + ackPort + ";sf_dir=" + sfDir + ";"; + + AtomicReference forged = new AtomicReference<>(); + IntFunction factory = idx -> { + Sender real = Sender.builder(cfg2).senderId("default-" + idx).build(); + if (idx == 0 && forged.compareAndSet(null, real)) { + try { + ((QwpWebSocketSender) real).setClosedForTesting(true); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + return real; + }; + + try (SenderPool pool = newPoolWithFactory(cfg2, 0, 1, 100, factory)) { + Assert.assertNotNull("recovery must have built slot 0", forged.get()); + Assert.assertEquals("precondition: startup recovery must retire the slot", + 1, pool.leakedSlotCount()); + + AtomicBoolean waitExpired = new AtomicBoolean(); + pool.setBorrowWaitExpiredHook(() -> { + Assert.assertTrue("expired-wait hook must run exactly once", + waitExpired.compareAndSet(false, true)); + Sender recoverer = forged.get(); + try { + ((QwpWebSocketSender) recoverer).setClosedForTesting(false); + } catch (Exception e) { + throw new RuntimeException(e); + } + // The flock drops only after the positive awaitNanos() + // budget is exhausted. This delegate-side release does + // not signal slotReleased. + recoverer.close(); + }); + try { + PooledSender b = pool.borrow(); + try { + Assert.assertTrue("borrow must exhaust its positive wait budget", + waitExpired.get()); + Assert.assertEquals("wake-up probe must recover the retired slot's capacity", + 0, pool.leakedSlotCount()); + } finally { + b.close(); } } finally { - b.close(); - } - } finally { - pool.close(); - // Release the forged recoverer's real flock + native - // resources: pool.close() never saw it (recovery never - // added the recoverer to `all`), so un-forge closed and - // close it for real, otherwise assertMemoryLeak trips. - Sender leaked = forged.get(); - if (leaked != null) { - setBooleanField(leaked, "closed", false); - leaked.close(); + pool.setBorrowWaitExpiredHook(null); } } } @@ -1270,123 +2767,810 @@ public void testDefaultConfigRecoversOutOfRangeSlotsAfterShrink() throws Excepti hasSegmentFile(slot("default-" + i))); } - // Phase 2: restart at maxSize=2 with the DEFAULT config (NO - // drain_orphans) against a healthy ack server. minSize=0 so startup - // recovery -- not prewarm "normal use" -- is the recovery path for - // the in-range slots, and pass 2 is the only path for default-2/3. + // Phase 2: restart at maxSize=2 with the DEFAULT config (NO + // drain_orphans) against a healthy ack server. minSize=0 so startup + // recovery -- not prewarm "normal use" -- is the recovery path for + // the in-range slots, and pass 2 is the only path for default-2/3. + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer ack = new TestWebSocketServer(handler)) { + int ackPort = ack.getPort(); + ack.start(); + Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + ackPort + ";sf_dir=" + sfDir + ";"; + try (SenderPool pool = new SenderPool(cfg, 0, 2, 5_000, Long.MAX_VALUE, Long.MAX_VALUE)) { + // In-range [0,2): startup recovery pass 1 drains these. + Assert.assertTrue("in-range default-0 must be recovered at startup", + awaitNoSegmentFile(slot("default-0"), 15_000)); + Assert.assertTrue("in-range default-1 must be recovered at startup", + awaitNoSegmentFile(slot("default-1"), 15_000)); + + // Out-of-range [2,4): pass 2 must deliver these too, under + // the default config (no drain_orphans). + Assert.assertTrue("default-2 unacked data must be recovered, not stranded", + awaitNoSegmentFile(slot("default-2"), 15_000)); + Assert.assertTrue("default-3 unacked data must be recovered, not stranded", + awaitNoSegmentFile(slot("default-3"), 15_000)); + Assert.assertFalse("out-of-range slot must not be abandoned as .failed", + Files.exists(slot("default-2") + "/" + OrphanScanner.FAILED_SENTINEL_NAME)); + Assert.assertFalse("out-of-range slot must not be abandoned as .failed", + Files.exists(slot("default-3") + "/" + OrphanScanner.FAILED_SENTINEL_NAME)); + + // Sanity: the pool is still usable for normal borrows. + PooledSender a = pool.borrow(); + a.close(); + } + } + }); + } + + @Test + public void testFailedOutOfRangeRecoveryRetriesAfterPrimaryReturns() throws Exception { + // A deferred pool can already have its sole in-range sender borrowed when + // startup recovery reaches an out-of-range slot left by a larger pool. + // A transient recoverer build failure must leave that candidate pending: + // after the primary lease returns, the SAME pool must retry and drain it. + TestUtils.assertMemoryLeak(() -> { + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String seedConfig = "ws::addr=localhost:" + silent.getPort() + ";sf_dir=" + sfDir + + ";sender_id=default-1;close_flush_timeout_millis=0;"; + try (Sender seed = Sender.fromConfig(seedConfig)) { + seed.table("recover").longColumn("v", 1L).atNow(); + seed.flush(); + } + } + Assert.assertTrue("out-of-range fixture must contain unacked data", + hasSegmentFile(slot("default-1"))); + + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer ack = new TestWebSocketServer(handler)) { + ack.start(); + Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS)); + String config = "ws::addr=localhost:" + ack.getPort() + ";sf_dir=" + sfDir + ";"; + AtomicBoolean primaryReturned = new AtomicBoolean(); + AtomicInteger recoveryAttempts = new AtomicInteger(); + IntFunction factory = idx -> { + if (idx == 1) { + recoveryAttempts.incrementAndGet(); + if (!primaryReturned.get()) { + throw new LineSenderException("transient out-of-range recovery failure"); + } + } + return Sender.builder(config).senderId("default-" + idx).build(); + }; + + try (SenderPool pool = newDeferredPoolWithFactory(config, 0, 1, 5_000, factory)) { + PooledSender primary = pool.borrow(); + + Assert.assertFalse("first recovery attempt must stop at the transient failure", + pool.runStartupRecoveryStepForTesting()); + Assert.assertEquals("exactly one out-of-range recovery attempt", 1, + recoveryAttempts.get()); + Assert.assertTrue("failed recovery must preserve the candidate", + hasSegmentFile(slot("default-1"))); + Assert.assertEquals("out-of-range failure must not consume in-range capacity", + 0, pool.leakedSlotCount()); + Assert.assertTrue("out-of-range recoverer must not enter retired-slot bookkeeping", + pool.getRetiredSlotCountForTesting() == 0); + + primary.close(); + primaryReturned.set(true); + pool.runStartupRecoveryToCompletionForTesting(); + + Assert.assertEquals("same live pool must retry the out-of-range candidate", 2, + recoveryAttempts.get()); + Assert.assertFalse("retry must drain the preserved out-of-range data", + hasSegmentFile(slot("default-1"))); + Assert.assertTrue("retry must deliver the recovered frame", handler.frames.get() >= 1); + Assert.assertEquals("successful out-of-range retry must not consume capacity", + 0, pool.leakedSlotCount()); + Assert.assertTrue("out-of-range retry must leave retired slots untouched", + pool.getRetiredSlotCountForTesting() == 0); + + PooledSender next = pool.borrow(); + try { + Assert.assertTrue("normal borrow must reuse the returned primary slot", + primary.hasSameSlotForTesting(next)); + } finally { + next.close(); + } + } + } + }); + } + + @Test + public void testDrainFailureRetriesInRangeAndOutOfRangeCandidates() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + for (int i = 0; i < 2; i++) { + String seedConfig = "ws::addr=localhost:" + silent.getPort() + ";sf_dir=" + sfDir + + ";sender_id=default-" + i + ";close_flush_timeout_millis=0;"; + try (Sender seed = Sender.fromConfig(seedConfig)) { + seed.table("recover").longColumn("v", i).atNow(); + seed.flush(); + } + } + } + Assert.assertTrue("in-range fixture must contain unacked data", + hasSegmentFile(slot("default-0"))); + Assert.assertTrue("out-of-range fixture must contain unacked data", + hasSegmentFile(slot("default-1"))); + + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer ack = new TestWebSocketServer(handler)) { + ack.start(); + Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS)); + String config = "ws::addr=localhost:" + ack.getPort() + ";sf_dir=" + sfDir + ";"; + AtomicInteger[] attempts = {new AtomicInteger(), new AtomicInteger()}; + IntFunction factory = idx -> { + if (attempts[idx].incrementAndGet() == 1) { + return (Sender) Proxy.newProxyInstance( + Sender.class.getClassLoader(), + new Class[]{Sender.class}, + (proxy, method, args) -> { + if ("drain".equals(method.getName())) { + throw new LineSenderException("transient drain failure for slot " + idx); + } + if ("close".equals(method.getName())) { + return null; + } + throw new AssertionError("unexpected recovery sender call: " + method.getName()); + }); + } + return Sender.builder(config).senderId("default-" + idx).build(); + }; + + try (SenderPool pool = newDeferredPoolWithFactory(config, 0, 1, 5_000, factory)) { + Assert.assertFalse("failed in-range drain must defer the same candidate", + pool.runStartupRecoveryStepForTesting()); + Assert.assertEquals(1, attempts[0].get()); + Assert.assertEquals(0, attempts[1].get()); + Assert.assertTrue("failed in-range drain must preserve durable data", + hasSegmentFile(slot("default-0"))); + + Assert.assertTrue("successful in-range retry may continue scanning", + pool.runStartupRecoveryStepForTesting()); + Assert.assertEquals("same live pool must retry the in-range candidate", + 2, attempts[0].get()); + Assert.assertFalse("in-range retry must drain the preserved data", + hasSegmentFile(slot("default-0"))); + + Assert.assertFalse("failed out-of-range drain must defer the same candidate", + pool.runStartupRecoveryStepForTesting()); + Assert.assertEquals(1, attempts[1].get()); + Assert.assertTrue("failed out-of-range drain must preserve durable data", + hasSegmentFile(slot("default-1"))); + + Assert.assertTrue("successful out-of-range retry may finish the candidate", + pool.runStartupRecoveryStepForTesting()); + Assert.assertEquals("same live pool must retry the out-of-range candidate", + 2, attempts[1].get()); + Assert.assertFalse("out-of-range retry must drain the preserved data", + hasSegmentFile(slot("default-1"))); + Assert.assertFalse("final scan step must mark recovery complete", + pool.runStartupRecoveryStepForTesting()); + Assert.assertTrue("both recovered frames must be delivered", handler.frames.get() >= 2); + } + } + }); + } + + @Test + public void testInRangeIdleSlotIsRecoveredAtStartupUnderSteadyLowLoad() throws Exception { + // The drain exclusion is bounded to [0, maxSize) so a sibling's drainer + // never adopts a slot dir the pool intends to (re)create -- that is what + // prevents "sf slot already in use" (see + // testDrainOrphansPoolDoesNotCannibalizeSiblingSlots). The trade-off was + // that an in-range slot left holding unacked data by a previous run was + // recovered ONLY when the pool happened to (re)create that index: the + // pool pre-warms [0, minSize) and builds [minSize, maxSize) lazily on + // demand, so under steady low load a high in-range index was never + // rebuilt -- neither drained (excluded) nor recovered -- and its data + // was stranded on disk until a restart or load spike. + // + // The fix has the pool recover its own stranded managed slots once, at + // construction, under its own slot reservation (so the cannibalization + // race the exclusion guards against still cannot happen). This test + // seeds a busy run, then restarts under steady low load and asserts the + // idle in-range slots are recovered anyway. + TestUtils.assertMemoryLeak(() -> { + // Phase 1: a busy run at maxSize=4 seeds unacked data into + // default-0..3 (silent server never acks; close_flush_timeout=0 so + // close() leaves the flushed-but-unacked .sfa on disk). + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int silentPort = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String seedCfg = "ws::addr=localhost:" + silentPort + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=0;"; + try (SenderPool seed = new SenderPool(seedCfg, 4, 4, 5_000, Long.MAX_VALUE, Long.MAX_VALUE)) { + PooledSender[] s = new PooledSender[4]; + for (int i = 0; i < 4; i++) { + s[i] = seed.borrow(); + } + for (int i = 0; i < 4; i++) { + s[i].table("recover").longColumn("v", i).atNow(); + s[i].flush(); + } + for (int i = 3; i >= 0; i--) { + s[i].close(); + } + } + } + for (int i = 0; i < 4; i++) { + Assert.assertTrue("default-" + i + " must hold unacked data", + hasSegmentFile(slot("default-" + i))); + } + + // Phase 2: restart at the SAME maxSize=4 (so default-0..3 stay in + // range) with steady low load. minSize=0 means prewarm builds + // nothing and the lowest-free allocator would never reach the high + // indices under a single in-flight borrow -- yet startup recovery + // must still empty every in-range slot. + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer ack = new TestWebSocketServer(handler)) { + int ackPort = ack.getPort(); + ack.start(); + Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + ackPort + ";sf_dir=" + sfDir + ";"; + try (SenderPool pool = new SenderPool(cfg, 0, 4, 5_000, Long.MAX_VALUE, Long.MAX_VALUE)) { + // All four in-range slots must be recovered by the startup + // pass, even though steady low load never grows the pool to + // their indices. + for (int i = 0; i < 4; i++) { + Assert.assertTrue("in-range idle default-" + i + + " must be recovered at startup, not stranded", + awaitNoSegmentFile(slot("default-" + i), 15_000)); + Assert.assertFalse("recovered slot must not be flagged .failed", + Files.exists(slot("default-" + i) + "/" + OrphanScanner.FAILED_SENTINEL_NAME)); + } + // The recovered frames must have actually reached the server. + Assert.assertTrue("recovered frames must be replayed to the server", + awaitAtLeast(handler.frames, 4, 15_000)); + + // Sanity: the pool is still usable for normal borrows. + PooledSender a = pool.borrow(); + a.close(); + } + } + }); + } + + @Test + public void testDirectRecoveryContinuesAfterFiniteBudgetExhaustion() throws Exception { + createCandidateSlot("default-0"); + createCandidateSlot("default-1"); + CountDownLatch drained = new CountDownLatch(2); + AtomicInteger[] attempts = {new AtomicInteger(), new AtomicInteger()}; + IntFunction factory = idx -> { + attempts[idx].incrementAndGet(); + return successfulRecoverySender(idx, drained); + }; + String config = "ws::addr=localhost:1;sf_dir=" + sfDir + ";"; + + try (SenderPool pool = newPoolWithFactory(config, 0, 1, 0, factory)) { + Assert.assertTrue("direct pool must continue recovery after its inline budget expires", + drained.await(5, TimeUnit.SECONDS)); + Assert.assertEquals("in-range candidate must be recovered", 1, attempts[0].get()); + Assert.assertEquals("out-of-range candidate must be recovered", 1, attempts[1].get()); + } + } + + @Test + public void testDirectRecoveryCloseJoinsDriverAndStopsRemainingCandidates() throws Exception { + createCandidateSlot("default-0"); + createCandidateSlot("default-1"); + CountDownLatch afterJoin = new CountDownLatch(1); + CountDownLatch beforeJoin = new CountDownLatch(1); + CountDownLatch closeReturned = new CountDownLatch(1); + CountDownLatch delegateCloseStarted = new CountDownLatch(1); + CountDownLatch drainStarted = new CountDownLatch(1); + CountDownLatch releaseDelegateClose = new CountDownLatch(1); + CountDownLatch releaseDrain = new CountDownLatch(1); + AtomicBoolean driverAliveAfterJoin = new AtomicBoolean(); + AtomicInteger[] attempts = {new AtomicInteger(), new AtomicInteger()}; + AtomicReference closeFailure = new AtomicReference<>(); + IntFunction senderFactory = idx -> { + attempts[idx].incrementAndGet(); + return blockingFakeSender( + idx, drainStarted, releaseDrain, delegateCloseStarted, releaseDelegateClose); + }; + + SenderPool pool = newPoolWithFactory( + "ws::addr=localhost:1;sf_dir=" + sfDir + ";", + 0, 2, 0, senderFactory); + Thread recoveryThread = pool.getStartupRecoveryThreadForTesting(); + pool.setStartupRecoveryJoinHooksForTesting( + beforeJoin::countDown, + () -> { + driverAliveAfterJoin.set(recoveryThread.isAlive()); + afterJoin.countDown(); + }); + Thread closeThread = new Thread(() -> { + try { + pool.close(); + } catch (Throwable t) { + closeFailure.set(t); + } finally { + closeReturned.countDown(); + } + }, "test-direct-pool-close"); + try { + Assert.assertTrue("direct recovery driver must enter the first drain", + drainStarted.await(10, TimeUnit.SECONDS)); + Assert.assertTrue("direct recovery driver must be running", recoveryThread.isAlive()); + + closeThread.start(); + Assert.assertTrue("close must enter its direct-driver join operation", + beforeJoin.await(10, TimeUnit.SECONDS)); + Assert.assertTrue("close itself must raise the shutdown signal before joining", + pool.isClosedForTesting()); + + releaseDrain.countDown(); + Assert.assertTrue("driver must remain deliberately held before termination", + delegateCloseStarted.await(10, TimeUnit.SECONDS)); + Assert.assertEquals("close must not return while its driver is held", + 1L, closeReturned.getCount()); + + releaseDelegateClose.countDown(); + Assert.assertTrue("close must return after the driver is released", + closeReturned.await(10, TimeUnit.SECONDS)); + Assert.assertTrue("close must complete its join operation", + afterJoin.await(10, TimeUnit.SECONDS)); + if (closeFailure.get() != null) { + throw new AssertionError("close failed", closeFailure.get()); + } + Assert.assertFalse("the recovery driver must be dead when close's join returns", + driverAliveAfterJoin.get()); + Assert.assertFalse("close must leave the direct recovery driver quiescent", + recoveryThread.isAlive()); + Assert.assertEquals("the in-flight candidate must have been built", 1, attempts[0].get()); + Assert.assertEquals("shutdown must prevent a later candidate build", 0, attempts[1].get()); + } finally { + releaseDrain.countDown(); + releaseDelegateClose.countDown(); + closeThread.join(TimeUnit.SECONDS.toMillis(10)); + pool.close(); + } + } + + @Test + public void testDirectRecoveryFailedAttemptEntersRetryWait() throws Throwable { + createCandidateSlot("default-0"); + CountDownLatch beforeJoin = new CountDownLatch(1); + CountDownLatch closeReturned = new CountDownLatch(1); + CountDownLatch recoveryOperationObserved = new CountDownLatch(1); + CountDownLatch releaseWait = new CountDownLatch(1); + CountDownLatch waitEntered = new CountDownLatch(1); + AtomicInteger attempts = new AtomicInteger(); + AtomicReference closeFailure = new AtomicReference<>(); + IntFunction senderFactory = idx -> { + if (attempts.incrementAndGet() > 1) { + recoveryOperationObserved.countDown(); + } + throw new LineSenderException("injected recovery failure"); + }; + Runnable recoveryWaiter = () -> { + waitEntered.countDown(); + recoveryOperationObserved.countDown(); + try { + releaseWait.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }; + + SenderPool pool = newPoolWithRecoveryControls( + "ws::addr=localhost:1;sf_dir=" + sfDir + ";", + 0, 1, 0, senderFactory, null, recoveryWaiter, null); + pool.setStartupRecoveryJoinHooksForTesting(beforeJoin::countDown, null); + Thread closeThread = new Thread(() -> { + try { + pool.close(); + } catch (Throwable t) { + closeFailure.set(t); + } finally { + closeReturned.countDown(); + } + }, "test-retry-wait-close"); + try { + Assert.assertTrue("recovery must either wait or retry after the failed attempt", + recoveryOperationObserved.await(10, TimeUnit.SECONDS)); + Assert.assertEquals("a failed recovery attempt must enter the retry-wait operation", + 0L, waitEntered.getCount()); + Assert.assertEquals("the driver must wait immediately after its first failed attempt", + 1, attempts.get()); + + closeThread.start(); + Assert.assertTrue("close must reach the driver join while the waiter is held", + beforeJoin.await(10, TimeUnit.SECONDS)); + releaseWait.countDown(); + Assert.assertTrue("close must finish after the retry waiter is released", + closeReturned.await(10, TimeUnit.SECONDS)); + if (closeFailure.get() != null) { + throw new AssertionError("close failed", closeFailure.get()); + } + Assert.assertEquals("shutdown must prevent another recovery attempt", 1, attempts.get()); + } finally { + releaseWait.countDown(); + closeThread.join(TimeUnit.SECONDS.toMillis(10)); + pool.close(); + } + } + + @Test + public void testDirectRecoveryThreadCreationFailureClosesPrewarmedDelegates() throws Throwable { + createCandidateSlot("default-2"); + AssertionError failure = new AssertionError("injected recovery thread creation failure"); + AtomicInteger[] closeCalls = {new AtomicInteger(), new AtomicInteger()}; + IntFunction senderFactory = idx -> closeCountingSender(idx, closeCalls, idx == 0); + ThreadFactory threadFactory = runnable -> { + throw failure; + }; + + try { + newPoolWithRecoveryThreadFactory( + "ws::addr=localhost:1;sf_dir=" + sfDir + ";", + 2, 2, 0, senderFactory, threadFactory); + Assert.fail("construction must propagate the recovery thread creation failure"); + } catch (AssertionError actual) { + Assert.assertSame("construction must preserve throwable identity", failure, actual); + } + Assert.assertEquals("cleanup must attempt the first prewarmed delegate", 1, closeCalls[0].get()); + Assert.assertEquals("one close failure must not strand the next delegate", 1, closeCalls[1].get()); + } + + @Test + public void testDirectRecoveryThreadCreationFailureReleasesPrewarmedSfFlocks() throws Exception { + TestUtils.assertMemoryLeak(() -> { + createCandidateSlot("default-2"); + AssertionError failure = new AssertionError("injected recovery thread creation failure"); + ThreadFactory threadFactory = runnable -> { + throw failure; + }; + + try (TestWebSocketServer ack = new TestWebSocketServer(new CountingAckHandler())) { + ack.start(); + Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS)); + String config = "ws::addr=localhost:" + ack.getPort() + ";sf_dir=" + sfDir + ";"; + try { + newPoolWithRecoveryThreadFactory(config, 2, 2, 0, null, threadFactory); + Assert.fail("construction must propagate the recovery thread creation failure"); + } catch (AssertionError actual) { + Assert.assertSame("construction must preserve throwable identity", failure, actual); + } catch (Throwable unexpected) { + throw new AssertionError("unexpected construction failure", unexpected); + } + + try (SlotLock ignored0 = SlotLock.acquire(slot("default-0")); + SlotLock ignored1 = SlotLock.acquire(slot("default-1"))) { + // Reacquisition proves failed construction closed both real + // prewarmed delegates and released their SF flocks. + } + } + }); + } + + @Test + public void testDirectRecoveryThreadStartFailureClosesPrewarmedDelegates() throws Throwable { + createCandidateSlot("default-2"); + AssertionError failure = new AssertionError("injected recovery thread start failure"); + AtomicBoolean cleanupBeforeDriverQuiescence = new AtomicBoolean(); + AtomicInteger failedJoinCalls = new AtomicInteger(); + AtomicInteger[] closeCalls = {new AtomicInteger(), new AtomicInteger()}; + AtomicReference recoveryThread = new AtomicReference<>(); + CountDownLatch releaseDriver = new CountDownLatch(1); + CountDownLatch running = new CountDownLatch(1); + IntFunction senderFactory = idx -> closeCountingSender( + idx, closeCalls, false, recoveryThread, cleanupBeforeDriverQuiescence, releaseDriver); + ThreadFactory threadFactory = runnable -> { + Thread thread = new Thread(() -> { + running.countDown(); + try { + releaseDriver.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } + runnable.run(); + }, "test-started-recovery-driver") { + @Override + public synchronized void start() { + super.start(); + try { + if (!running.await(10, TimeUnit.SECONDS)) { + throw new AssertionError("recovery driver did not start"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError("interrupted waiting for recovery driver", e); + } + throw failure; + } + }; + recoveryThread.set(thread); + return thread; + }; + Runnable beforeFailedJoin = () -> { + failedJoinCalls.incrementAndGet(); + releaseDriver.countDown(); + }; + + try { + newPoolWithRecoveryControls( + "ws::addr=localhost:1;sf_dir=" + sfDir + ";", + 2, 2, 0, senderFactory, threadFactory, null, beforeFailedJoin); + Assert.fail("construction must propagate the recovery thread start failure"); + } catch (AssertionError actual) { + Assert.assertSame("construction must preserve throwable identity", failure, actual); + } finally { + releaseDriver.countDown(); + } + Assert.assertEquals("constructor cleanup must enter the failed-driver join", 1, failedJoinCalls.get()); + Assert.assertFalse("delegate cleanup must not begin while the failed driver is alive", + cleanupBeforeDriverQuiescence.get()); + Assert.assertFalse("a possibly-started failed driver must terminate before delegate cleanup", + recoveryThread.get().isAlive()); + Assert.assertEquals("cleanup must close the first prewarmed delegate", 1, closeCalls[0].get()); + Assert.assertEquals("cleanup must close the second prewarmed delegate", 1, closeCalls[1].get()); + } + + @Test + public void testDirectRecoveryRetriesTransientFailureAndRemainingSlots() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + for (int i = 0; i < 2; i++) { + String seedConfig = "ws::addr=localhost:" + silent.getPort() + ";sf_dir=" + sfDir + + ";sender_id=default-" + i + ";close_flush_timeout_millis=0;"; + try (Sender seed = Sender.fromConfig(seedConfig)) { + seed.table("recover").longColumn("v", i).atNow(); + seed.flush(); + } + } + } + Assert.assertTrue("in-range fixture must contain unacked data", + hasSegmentFile(slot("default-0"))); + Assert.assertTrue("out-of-range fixture must contain unacked data", + hasSegmentFile(slot("default-1"))); + CountingAckHandler handler = new CountingAckHandler(); try (TestWebSocketServer ack = new TestWebSocketServer(handler)) { - int ackPort = ack.getPort(); ack.start(); Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS)); - String cfg = "ws::addr=localhost:" + ackPort + ";sf_dir=" + sfDir + ";"; - try (SenderPool pool = new SenderPool(cfg, 0, 2, 5_000, Long.MAX_VALUE, Long.MAX_VALUE)) { - // In-range [0,2): startup recovery pass 1 drains these. - Assert.assertTrue("in-range default-0 must be recovered at startup", - awaitNoSegmentFile(slot("default-0"), 15_000)); - Assert.assertTrue("in-range default-1 must be recovered at startup", - awaitNoSegmentFile(slot("default-1"), 15_000)); - - // Out-of-range [2,4): pass 2 must deliver these too, under - // the default config (no drain_orphans). - Assert.assertTrue("default-2 unacked data must be recovered, not stranded", - awaitNoSegmentFile(slot("default-2"), 15_000)); - Assert.assertTrue("default-3 unacked data must be recovered, not stranded", - awaitNoSegmentFile(slot("default-3"), 15_000)); - Assert.assertFalse("out-of-range slot must not be abandoned as .failed", - Files.exists(slot("default-2") + "/" + OrphanScanner.FAILED_SENTINEL_NAME)); - Assert.assertFalse("out-of-range slot must not be abandoned as .failed", - Files.exists(slot("default-3") + "/" + OrphanScanner.FAILED_SENTINEL_NAME)); + String config = "ws::addr=localhost:" + ack.getPort() + ";sf_dir=" + sfDir + ";"; + AtomicInteger[] attempts = {new AtomicInteger(), new AtomicInteger()}; + CountDownLatch drained = new CountDownLatch(2); + IntFunction factory = idx -> { + int attempt = attempts[idx].incrementAndGet(); + if (idx == 0 && attempt == 1) { + throw new LineSenderException("transient direct recovery failure"); + } + Sender delegate = Sender.builder(config).senderId("default-" + idx).build(); + return notifyingCloseSender(delegate, drained); + }; - // Sanity: the pool is still usable for normal borrows. - PooledSender a = pool.borrow(); - a.close(); + try (SenderPool pool = newPoolWithFactory(config, 0, 1, 5_000, factory)) { + Assert.assertTrue("both recovery delegates must drain and close", + drained.await(15, TimeUnit.SECONDS)); + Assert.assertFalse("failed in-range candidate must be retried and delivered", + hasSegmentFile(slot("default-0"))); + Assert.assertFalse("remaining out-of-range candidate must also be delivered", + hasSegmentFile(slot("default-1"))); + Assert.assertEquals("failed in-range candidate must be retried", 2, attempts[0].get()); + Assert.assertEquals("remaining out-of-range candidate must also be recovered", + 1, attempts[1].get()); + Assert.assertTrue("both recovered frames must reach the server", + handler.frames.get() >= 2); } } }); } @Test - public void testInRangeIdleSlotIsRecoveredAtStartupUnderSteadyLowLoad() throws Exception { - // The drain exclusion is bounded to [0, maxSize) so a sibling's drainer - // never adopts a slot dir the pool intends to (re)create -- that is what - // prevents "sf slot already in use" (see - // testDrainOrphansPoolDoesNotCannibalizeSiblingSlots). The trade-off was - // that an in-range slot left holding unacked data by a previous run was - // recovered ONLY when the pool happened to (re)create that index: the - // pool pre-warms [0, minSize) and builds [minSize, maxSize) lazily on - // demand, so under steady low load a high in-range index was never - // rebuilt -- neither drained (excluded) nor recovered -- and its data - // was stranded on disk until a restart or load spike. - // - // The fix has the pool recover its own stranded managed slots once, at - // construction, under its own slot reservation (so the cannibalization - // race the exclusion guards against still cannot happen). This test - // seeds a busy run, then restarts under steady low load and asserts the - // idle in-range slots are recovered anyway. + public void testDirectRecoveryContendedSlotDoesNotStarveRemainingSlots() throws Exception { + // C2 regression, end to end: a slot whose createRecoverer PERSISTENTLY + // throws SlotLockContentionException (its flock is held by another live + // owner, e.g. a sibling process sharing the slot dir) must not pin the + // startup-recovery cursor: pre-fix the driver retried that one slot + // every second forever and the higher-index slot's durable orphan data + // was never forwarded under idle load. The transient-failure twin + // (testDirectRecoveryRetriesTransientFailureAndRemainingSlots) covers a + // failure that CLEARS; this covers one that never does. TestUtils.assertMemoryLeak(() -> { - // Phase 1: a busy run at maxSize=4 seeds unacked data into - // default-0..3 (silent server never acks; close_flush_timeout=0 so - // close() leaves the flushed-but-unacked .sfa on disk). try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { - int silentPort = silent.getPort(); silent.start(); Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); - String seedCfg = "ws::addr=localhost:" + silentPort + ";sf_dir=" + sfDir - + ";close_flush_timeout_millis=0;"; - try (SenderPool seed = new SenderPool(seedCfg, 4, 4, 5_000, Long.MAX_VALUE, Long.MAX_VALUE)) { - PooledSender[] s = new PooledSender[4]; - for (int i = 0; i < 4; i++) { - s[i] = seed.borrow(); - } - for (int i = 0; i < 4; i++) { - s[i].table("recover").longColumn("v", i).atNow(); - s[i].flush(); - } - for (int i = 3; i >= 0; i--) { - s[i].close(); + for (int i = 0; i < 2; i++) { + String seedConfig = "ws::addr=localhost:" + silent.getPort() + ";sf_dir=" + sfDir + + ";sender_id=default-" + i + ";close_flush_timeout_millis=0;"; + try (Sender seed = Sender.fromConfig(seedConfig)) { + seed.table("recover").longColumn("v", i).atNow(); + seed.flush(); } } } - for (int i = 0; i < 4; i++) { - Assert.assertTrue("default-" + i + " must hold unacked data", - hasSegmentFile(slot("default-" + i))); - } + Assert.assertTrue("in-range fixture must contain unacked data", + hasSegmentFile(slot("default-0"))); + Assert.assertTrue("out-of-range fixture must contain unacked data", + hasSegmentFile(slot("default-1"))); - // Phase 2: restart at the SAME maxSize=4 (so default-0..3 stay in - // range) with steady low load. minSize=0 means prewarm builds - // nothing and the lowest-free allocator would never reach the high - // indices under a single in-flight borrow -- yet startup recovery - // must still empty every in-range slot. CountingAckHandler handler = new CountingAckHandler(); try (TestWebSocketServer ack = new TestWebSocketServer(handler)) { - int ackPort = ack.getPort(); ack.start(); Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS)); - String cfg = "ws::addr=localhost:" + ackPort + ";sf_dir=" + sfDir + ";"; - try (SenderPool pool = new SenderPool(cfg, 0, 4, 5_000, Long.MAX_VALUE, Long.MAX_VALUE)) { - // All four in-range slots must be recovered by the startup - // pass, even though steady low load never grows the pool to - // their indices. - for (int i = 0; i < 4; i++) { - Assert.assertTrue("in-range idle default-" + i - + " must be recovered at startup, not stranded", - awaitNoSegmentFile(slot("default-" + i), 15_000)); - Assert.assertFalse("recovered slot must not be flagged .failed", - Files.exists(slot("default-" + i) + "/" + OrphanScanner.FAILED_SENTINEL_NAME)); + String config = "ws::addr=localhost:" + ack.getPort() + ";sf_dir=" + sfDir + ";"; + AtomicInteger[] attempts = {new AtomicInteger(), new AtomicInteger()}; + CountDownLatch drained = new CountDownLatch(1); + IntFunction factory = idx -> { + attempts[idx].incrementAndGet(); + if (idx == 0) { + // Persistent: never clears for the life of the pool. + throw new SlotLockContentionException( + "sf slot already in use by another process [slot=" + + slot("default-0") + ", holder=pid=test]"); } - // The recovered frames must have actually reached the server. - Assert.assertTrue("recovered frames must be replayed to the server", - awaitAtLeast(handler.frames, 4, 15_000)); + Sender delegate = Sender.builder(config).senderId("default-" + idx).build(); + return notifyingCloseSender(delegate, drained); + }; - // Sanity: the pool is still usable for normal borrows. - PooledSender a = pool.borrow(); - a.close(); + try (SenderPool pool = newPoolWithFactory(config, 0, 1, 5_000, factory)) { + Assert.assertTrue( + "a persistently contended slot 0 must not starve slot 1's recovery", + drained.await(15, TimeUnit.SECONDS)); + Assert.assertFalse("higher-index slot's orphan data must be delivered", + hasSegmentFile(slot("default-1"))); + Assert.assertEquals("recovered slot must be drained exactly once", + 1, attempts[1].get()); + Assert.assertTrue("contended slot must have been probed", attempts[0].get() >= 1); + Assert.assertTrue("contended slot's durable data must be preserved on disk", + hasSegmentFile(slot("default-0"))); + Assert.assertTrue("recovered frame must reach the server", handler.frames.get() >= 1); + Assert.assertFalse("recovery must not report complete while the contended slot holds data", + pool.isRecoveryCompleteForTesting()); } } }); } + @Test + public void testStartupRecoveryParksContendedSlotAndContinuesScan() throws Exception { + // C2, white-box and fully step-driven (no live driver, no wall clock): + // one recovery step must park a contended in-range slot 0, continue to + // the out-of-range slot 1 within the SAME step, keep the parked slot + // retryable across scan cycles (never abandoned, never complete), and + // WARN about the contended slot exactly once, not once per retry. + createCandidateSlot("default-0"); + createCandidateSlot("default-1"); + AtomicInteger[] attempts = {new AtomicInteger(), new AtomicInteger()}; + CountDownLatch drained = new CountDownLatch(1); + IntFunction factory = idx -> { + attempts[idx].incrementAndGet(); + if (idx == 0) { + throw new SlotLockContentionException( + "sf slot already in use by another process [slot=" + + slot("default-0") + ", holder=pid=test]"); + } + return successfulRecoverySender(idx, drained); + }; + String config = "ws::addr=localhost:1;sf_dir=" + sfDir + ";"; + + Logger poolLogger = (Logger) LoggerFactory.getLogger(SenderPool.class); + ListAppender appender = new ListAppender<>(); + appender.start(); + Level savedLevel = poolLogger.getLevel(); + poolLogger.setLevel(Level.ALL); + poolLogger.addAppender(appender); + try (SenderPool pool = newDeferredPoolWithFactory(config, 0, 1, 0, factory)) { + // Cycle 1: park the contended slot 0, then drain slot 1 in the + // same step (the park must not consume the step's single drain). + Assert.assertTrue("step must park the contended slot and drain the next candidate", + pool.runStartupRecoveryStepForTesting()); + Assert.assertEquals(1, attempts[0].get()); + Assert.assertEquals("higher-index slot must be recovered despite the parked slot", + 1, attempts[1].get()); + Assert.assertEquals(0, drained.getCount()); + Assert.assertFalse("cycle with a parked slot must defer, not complete", + pool.runStartupRecoveryStepForTesting()); + Assert.assertFalse("recovery must not report complete while the contended slot holds data", + pool.isRecoveryCompleteForTesting()); + + // Cycle 2: the parked slot is re-probed (retryable, not abandoned). + Assert.assertTrue(pool.runStartupRecoveryStepForTesting()); + Assert.assertEquals("parked slot must be re-probed on the next cycle", + 2, attempts[0].get()); + Assert.assertFalse(pool.runStartupRecoveryStepForTesting()); + + // Bounded logging: the contended slot warned once, not per retry. + long contentionWarns = appender.list.stream().filter(e -> + e.getLevel().isGreaterOrEqual(Level.WARN) + && e.getFormattedMessage().contains("default-0")).count(); + Assert.assertEquals("contended slot must WARN once per episode, not per retry; captured=" + + appender.list, 1, contentionWarns); + } finally { + poolLogger.detachAppender(appender); + poolLogger.setLevel(savedLevel); + appender.stop(); + } + } + + @Test + public void testStartupRecoveryParksPersistentlyFailingSlotAfterBoundedRetries() throws Exception { + // C2, generic-failure flavor, step-driven: a slot whose recovery build + // persistently fails with a NON-contention error keeps the existing + // retry-in-place behavior for the first attempts (the server-wide + // transient heuristic) but must be parked after a bounded streak so it + // cannot starve the higher-index in-range slot, and must be re-probed + // on the next scan cycle rather than abandoned. + createCandidateSlot("default-0"); + createCandidateSlot("default-1"); + AtomicInteger[] attempts = {new AtomicInteger(), new AtomicInteger()}; + CountDownLatch drained = new CountDownLatch(1); + IntFunction factory = idx -> { + attempts[idx].incrementAndGet(); + if (idx == 0) { + throw new LineSenderException("persistent per-slot recovery failure"); + } + return successfulRecoverySender(idx, drained); + }; + String config = "ws::addr=localhost:1;sf_dir=" + sfDir + ";"; + + try (SenderPool pool = newDeferredPoolWithFactory(config, 0, 2, 0, factory)) { + // Attempts 1 and 2: presumed transient, same candidate retried. + Assert.assertFalse("first failure must defer the same candidate", + pool.runStartupRecoveryStepForTesting()); + Assert.assertFalse("second failure must defer the same candidate", + pool.runStartupRecoveryStepForTesting()); + Assert.assertEquals(2, attempts[0].get()); + Assert.assertEquals("failing slot must not have blocked past its streak yet", + 0, attempts[1].get()); + + // Attempt 3 exhausts the streak: the slot is parked and the scan + // may continue to the next candidate. + Assert.assertTrue("streak exhaustion must park the slot and continue the scan", + pool.runStartupRecoveryStepForTesting()); + Assert.assertEquals(3, attempts[0].get()); + Assert.assertTrue("higher-index slot must be recovered despite the parked slot", + pool.runStartupRecoveryStepForTesting()); + Assert.assertEquals("higher-index slot must be drained exactly once", + 1, attempts[1].get()); + Assert.assertEquals(0, drained.getCount()); + + // End of cycle: parked slot outstanding -> defer, not complete. + Assert.assertFalse("cycle with a parked slot must defer, not complete", + pool.runStartupRecoveryStepForTesting()); + Assert.assertFalse("recovery must not report complete while the parked slot holds data", + pool.isRecoveryCompleteForTesting()); + + // Next cycle: the parked slot is re-probed with a fresh streak. + Assert.assertFalse("re-probed slot restarts its bounded retry streak", + pool.runStartupRecoveryStepForTesting()); + Assert.assertEquals(4, attempts[0].get()); + } + } + + @Test + public void testLongMaxStartupRecoveryBudgetDoesNotOverflow() throws Exception { + createCandidateSlot("default-0"); + AtomicInteger attempts = new AtomicInteger(); + IntFunction factory = idx -> { + attempts.incrementAndGet(); + return successfulRecoverySender(idx, new CountDownLatch(0)); + }; + String config = "ws::addr=localhost:1;sf_dir=" + sfDir + ";"; + + try (SenderPool pool = newPoolWithFactory(config, 0, 1, Long.MAX_VALUE, factory)) { + Assert.assertEquals("Long.MAX_VALUE must leave a positive inline recovery budget", + 1, attempts.get()); + Assert.assertTrue("the inline scan must complete", pool.isRecoveryCompleteForTesting()); + } + } + @Test public void testStartupRecoveryIsBoundedByASharedBudget() throws Exception { // Regression for the startup-recovery budget (M1). @@ -1541,7 +3725,7 @@ public void testRecoveryStepStaysBoundedWithDrainOrphansAgainstNonAckingServer() SenderPool pool = newDeferredPool(cfg, 0, maxSize, acquireTimeoutMillis); try { long startNanos = System.nanoTime(); - invokeRunStartupRecoveryStep(pool); + pool.runStartupRecoveryStepForTesting(); long stepMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos); // Headline guarantee: a recovery delegate must not stand up a @@ -1644,7 +3828,7 @@ public void testDeferredStartupRecoveryDoesNotBlockConstruction() throws Excepti // lost) for a later attempt -- exercising the deferred path's // concurrency-safe slot reservation too. long recoverStart = System.nanoTime(); - invokeRunStartupRecoveryOnce(pool); + pool.runStartupRecoveryToCompletionForTesting(); long recoverMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - recoverStart); Assert.assertTrue( "driven recovery must be bounded by the shared budget: took " + recoverMillis @@ -1699,7 +3883,7 @@ public void testDeferredStartupRecoveryDeliversWhenDriven() throws Exception { hasSegmentFile(slot("default-0"))); // Drive it (what the housekeeper does on its first tick). - invokeRunStartupRecoveryOnce(pool); + pool.runStartupRecoveryToCompletionForTesting(); Assert.assertTrue("driven recovery must empty default-0", awaitNoSegmentFile(slot("default-0"), 15_000)); Assert.assertTrue("recovered frames must reach the server", @@ -1708,7 +3892,7 @@ public void testDeferredStartupRecoveryDeliversWhenDriven() throws Exception { Files.exists(slot("default-0") + "/" + OrphanScanner.FAILED_SENTINEL_NAME)); // Idempotent: a second drive is a no-op and must not throw. - invokeRunStartupRecoveryOnce(pool); + pool.runStartupRecoveryToCompletionForTesting(); Assert.assertFalse("default-0 stays recovered", hasSegmentFile(slot("default-0"))); // Pool still usable for normal borrows. @@ -1769,7 +3953,7 @@ public void testCloseDuringDeferredRecoveryStopsBuildingOnClosingPool() throws E // Mimic the housekeeper: drive steps back-to-back until done/closing. Thread recovery = new Thread(() -> { try { - while (invokeRunStartupRecoveryStep(pool)) { + while (pool.runStartupRecoveryStepForTesting()) { // keep stepping } } catch (Exception ignored) { @@ -1782,7 +3966,7 @@ public void testCloseDuringDeferredRecoveryStopsBuildingOnClosingPool() throws E // Raise the shutdown signal mid-drain, exactly as QuestDBImpl.close() // does before stopping the housekeeper. - invokeMarkClosing(pool); + pool.markClosingForTesting(); releaseSlot0Drain.countDown(); recovery.join(TimeUnit.SECONDS.toMillis(10)); Assert.assertFalse("recovery thread must finish", recovery.isAlive()); @@ -1802,10 +3986,225 @@ public void testCloseDuringDeferredRecoveryStopsBuildingOnClosingPool() throws E // Helpers. // ---------------------------------------------------------------------- + private static Sender closeCountingSender( + int idx, AtomicInteger[] closeCalls, boolean throwOnClose + ) { + return closeCountingSender(idx, closeCalls, throwOnClose, null, null, null); + } + + private static Sender closeCountingSender( + int idx, + AtomicInteger[] closeCalls, + boolean throwOnClose, + AtomicReference recoveryThread, + AtomicBoolean cleanupBeforeDriverQuiescence, + CountDownLatch releaseDriver + ) { + return (Sender) Proxy.newProxyInstance( + Sender.class.getClassLoader(), + new Class[]{Sender.class}, + (proxy, method, args) -> { + switch (method.getName()) { + case "close": + if (recoveryThread != null && recoveryThread.get().isAlive()) { + cleanupBeforeDriverQuiescence.set(true); + releaseDriver.countDown(); + } + closeCalls[idx].incrementAndGet(); + if (throwOnClose) { + throw new AssertionError("injected delegate close failure"); + } + return null; + case "toString": + return "CloseCountingSender-" + idx; + case "hashCode": + return System.identityHashCode(proxy); + case "equals": + return proxy == args[0]; + default: + throw new AssertionError("unexpected prewarmed sender call: " + method.getName()); + } + }); + } + + private void createCandidateSlot(String name) throws IOException { + java.nio.file.Path dir = Paths.get(slot(name)); + java.nio.file.Files.createDirectories(dir); + java.nio.file.Files.write(dir.resolve("0.sfa"), new byte[]{1}); + } + + private static Sender notifyingCloseSender(Sender delegate, CountDownLatch closed) { + return (Sender) Proxy.newProxyInstance( + Sender.class.getClassLoader(), + new Class[]{Sender.class}, + (proxy, method, args) -> { + try { + Object result = method.invoke(delegate, args); + if ("close".equals(method.getName())) { + closed.countDown(); + } + return result; + } catch (InvocationTargetException e) { + throw e.getCause(); + } + }); + } + private String slot(String name) { return sfDir + "/" + name; } + private static Sender successfulRecoverySender(int idx, CountDownLatch drained) { + return (Sender) Proxy.newProxyInstance( + Sender.class.getClassLoader(), + new Class[]{Sender.class}, + (proxy, method, args) -> { + switch (method.getName()) { + case "drain": + drained.countDown(); + return true; + case "close": + return null; + case "toString": + return "SuccessfulRecoverySender-" + idx; + case "hashCode": + return System.identityHashCode(proxy); + case "equals": + return proxy == args[0]; + default: + throw new AssertionError("unexpected recovery sender call: " + method.getName()); + } + }); + } + + private void assertPreallocatedExitHandoffCleansStartupRecoverer( + int strandedIndex, int maxSize) throws Exception { + TestUtils.assertMemoryLeak(() -> { + String strandedId = "default-" + strandedIndex; + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String config = "ws::addr=localhost:" + silent.getPort() + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=100;"; + Sender sender = Sender.builder(config).senderId(strandedId).build(); + sender.table("recover").longColumn("v", strandedIndex).atNow(); + sender.flush(); + try { + sender.close(); + } catch (LineSenderException expected) { + Assert.assertTrue(expected.getMessage(), + expected.getMessage().contains("drain timed out")); + } + } + Assert.assertTrue("startup-recovery fixture must contain an unacked segment", + hasSegmentFile(slot(strandedId))); + + CountingAckHandler handler = new CountingAckHandler(); + try (TestWebSocketServer ack = new TestWebSocketServer(handler)) { + ack.start(); + Assert.assertTrue(ack.awaitStart(5, TimeUnit.SECONDS)); + String config = "ws::addr=localhost:" + ack.getPort() + ";sf_dir=" + sfDir + ";"; + AtomicBoolean instrumented = new AtomicBoolean(); + AtomicReference engineRef = new AtomicReference<>(); + AtomicReference managerRef = new AtomicReference<>(); + AtomicReference hookErr = new AtomicReference<>(); + CountDownLatch releaseWorker = new CountDownLatch(1); + IntFunction factory = idx -> { + Sender sender = Sender.builder(config).senderId("default-" + idx).build(); + if (idx == strandedIndex && instrumented.compareAndSet(false, true)) { + try { + CursorSendEngine engine = ((QwpWebSocketSender) sender).getCursorEngineForTesting(); + SegmentManager manager = engine.getManagerForTesting(); + CountDownLatch workerBlocked = new CountDownLatch(1); + AtomicBoolean fired = new AtomicBoolean(); + manager.setBeforeTrimSyncHook(() -> { + if (!fired.compareAndSet(false, true)) { + return; + } + workerBlocked.countDown(); + try { + if (!releaseWorker.await(20, TimeUnit.SECONDS)) { + hookErr.compareAndSet(null, + new AssertionError("timed out waiting to release recovery worker")); + } + } catch (Throwable t) { + hookErr.compareAndSet(null, t); + } + }); + if (!workerBlocked.await(5, TimeUnit.SECONDS)) { + throw new AssertionError("recovery manager never entered a service pass"); + } + manager.setBeforeExitCleanupRegistrationHook(() -> { + throw new OutOfMemoryError("simulated callback allocation failure"); + }); + manager.setWorkerJoinTimeoutMillis(50L); + engineRef.set(engine); + managerRef.set(manager); + } catch (Throwable t) { + try { + sender.close(); + } catch (Throwable ignored) { + } + throw new RuntimeException(t); + } + } + return sender; + }; + + SenderPool pool = null; + try { + pool = newPoolWithFactory(config, 0, maxSize, 2_000, factory); + CursorSendEngine engine = engineRef.get(); + SegmentManager manager = managerRef.get(); + Assert.assertNotNull("startup recovery must build the stranded slot", engine); + if (strandedIndex < maxSize) { + Assert.assertEquals("in-range recoverer must remain retired while worker is live", + 1, pool.leakedSlotCount()); + } else { + Assert.assertEquals("out-of-range recovery must not consume pool capacity", + 0, pool.leakedSlotCount()); + } + Assert.assertFalse("cleanup must remain pending while the worker is live", + engine.isCloseCompleted()); + + releaseWorker.countDown(); + manager.close(); + Assert.assertTrue("recovery manager worker must be reaped", manager.isWorkerReaped()); + Assert.assertTrue("worker exit must complete startup-recoverer cleanup without " + + "a sender or engine close retry [index=" + strandedIndex + "]", + engine.isCloseCompleted()); + if (hookErr.get() != null) { + throw new AssertionError("recovery worker hook failed", hookErr.get()); + } + if (strandedIndex < maxSize) { + pool.reapIdle(); + Assert.assertEquals("late cleanup must restore in-range pool capacity", + 0, pool.leakedSlotCount()); + } + try (SlotLock ignored = SlotLock.acquire(slot(strandedId))) { + // Completion must mean the real slot flock is reusable. + } + } finally { + releaseWorker.countDown(); + SegmentManager manager = managerRef.get(); + if (manager != null) { + manager.setBeforeExitCleanupRegistrationHook(null); + manager.setBeforeTrimSyncHook(null); + manager.setWorkerJoinTimeoutMillis(TimeUnit.SECONDS.toMillis(60)); + manager.close(); + } + CursorSendEngine engine = engineRef.get(); + if (engine != null && !engine.isCloseCompleted()) { + engine.close(); + } + if (pool != null) { + pool.close(); + } + } + } + }); + } + private int countSlotDirs() { if (!Files.exists(sfDir)) { return 0; @@ -1869,6 +4268,86 @@ private static boolean awaitAtLeast(AtomicInteger counter, int target, long time return counter.get() >= target; } + @Test + public void testContendedSlotReprobeUsesFlockProbeNotFullBuild() throws Exception { + // A slot whose flock is held by another LIVE owner is parked and + // re-probed on every retry cycle -- potentially for the owner's whole + // lifetime. The re-probe must ask the flock directly (O(1) probe, + // a few syscalls), not pay a full recovery build (config re-parse, + // builder graph, parent-dir fsync barriers in periodic durability, + // owned SegmentManager allocation) per cycle just to reach + // SlotLock.acquire and throw. + TestUtils.assertMemoryLeak(() -> { + String config = "ws::addr=localhost:1;sf_dir=" + sfDir + ";"; + String slot0 = slot("default-0"); + + // Seed one unacked frame so slot 0 is a candidate orphan. The + // group root normally comes from Sender.build(); this test seeds + // the slot directly, so create the parent first (mkdir in + // SlotLock.acquire is non-recursive). + Assert.assertEquals(0, Files.mkdir(sfDir, Files.DIR_MODE_DEFAULT)); + long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + try (CursorSendEngine seed = new CursorSendEngine(slot0, 1 << 20)) { + Unsafe.getUnsafe().setMemory(buf, 16, (byte) 1); + Assert.assertEquals(0L, seed.appendBlocking(buf, 16)); + } finally { + Unsafe.free(buf, 16, MemoryTag.NATIVE_DEFAULT); + } + Assert.assertTrue("seeded slot must be a candidate orphan", + OrphanScanner.isCandidateOrphan(slot0)); + + AtomicInteger builds = new AtomicInteger(); + IntFunction factory = idx -> { + builds.incrementAndGet(); + // Fidelity with the production build: the recovery build's + // fate is decided by the real slot flock, exactly like + // defaultRecoverySender's engine construction. + SlotLock lock = SlotLock.acquire(slot("default-" + idx)); + return (Sender) Proxy.newProxyInstance( + Sender.class.getClassLoader(), + new Class[]{Sender.class}, + (proxy, method, args) -> { + if ("drain".equals(method.getName())) { + return Boolean.TRUE; + } + if ("close".equals(method.getName())) { + lock.close(); + return null; + } + throw new AssertionError( + "unexpected recovery sender call: " + method.getName()); + }); + }; + + SlotLock held = SlotLock.acquire(slot0); + try (SenderPool pool = newDeferredPoolWithFactory(config, 0, 1, 5_000, factory)) { + // Steady-state re-probe of a parked contended slot: three + // full cycles while the flock is held. + for (int cycle = 0; cycle < 3; cycle++) { + Assert.assertFalse("a cycle with only a contended slot must defer", + pool.runStartupRecoveryStepForTesting()); + } + Assert.assertEquals("re-probing a contended slot must be a flock probe, " + + "not a full recovery build per cycle", 0, builds.get()); + Assert.assertTrue("parked slot must keep its durable data", + hasSegmentFile(slot0)); + + // The probe must not dampen recovery: once the owner lets + // go, the very next cycle pays exactly one real build and + // drains the slot. + held.close(); + Assert.assertTrue("released slot must be recovered on the next cycle", + pool.runStartupRecoveryStepForTesting()); + Assert.assertEquals("released slot must be recovered with exactly one build", + 1, builds.get()); + Assert.assertFalse("final scan step must mark recovery complete", + pool.runStartupRecoveryStepForTesting()); + } finally { + held.close(); // idempotent when already released + } + }); + } + private static boolean awaitNoSegmentFile(String slotPath, long timeoutMillis) throws InterruptedException { long deadline = System.currentTimeMillis() + timeoutMillis; @@ -1906,66 +4385,8 @@ private static void rmDir(String dir) { Files.remove(dir); } - private static Sender getDelegate(PooledSender ps) throws Exception { - Field slotF = PooledSender.class.getDeclaredField("slot"); - slotF.setAccessible(true); - Object slot = slotF.get(ps); - Field f = slot.getClass().getDeclaredField("delegate"); - f.setAccessible(true); - return (Sender) f.get(slot); - } - - // Invokes one of the pool's private managed-slot delegate factories - // (defaultSender / defaultRecoverySender) so a test can inspect the raw - // delegate it would build for a given slot index. - private static Sender invokeBuildSlotDelegate(SenderPool pool, String methodName, int slotIndex) - throws Exception { - Method m = SenderPool.class.getDeclaredMethod(methodName, int.class); - m.setAccessible(true); - return (Sender) m.invoke(pool, slotIndex); - } - - // Reads the resolved initial-connect mode a built QwpWebSocketSender delegate - // is using (the value after the builder's SYNC auto-promotion / explicit - // override has been applied). - private static Sender.InitialConnectMode readInitialConnectMode(Sender delegate) throws Exception { - Field f = delegate.getClass().getDeclaredField("initialConnectMode"); - f.setAccessible(true); - return (Sender.InitialConnectMode) f.get(delegate); - } - - private static void setBooleanField(Object target, String name, boolean value) throws Exception { - Field f = target.getClass().getDeclaredField(name); - f.setAccessible(true); - f.setBoolean(target, value); - } - - private static int getIntField(Object target, String name) throws Exception { - Field f = target.getClass().getDeclaredField(name); - f.setAccessible(true); - return f.getInt(target); - } - - private static Object getField(Object target, String name) throws Exception { - Field f = target.getClass().getDeclaredField(name); - f.setAccessible(true); - return f.get(target); - } - - // Reads the package-private PooledSender.slot -- the identity that the pool - // actually recycles. Wrapper identity is useless for aliasing checks because - // borrow() allocates a fresh wrapper every call (mirrors SenderPoolTest and - // SenderPoolErrorSafetyTest). - private static Object slotOf(PooledSender pooledWrapper) throws Exception { - Field f = PooledSender.class.getDeclaredField("slot"); - f.setAccessible(true); - return f.get(pooledWrapper); - } - - private static void invokeDiscardBroken(SenderPool pool, PooledSender ps) throws Exception { - Method m = SenderPool.class.getDeclaredMethod("discardBroken", PooledSender.class); - m.setAccessible(true); - m.invoke(pool, ps); + private static Sender getDelegate(PooledSender ps) { + return ps.getDelegateForTesting(); } // Uses the @TestOnly senderFactory seam so a test can inject a fake/forged @@ -1976,6 +4397,33 @@ private static SenderPool newPoolWithFactory( return new SenderPool(cfg, min, max, acquireMs, Long.MAX_VALUE, Long.MAX_VALUE, senderFactory); } + private static SenderPool newPoolWithRecoveryControls( + String cfg, + int min, + int max, + long acquireMs, + IntFunction senderFactory, + ThreadFactory threadFactory, + Runnable recoveryWaiter, + Runnable beforeFailedRecoveryJoinHook + ) { + return SenderPool.createWithRecoveryControlsForTesting( + cfg, min, max, acquireMs, senderFactory, threadFactory, + recoveryWaiter, beforeFailedRecoveryJoinHook); + } + + private static SenderPool newPoolWithRecoveryThreadFactory( + String cfg, + int min, + int max, + long acquireMs, + IntFunction senderFactory, + ThreadFactory threadFactory + ) throws Throwable { + return newPoolWithRecoveryControls( + cfg, min, max, acquireMs, senderFactory, threadFactory, null, null); + } + // Uses the @TestOnly 8-arg constructor (deferStartupRecovery=true) so a test // can build a pool whose SF startup recovery is NOT run inline -- mirroring // the pooled QuestDB handle, which defers it to the housekeeper. @@ -1984,30 +4432,6 @@ private static SenderPool newDeferredPool(String cfg, int min, int max, long acq return new SenderPool(cfg, min, max, acquireMs, Long.MAX_VALUE, Long.MAX_VALUE, null, true); } - // Drives a deferred pool's startup recovery to completion (the housekeeper - // drives it one slot per tick; tests drive the whole backlog in one call). - private static void invokeRunStartupRecoveryOnce(SenderPool pool) throws Exception { - Method m = SenderPool.class.getDeclaredMethod("runStartupRecoveryToCompletion"); - m.setAccessible(true); - m.invoke(pool); - } - - // Drives a SINGLE recovery step (the housekeeper's per-tick unit); returns - // whether more stranded slots remain. - private static boolean invokeRunStartupRecoveryStep(SenderPool pool) throws Exception { - Method m = SenderPool.class.getDeclaredMethod("runStartupRecoveryStep"); - m.setAccessible(true); - return (Boolean) m.invoke(pool); - } - - // Raises the pool's shutdown signal early, exactly as QuestDBImpl.close() - // does before stopping the housekeeper. - private static void invokeMarkClosing(SenderPool pool) throws Exception { - Method m = SenderPool.class.getDeclaredMethod("markClosing"); - m.setAccessible(true); - m.invoke(pool); - } - // Deferred pool (deferStartupRecovery=true) WITH an injected factory, so a // test can drive the housekeeper recovery path against fully controlled // (fake) recoverers. @@ -2016,9 +4440,23 @@ private static SenderPool newDeferredPoolWithFactory( return new SenderPool(cfg, min, max, acquireMs, Long.MAX_VALUE, Long.MAX_VALUE, factory, true); } - // Fake Sender whose drain() (for slot 0 only) parks until released, opening a - // deterministic shutdown-during-recovery window. Holds no native resources. - private static Sender blockingFakeSender(int idx, CountDownLatch drainStarted, CountDownLatch release) { + private static Sender blockingFakeSender( + int idx, CountDownLatch drainStarted, CountDownLatch releaseDrain + ) { + return blockingFakeSender( + idx, drainStarted, releaseDrain, new CountDownLatch(0), new CountDownLatch(0)); + } + + // Fake Sender whose drain() and close() (for slot 0 only) park until + // released, opening deterministic shutdown and pre-termination windows. + // Holds no native resources. + private static Sender blockingFakeSender( + int idx, + CountDownLatch drainStarted, + CountDownLatch releaseDrain, + CountDownLatch closeStarted, + CountDownLatch releaseClose + ) { return (Sender) java.lang.reflect.Proxy.newProxyInstance( Sender.class.getClassLoader(), new Class[]{Sender.class}, @@ -2027,10 +4465,14 @@ private static Sender blockingFakeSender(int idx, CountDownLatch drainStarted, C case "drain": if (idx == 0) { drainStarted.countDown(); - release.await(); + releaseDrain.await(); } return true; case "close": + if (idx == 0) { + closeStarted.countDown(); + releaseClose.await(); + } return null; case "toString": return "BlockingFakeSender-" + idx; diff --git a/core/src/test/java/io/questdb/client/test/impl/WsSenderConfigHonoredTest.java b/core/src/test/java/io/questdb/client/test/impl/WsSenderConfigHonoredTest.java index dba5765f..7fe602d3 100644 --- a/core/src/test/java/io/questdb/client/test/impl/WsSenderConfigHonoredTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/WsSenderConfigHonoredTest.java @@ -63,8 +63,9 @@ public void testEveryIngressKeyIsHonored() { assertHonored("sf_dir=/var/probe", "sf_dir", "/var/probe"); assertHonored("sf_max_bytes=4096", "sf_max_bytes", 4096L); assertHonored("sf_max_total_bytes=8192", "sf_max_total_bytes", 8192L); - assertHonored("sf_durability=flush", "sf_durability", "FLUSH"); + assertHonored("sf_durability=periodic", "sf_durability", "PERIODIC"); assertHonored("sf_append_deadline_millis=1500", "sf_append_deadline_millis", 1500L); + assertHonored("sf_sync_interval_millis=5000", "sf_sync_interval_millis", 5000L); assertHonored("close_flush_timeout_millis=2500", "close_flush_timeout_millis", 2500L); assertHonored("durable_ack_keepalive_interval_millis=900", "durable_ack_keepalive_interval_millis", 900L); assertHonored("initial_connect_retry=async", "initial_connect_retry", "ASYNC"); diff --git a/core/src/test/java/io/questdb/client/test/network/SocketTrafficShutdownTest.java b/core/src/test/java/io/questdb/client/test/network/SocketTrafficShutdownTest.java new file mode 100644 index 00000000..67d66b95 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/network/SocketTrafficShutdownTest.java @@ -0,0 +1,599 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.network; + +import io.questdb.client.network.JavaTlsClientSocket; +import io.questdb.client.network.JavaTlsClientSocketFactory; +import io.questdb.client.network.Kqueue; +import io.questdb.client.network.KqueueFacade; +import io.questdb.client.network.KqueueFacadeImpl; +import io.questdb.client.network.NetworkFacade; +import io.questdb.client.network.NetworkFacadeImpl; +import io.questdb.client.network.PlainSocket; +import io.questdb.client.network.Socket; +import io.questdb.client.network.SocketReadinessWaiter; +import io.questdb.client.network.TlsSessionInitFailedException; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Os; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; +import org.junit.Assume; +import org.junit.Test; +import org.slf4j.LoggerFactory; + +import javax.net.ssl.SSLContext; +import java.lang.reflect.Proxy; +import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +public class SocketTrafficShutdownTest { + private static final NetworkFacade NF = NetworkFacadeImpl.INSTANCE; + + private static class CompatibilityNetworkFacade implements NetworkFacade { + private final AtomicInteger closeCount; + + private CompatibilityNetworkFacade(AtomicInteger closeCount) { + this.closeCount = closeCount; + } + + @Override + public int close(int fd) { + closeCount.incrementAndGet(); + return 0; + } + + @Override + public void close(int fd, org.slf4j.Logger logger) { + close(fd); + } + + @Override + public void configureKeepAlive(int fd) { + } + + @Override + public int configureNonBlocking(int fd) { + return 0; + } + + @Override + public int connect(int fd, long pSockaddr) { + return 0; + } + + @Override + public int connectAddrInfo(int fd, long pAddrInfo) { + return 0; + } + + @Override + public int connectAddrInfoTimeout(int fd, long pAddrInfo, int timeoutMillis) { + return 0; + } + + @Override + public int errno() { + return 0; + } + + @Override + public void freeAddrInfo(long pAddrInfo) { + } + + @Override + public void freeSockAddr(long pSockaddr) { + } + + @Override + public long getAddrInfo(CharSequence host, int port) { + return 0; + } + + @Override + public int getSndBuf(int fd) { + return 0; + } + + @Override + public int recvRaw(int fd, long buffer, int bufferLen) { + return 0; + } + + @Override + public int sendRaw(int fd, long buffer, int bufferLen) { + return 0; + } + + @Override + public int sendToRaw(int fd, long lo, int len, long socketAddress) { + return 0; + } + + @Override + public int sendToRawScatter(int fd, long segmentsPtr, int segmentCount, long socketAddress) { + return 0; + } + + @Override + public int setMulticastInterface(int fd, int ipv4Address) { + return 0; + } + + @Override + public int setMulticastTtl(int fd, int ttl) { + return 0; + } + + @Override + public boolean setSndBuf(int fd, int size) { + return false; + } + + @Override + public int setTcpNoDelay(int fd, boolean noDelay) { + return 0; + } + + @Override + public long sockaddr(int address, int port) { + return 0; + } + + @Override + public int socketTcp(boolean blocking) { + return 0; + } + + @Override + public int socketUdp() { + return 0; + } + + @Override + public boolean testConnection(int fd, long buffer, int bufferSize) { + return false; + } + } + + private static class CompatibilitySocket implements Socket { + private final AtomicInteger closeCount; + + private CompatibilitySocket(AtomicInteger closeCount) { + this.closeCount = closeCount; + } + + @Override + public void close() { + closeCount.incrementAndGet(); + } + + @Override + public int getFd() { + return -1; + } + + @Override + public boolean isClosed() { + return false; + } + + @Override + public void of(int fd) { + } + + @Override + public int recv(long bufferPtr, int bufferLen) { + return -1; + } + + @Override + public int send(long bufferPtr, int bufferLen) { + return -1; + } + + @Override + public void startTlsSession(CharSequence peerName, SocketReadinessWaiter waiter) throws TlsSessionInitFailedException { + } + + @Override + public boolean supportsTls() { + return false; + } + + @Override + public int tlsIO(int readinessFlags) { + return 0; + } + + @Override + public boolean wantsTlsWrite() { + return false; + } + } + + @Test + public void testCompatibilityDefaultsDoNotBypassCustomTransportOwnership() { + AtomicInteger facadeCloseCount = new AtomicInteger(); + NetworkFacade customFacade = new CompatibilityNetworkFacade(facadeCloseCount); + + PlainSocket plainSocket = new PlainSocket(customFacade, LoggerFactory.getLogger(SocketTrafficShutdownTest.class)); + plainSocket.of(42); + assertUnsupported(plainSocket::closeTraffic); + Assert.assertEquals("facade compatibility fallback must not release a synthetic descriptor", + 0, facadeCloseCount.get()); + Assert.assertEquals(42, plainSocket.getFd()); + plainSocket.close(); + Assert.assertEquals(1, facadeCloseCount.get()); + + AtomicInteger socketCloseCount = new AtomicInteger(); + Socket customSocket = new CompatibilitySocket(socketCloseCount); + assertUnsupported(customSocket::closeTraffic); + Assert.assertEquals("socket compatibility fallback must not run destructive close", + 0, socketCloseCount.get()); + } + + @Test(timeout = 30_000L) + public void testPlainSocketShutdownAfterPeerDisconnectRetainsFd() throws Exception { + Socket socket = new PlainSocket(NF, LoggerFactory.getLogger(SocketTrafficShutdownTest.class)); + + long buffer = 0; + int fd = -1; + try (ServerSocket listener = new ServerSocket()) { + listener.bind(new InetSocketAddress("127.0.0.1", 0)); + long addrInfo = NF.getAddrInfo("127.0.0.1", listener.getLocalPort()); + Assert.assertNotEquals(-1L, addrInfo); + try { + fd = NF.socketTcp(true); + Assert.assertTrue("could not allocate client socket", fd >= 0); + Assert.assertEquals(0, NF.connectAddrInfo(fd, addrInfo)); + } finally { + NF.freeAddrInfo(addrInfo); + } + + try (java.net.Socket peer = listener.accept()) { + socket.of(fd); + int retainedFd = fd; + fd = -1; + buffer = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT); + + peer.close(); + Assert.assertTrue("client must observe the peer disconnect", socket.recv(buffer, 1) < 0); + + socket.closeTraffic(); + Assert.assertEquals("traffic cancellation must retain fd ownership", retainedFd, socket.getFd()); + Assert.assertFalse("traffic cancellation must not perform full close", socket.isClosed()); + Assert.assertTrue("shutdown must leave the descriptor allocated", NF.getSndBuf(retainedFd) > 0); + + socket.close(); + Assert.assertTrue("full close must release the retained fd", socket.isClosed()); + Assert.assertEquals("released descriptor must reject socket operations", -1, NF.getSndBuf(retainedFd)); + } + } finally { + socket.close(); + if (buffer != 0) { + Unsafe.free(buffer, 1, MemoryTag.NATIVE_DEFAULT); + } + if (fd != -1) { + NF.close(fd); + } + } + } + + @Test + public void testPlainSocketShutdownFailureStillThrows() { + AtomicInteger closeCount = new AtomicInteger(); + NetworkFacade failingFacade = new CompatibilityNetworkFacade(closeCount) { + @Override + public int errno() { + return 1234; + } + + @Override + public int shutdown(int fd) { + Assert.assertEquals(42, fd); + return -1; + } + }; + PlainSocket socket = new PlainSocket(failingFacade, LoggerFactory.getLogger(SocketTrafficShutdownTest.class)); + socket.of(42); + + try { + socket.closeTraffic(); + Assert.fail("expected genuine traffic shutdown failure"); + } catch (IllegalStateException expected) { + Assert.assertEquals("could not shut down socket traffic [fd=42, errno=1234]", expected.getMessage()); + } finally { + socket.close(); + } + + Assert.assertTrue(socket.isClosed()); + Assert.assertEquals("full close must release facade ownership exactly once", 1, closeCount.get()); + } + + @Test(timeout = 30_000L) + public void testPlainSocketShutdownWakesMacOsKqueueAndRetainsFd() throws Exception { + assertShutdownWakesMacOsKqueue(new PlainSocket(NF, LoggerFactory.getLogger(SocketTrafficShutdownTest.class))); + } + + @Test(timeout = 30_000L) + public void testPlainSocketShutdownWakesWindowsRecvAndRetainsFd() throws Exception { + Assume.assumeTrue("real Winsock cancellation coverage runs on Windows", Os.type == Os.WINDOWS); + + Socket socket = new PlainSocket(NF, LoggerFactory.getLogger(SocketTrafficShutdownTest.class)); + AtomicBoolean recvDone = new AtomicBoolean(); + AtomicInteger recvResult = new AtomicInteger(Integer.MIN_VALUE); + AtomicReference recvFailure = new AtomicReference<>(); + CountDownLatch recvStarted = new CountDownLatch(1); + + long buffer = 0; + int fd = -1; + Thread waiter = null; + try (ServerSocket listener = new ServerSocket()) { + listener.bind(new InetSocketAddress("127.0.0.1", 0)); + long addrInfo = NF.getAddrInfo("127.0.0.1", listener.getLocalPort()); + Assert.assertNotEquals(-1L, addrInfo); + try { + fd = NF.socketTcp(true); + Assert.assertTrue("could not allocate client socket", fd >= 0); + Assert.assertEquals(0, NF.connectAddrInfo(fd, addrInfo)); + } finally { + NF.freeAddrInfo(addrInfo); + } + + try (java.net.Socket peer = listener.accept()) { + socket.of(fd); + int retainedFd = fd; + fd = -1; + buffer = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT); + long recvBuffer = buffer; + + waiter = new Thread(() -> { + recvStarted.countDown(); + try { + recvResult.set(socket.recv(recvBuffer, 1)); + } catch (Throwable t) { + recvFailure.set(t); + } finally { + recvDone.set(true); + } + }, "socket-traffic-windows-recv-waiter"); + waiter.setDaemon(true); + waiter.start(); + + Assert.assertTrue("waiter did not reach the receive call", + recvStarted.await(5, TimeUnit.SECONDS)); + Assert.assertFalse("peer unexpectedly made the receive complete", recvDone.get()); + + socket.closeTraffic(); + + waiter.join(TimeUnit.SECONDS.toMillis(5)); + Assert.assertFalse("shutdown did not wake the native receive", waiter.isAlive()); + Assert.assertNull("native receive failed", recvFailure.get()); + Assert.assertTrue("shutdown must disconnect the native receive", recvResult.get() < 0); + Assert.assertEquals("traffic cancellation must retain fd ownership", retainedFd, socket.getFd()); + Assert.assertFalse("traffic cancellation must not perform full close", socket.isClosed()); + Assert.assertTrue("shutdown must leave the Winsock descriptor allocated", NF.getSndBuf(retainedFd) > 0); + + socket.close(); + Assert.assertTrue("full close must release the retained fd", socket.isClosed()); + Assert.assertEquals("released Winsock descriptor must reject socket operations", -1, NF.getSndBuf(retainedFd)); + } + } finally { + if (waiter != null && waiter.isAlive()) { + try { + socket.closeTraffic(); + } catch (Throwable ignored) { + // Full close below is the final wake-up fallback. + } + } + socket.close(); + if (waiter != null) { + waiter.join(TimeUnit.SECONDS.toMillis(5)); + } + if (buffer != 0 && (waiter == null || !waiter.isAlive())) { + Unsafe.free(buffer, 1, MemoryTag.NATIVE_DEFAULT); + } + if (fd != -1) { + NF.close(fd); + } + } + } + + @Test + public void testTlsSocketTrafficGatePreservesTlsStateUntilFullClose() throws Exception { + AtomicInteger closeCount = new AtomicInteger(); + AtomicInteger shutdownCount = new AtomicInteger(); + NetworkFacade facade = (NetworkFacade) Proxy.newProxyInstance( + NetworkFacade.class.getClassLoader(), + new Class[]{NetworkFacade.class}, + (proxy, method, args) -> { + if ("close".equals(method.getName())) { + closeCount.incrementAndGet(); + return method.getReturnType() == int.class ? 0 : null; + } + if ("shutdown".equals(method.getName())) { + shutdownCount.incrementAndGet(); + return 0; + } + if (method.getReturnType() == boolean.class) { + return false; + } + if (method.getReturnType() == int.class) { + return 0; + } + if (method.getReturnType() == long.class) { + return 0L; + } + return null; + } + ); + JavaTlsClientSocket socket = (JavaTlsClientSocket) JavaTlsClientSocketFactory + .INSECURE_NO_VALIDATION.newInstance( + facade, + LoggerFactory.getLogger(SocketTrafficShutdownTest.class) + ); + socket.of(42); + socket.setTlsStateForTesting(SSLContext.getDefault().createSSLEngine()); + JavaTlsClientSocket.TlsStateForTesting tlsState = socket.snapshotTlsStateForTesting(); + + try { + socket.closeTraffic(); + + JavaTlsClientSocket.TlsStateForTesting stateAfterTrafficClose = + socket.snapshotTlsStateForTesting(); + Assert.assertTrue("traffic cancellation must preserve TLS state and buffer references", + tlsState.hasSameStateForTesting(stateAfterTrafficClose)); + Assert.assertEquals(1, shutdownCount.get()); + Assert.assertEquals("traffic cancellation must not release the delegate fd", 0, closeCount.get()); + Assert.assertEquals(42, socket.getFd()); + Assert.assertFalse(socket.isClosed()); + } finally { + // Restore a valid plaintext state so full close does not attempt a + // synthetic TLS close_notify with uninitialised session buffers. + socket.setPlaintextStateForTesting(); + socket.close(); + } + Assert.assertEquals(1, closeCount.get()); + Assert.assertTrue(socket.isClosed()); + } + + @Test(timeout = 30_000L) + public void testTlsSocketTrafficGateUsesDelegateShutdownAndRetainsFd() throws Exception { + assertShutdownWakesMacOsKqueue(JavaTlsClientSocketFactory.INSECURE_NO_VALIDATION.newInstance( + NF, + LoggerFactory.getLogger(SocketTrafficShutdownTest.class) + )); + } + + private static void assertShutdownWakesMacOsKqueue(Socket socket) throws Exception { + Assume.assumeTrue("real kqueue cancellation coverage runs on macOS", Os.type == Os.DARWIN); + + AtomicBoolean pollDone = new AtomicBoolean(); + AtomicInteger pollResult = new AtomicInteger(Integer.MIN_VALUE); + AtomicReference pollFailure = new AtomicReference<>(); + CountDownLatch pollEntered = new CountDownLatch(1); + KqueueFacade facade = new KqueueFacade() { + private final KqueueFacade delegate = KqueueFacadeImpl.INSTANCE; + + @Override + public NetworkFacade getNetworkFacade() { + return delegate.getNetworkFacade(); + } + + @Override + public int kevent(int kq, long changeList, int nChanges, long eventList, int nEvents, int timeout) { + if (eventList != 0 && nEvents > 0) { + pollEntered.countDown(); + } + return delegate.kevent(kq, changeList, nChanges, eventList, nEvents, timeout); + } + + @Override + public int kqueue() { + return delegate.kqueue(); + } + }; + + int fd = -1; + Thread waiter = null; + try (ServerSocket listener = new ServerSocket()) { + listener.bind(new InetSocketAddress("127.0.0.1", 0)); + long addrInfo = NF.getAddrInfo("127.0.0.1", listener.getLocalPort()); + Assert.assertNotEquals(-1L, addrInfo); + try { + fd = NF.socketTcp(true); + Assert.assertTrue("could not allocate client socket", fd >= 0); + Assert.assertEquals(0, NF.connectAddrInfoTimeout(fd, addrInfo, 5_000)); + } finally { + NF.freeAddrInfo(addrInfo); + } + + try (java.net.Socket peer = listener.accept(); Kqueue kqueue = new Kqueue(facade, 1)) { + try { + Assert.assertEquals(0, NF.configureNonBlocking(fd)); + socket.of(fd); + int retainedFd = fd; + fd = -1; + + kqueue.setWriteOffset(0); + kqueue.readFD(retainedFd, 0); + Assert.assertEquals(0, kqueue.register(1)); + + waiter = new Thread(() -> { + try { + pollResult.set(kqueue.poll(10_000)); + } catch (Throwable t) { + pollFailure.set(t); + } finally { + pollDone.set(true); + } + }, "socket-traffic-kqueue-waiter"); + waiter.start(); + + Assert.assertTrue("waiter did not enter the native kqueue wait", + pollEntered.await(5, TimeUnit.SECONDS)); + Assert.assertFalse("peer unexpectedly made the read wait ready", pollDone.get()); + + socket.closeTraffic(); + + waiter.join(TimeUnit.SECONDS.toMillis(5)); + Assert.assertFalse("shutdown did not wake the native kqueue wait", waiter.isAlive()); + Assert.assertNull("native kqueue wait failed", pollFailure.get()); + Assert.assertTrue("shutdown must produce a readiness event", pollResult.get() > 0); + Assert.assertEquals("traffic cancellation must retain fd ownership", retainedFd, socket.getFd()); + Assert.assertFalse("traffic cancellation must not perform full close", socket.isClosed()); + + socket.close(); + Assert.assertTrue("full close must release the retained fd", socket.isClosed()); + } finally { + socket.closeTraffic(); + if (waiter != null) { + waiter.join(TimeUnit.SECONDS.toMillis(5)); + } + socket.close(); + } + } + } finally { + if (fd != -1) { + NF.close(fd); + } + } + } + + private static void assertUnsupported(Runnable operation) { + try { + operation.run(); + Assert.fail("expected unsupported traffic shutdown"); + } catch (UnsupportedOperationException expected) { + // Expected compatibility behavior: no native or destructive fallback. + } + } + +} diff --git a/core/src/test/java/io/questdb/client/test/std/FilesTest.java b/core/src/test/java/io/questdb/client/test/std/FilesTest.java index ebef927a..63252c45 100644 --- a/core/src/test/java/io/questdb/client/test/std/FilesTest.java +++ b/core/src/test/java/io/questdb/client/test/std/FilesTest.java @@ -73,6 +73,16 @@ public void tearDown() { Files.remove(tmpDir); } + @Test + public void testFsyncParentDir() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String child = tmpDir + "/child"; + assertEquals(0, Files.mkdir(child, Files.DIR_MODE_DEFAULT)); + assertEquals(0, Files.fsyncParentDir(child)); + assertTrue(Files.remove(child)); + }); + } + @Test public void testWriteReadRoundtrip() throws Exception { TestUtils.assertMemoryLeak(() -> { diff --git a/core/src/test/java/io/questdb/client/test/std/ObjListTest.java b/core/src/test/java/io/questdb/client/test/std/ObjListTest.java index b2efff7b..cc8c6e8b 100644 --- a/core/src/test/java/io/questdb/client/test/std/ObjListTest.java +++ b/core/src/test/java/io/questdb/client/test/std/ObjListTest.java @@ -28,6 +28,8 @@ import org.junit.Assert; import org.junit.Test; +import java.lang.reflect.Field; + /** * */ @@ -58,6 +60,21 @@ public void testRemoveFromTo() { Assert.assertEquals(list(), remove(list("a", "b", "c"), 4, 10)); } + @Test + public void testRemoveFromToClearsFinalBackingSlot() throws Exception { + ObjList values = new ObjList<>(16); + for (int i = 0; i < 16; i++) { + values.add(new Object()); + } + + values.remove(0, 0); + + Field bufferField = ObjList.class.getDeclaredField("buffer"); + bufferField.setAccessible(true); + Object[] buffer = (Object[]) bufferField.get(values); + Assert.assertNull(buffer[buffer.length - 1]); + } + private static ObjList remove(ObjList o, int from, int to) { o.remove(from, to); return o; diff --git a/examples/src/main/java/com/example/sender/WsStoreAndForwardExample.java b/examples/src/main/java/com/example/sender/WsStoreAndForwardExample.java index 18cc7b52..082dc4c4 100644 --- a/examples/src/main/java/com/example/sender/WsStoreAndForwardExample.java +++ b/examples/src/main/java/com/example/sender/WsStoreAndForwardExample.java @@ -4,7 +4,7 @@ import io.questdb.client.Sender; /** - * Crash-durable ingest with store-and-forward ({@code sf_dir}). + * Process-restart and periodic power-loss protection with store-and-forward. *

      * Ingestion is asynchronous in every mode: {@code flush()} hands rows to a * background send engine that delivers them and tracks the server's @@ -17,7 +17,10 @@ *

    3. Store-and-forward ({@code sf_dir} set): the engine backs its * buffer with memory-mapped files, so unacked rows survive a producer * process restart -- on the next startup it recovers the tail from - * disk and replays it once the server is reachable.
      4. + * disk and replays it once the server is reachable. Add + * {@code sf_durability=periodic} for background power-loss checkpoints; + * the configured interval is a target cadence and storage latency adds + * to the actual recovery window. * * {@code request_durable_ack=on} (Enterprise + replication) makes the ack wait * for the durable upload to object storage rather than the ordinary WAL commit. @@ -28,6 +31,8 @@ public static void main(String[] args) { try (QuestDB db = QuestDB.connect( "ws::addr=localhost:9000;" + "sf_dir=/var/lib/questdb/sf;" + + "sf_durability=periodic;" + + "sf_sync_interval_millis=5000;" + "request_durable_ack=on;")) { try (Sender sender = db.borrowSender()) {