diff --git a/include/chunkio/cio_file_st.h b/include/chunkio/cio_file_st.h index 231cb58..8946c82 100644 --- a/include/chunkio/cio_file_st.h +++ b/include/chunkio/cio_file_st.h @@ -37,9 +37,10 @@ * | 0xC1 | 0x00 +--> Header 2 bytes * +--------------+----------------+ * | 4 BYTES +--> CRC32(Content) - * | 4 BYTES +--> CRC32(Padding) + * | 4 BYTES +--> Legacy CRC padding * | 4 BYTES +--> Content length - * | 8 BYTES +--> Padding + * | 4 BYTES +--> Shadow content length + * | 4 BYTES +--> Shadow CRC32(Content) * +-------------------------------+ * | Content | * | +-------------------------+ | @@ -62,10 +63,14 @@ #define CIO_FILE_ID_01 0x00 /* header: second byte */ #define CIO_FILE_HEADER_MIN 24 /* 24 bytes for the header */ #define CIO_FILE_CONTENT_OFFSET 22 +#define CIO_FILE_LEGACY_CRC_PADDING_OFFSET 6 +#define CIO_FILE_LEGACY_CRC_PADDING_SIZE 4 #define CIO_FILE_CONTENT_LENGTH_OFFSET 10 /* We store the content length * right after the checksum in * what used to be padding */ +#define CIO_FILE_SHADOW_CONTENT_LENGTH_OFFSET 14 +#define CIO_FILE_SHADOW_HASH_OFFSET 18 /* Return pointer to hash position */ static inline char *cio_file_st_get_hash(char *map) { diff --git a/src/cio_file.c b/src/cio_file.c index 1bd4ef6..a869c17 100644 --- a/src/cio_file.c +++ b/src/cio_file.c @@ -61,32 +61,44 @@ char cio_file_init_bytes[] = { }; #define ROUND_UP(N, S) ((((N) + (S) - 1) / (S)) * (S)) +#define CIO_FILE_CRC_FIELD_SIZE 4 +static uint32_t get_header_u32(char *map, int offset) +{ + uint8_t *buffer; -/* Calculate content checksum in a variable */ -void cio_file_calculate_checksum(struct cio_file *cf, crc_t *out) + buffer = (uint8_t *) &map[offset]; + + return (((uint32_t) buffer[0]) << 24) | + (((uint32_t) buffer[1]) << 16) | + (((uint32_t) buffer[2]) << 8) | + (((uint32_t) buffer[3]) << 0); +} + +static void set_header_u32(char *map, int offset, uint32_t value) +{ + uint8_t *buffer; + + buffer = (uint8_t *) &map[offset]; + + buffer[0] = (uint8_t) ((value & 0xFF000000) >> 24); + buffer[1] = (uint8_t) ((value & 0x00FF0000) >> 16); + buffer[2] = (uint8_t) ((value & 0x0000FF00) >> 8); + buffer[3] = (uint8_t) ((value & 0x000000FF) >> 0); +} + +static void calculate_checksum_len(struct cio_file *cf, + size_t content_length, + crc_t *out) { crc_t val; size_t len; - ssize_t content_length; unsigned char *in_data; - if (cf->fs_size == 0) { - cio_file_update_size(cf); - } - /* Metadata length header + metadata length + content length */ len = 2; len += cio_file_st_get_meta_len(cf->map); - - content_length = cio_file_st_get_content_len(cf->map, - cf->fs_size, - cf->page_size, - cf->taint_flag); - - if (content_length > 0) { - len += content_length; - } + len += content_length; in_data = (unsigned char *) cf->map + CIO_FILE_CONTENT_OFFSET; @@ -94,6 +106,16 @@ void cio_file_calculate_checksum(struct cio_file *cf, crc_t *out) *out = val; } +/* Calculate content checksum in a variable */ +void cio_file_calculate_checksum(struct cio_file *cf, crc_t *out) +{ + if (cf->fs_size == 0) { + cio_file_update_size(cf); + } + + calculate_checksum_len(cf, cf->data_size, out); +} + /* Update crc32 checksum into the memory map */ static void update_checksum(struct cio_file *cf, unsigned char *data, size_t len) @@ -109,19 +131,228 @@ static void update_checksum(struct cio_file *cf, } crc = cio_crc32_update(cf->crc_cur, data, len); - memcpy(cf->map + 2, &crc, sizeof(crc)); cf->crc_cur = crc; } +static uint32_t finalize_checksum_value(crc_t crc) +{ + return (uint32_t) cio_crc32_finalize(crc); +} + +static void write_checksum(struct cio_file *cf, uint32_t crc) +{ + set_header_u32(cf->map, 2, crc); + memset(cf->map + CIO_FILE_LEGACY_CRC_PADDING_OFFSET, 0, + CIO_FILE_LEGACY_CRC_PADDING_SIZE); +} + /* Finalize CRC32 context and update the memory map */ static void finalize_checksum(struct cio_file *cf) { + uint32_t crc; + + crc = finalize_checksum_value(cf->crc_cur); + + write_checksum(cf, crc); +} + +static int shadow_header_exists(struct cio_file *cf) +{ + uint32_t content_length; + uint32_t crc; + + content_length = get_header_u32(cf->map, + CIO_FILE_SHADOW_CONTENT_LENGTH_OFFSET); + crc = get_header_u32(cf->map, CIO_FILE_SHADOW_HASH_OFFSET); + + if (content_length == 0 && crc == 0) { + return CIO_FALSE; + } + + return CIO_TRUE; +} + +static void write_shadow_header(struct cio_file *cf, uint32_t crc) +{ + set_header_u32(cf->map, + CIO_FILE_SHADOW_CONTENT_LENGTH_OFFSET, + (uint32_t) cf->data_size); + + set_header_u32(cf->map, CIO_FILE_SHADOW_HASH_OFFSET, crc); +} + +static int recover_unfinalized_checksum(struct cio_chunk *ch, + struct cio_file *cf, + unsigned char *hash, + crc_t crc) +{ + int ret; + + /* + * Legacy chunks stored the in-progress (non finalized) crc by copying the + * raw crc_t with memcpy(map + 2, &crc, sizeof(crc)). crc_t is uint_fast32_t, + * which is 8 bytes on most LP64 targets, so the comparison must use the same + * native width: on big-endian 64-bit the significant bytes live at offset + * 6..9 with zeros at 2..5, and a fixed 4-byte compare would spuriously match + * the zero bytes. Comparing sizeof(crc) reproduces exactly what the legacy + * writer emitted on this architecture, on both little- and big-endian. + */ + if (memcmp(hash, &crc, sizeof(crc)) != 0) { + return CIO_FALSE; + } + + cf->crc_cur = crc; + + if (!(cf->flags & CIO_OPEN_RW)) { + cio_log_info(ch->ctx, + "[cio file] accepted unfinalized crc32 at %s/%s (read-only)", + ch->name, cf->path); + + return CIO_TRUE; + } + + finalize_checksum(cf); + cf->synced = CIO_FALSE; + + ret = cio_file_native_sync(cf, CIO_FULL_SYNC); + if (ret == CIO_OK) { + cf->synced = CIO_TRUE; + cio_log_info(ch->ctx, "[cio file] recovered unfinalized crc32 at %s/%s", + ch->name, cf->path); + } + else { + cio_log_warn(ch->ctx, + "[cio file] could not persist recovered crc32 at %s/%s", + ch->name, cf->path); + } + + return CIO_TRUE; +} + +static int recover_shadow_header(struct cio_chunk *ch, struct cio_file *cf) +{ + int ret; + size_t metadata_length; + size_t content_length; crc_t crc; + uint32_t crc_check; + uint32_t stored_crc; - crc = cio_crc32_finalize(cf->crc_cur); - crc = htonl(crc); + if (shadow_header_exists(cf) == CIO_FALSE) { + return CIO_FALSE; + } - memcpy(cf->map + 2, &crc, sizeof(crc)); + metadata_length = cio_file_st_get_meta_len(cf->map); + content_length = get_header_u32(cf->map, + CIO_FILE_SHADOW_CONTENT_LENGTH_OFFSET); + + /* Validate by subtraction so a corrupt length cannot wrap on 32-bit. */ + if (cf->fs_size < CIO_FILE_HEADER_MIN || + metadata_length > cf->fs_size - CIO_FILE_HEADER_MIN || + content_length > cf->fs_size - CIO_FILE_HEADER_MIN - metadata_length) { + return CIO_FALSE; + } + + cf->crc_cur = cio_crc32_init(); + calculate_checksum_len(cf, content_length, &crc); + + crc_check = finalize_checksum_value(crc); + stored_crc = get_header_u32(cf->map, CIO_FILE_SHADOW_HASH_OFFSET); + + if (stored_crc != crc_check) { + return CIO_FALSE; + } + + cf->data_size = content_length; + cf->crc_cur = crc; + + if (!(cf->flags & CIO_OPEN_RW)) { + cio_log_info(ch->ctx, + "[cio file] accepted shadow crc32 at %s/%s (read-only)", + ch->name, cf->path); + + return CIO_TRUE; + } + + cio_file_st_set_content_len(cf->map, (uint32_t) cf->data_size); + write_checksum(cf, crc_check); + cf->synced = CIO_FALSE; + + ret = cio_file_native_sync(cf, CIO_FULL_SYNC); + if (ret == CIO_OK) { + cf->synced = CIO_TRUE; + cio_log_info(ch->ctx, "[cio file] recovered shadow crc32 at %s/%s", + ch->name, cf->path); + } + else { + cio_log_warn(ch->ctx, + "[cio file] could not persist shadow crc32 at %s/%s", + ch->name, cf->path); + } + + return CIO_TRUE; +} + +/* + * Older 64-bit big-endian builds finalized the crc and stored it with + * memcpy(map + 2, &crc, sizeof(crc_t)). With an 8-byte crc_t the big-endian + * representation placed the four significant bytes at offset 6..9 and left the + * canonical primary slot (offset 2..5) zero -- the mirror image of the + * little-endian layout the current code reads. (Old 32-bit big-endian builds + * used a 4-byte crc_t and already wrote the canonical offset, so only LP64 + * big-endian chunks need this fallback.) + * + * Detect that layout in an endianness-independent way: the canonical slot must + * be empty and the big-endian value at offset 6 must equal the finalized crc of + * the content. When matched, accept it and, if the chunk is writable, rewrite + * it into the canonical portable layout so future reads are O(1) and the file + * becomes architecture independent. + */ +static int recover_legacy_wide_checksum(struct cio_chunk *ch, + struct cio_file *cf, + crc_t crc, uint32_t crc_check) +{ + int ret; + uint32_t legacy_crc; + + /* The canonical primary crc slot must be empty for this legacy layout */ + if (get_header_u32(cf->map, 2) != 0) { + return CIO_FALSE; + } + + legacy_crc = get_header_u32(cf->map, CIO_FILE_LEGACY_CRC_PADDING_OFFSET); + if (legacy_crc == 0 || legacy_crc != crc_check) { + return CIO_FALSE; + } + + cf->crc_cur = crc; + + if (!(cf->flags & CIO_OPEN_RW)) { + cio_log_info(ch->ctx, + "[cio file] accepted legacy big-endian crc32 at %s/%s " + "(read-only)", ch->name, cf->path); + + return CIO_TRUE; + } + + /* Migrate to the canonical layout: crc at offset 2, padding zeroed */ + write_checksum(cf, crc_check); + cf->synced = CIO_FALSE; + + ret = cio_file_native_sync(cf, CIO_FULL_SYNC); + if (ret == CIO_OK) { + cf->synced = CIO_TRUE; + cio_log_info(ch->ctx, + "[cio file] migrated legacy big-endian crc32 at %s/%s", + ch->name, cf->path); + } + else { + cio_log_warn(ch->ctx, + "[cio file] could not persist migrated crc32 at %s/%s", + ch->name, cf->path); + } + + return CIO_TRUE; } /* @@ -188,11 +419,13 @@ static size_t get_available_size(struct cio_file *cf, int *meta_len) static int cio_file_format_check(struct cio_chunk *ch, struct cio_file *cf, int flags) { + int ret; size_t metadata_length; ssize_t content_length; ssize_t logical_length; unsigned char *p; - crc_t crc_check; + uint32_t crc_check; + uint32_t stored_crc; crc_t crc; (void) flags; @@ -257,6 +490,11 @@ static int cio_file_format_check(struct cio_chunk *ch, content_length; if (logical_length > cf->fs_size) { + if ((ch->ctx->options.flags & CIO_CHECKSUM) && + recover_shadow_header(ch, cf) == CIO_TRUE) { + return 0; + } + cio_log_debug(ch->ctx, "[cio file] truncated file (%zd / %zd) %s", cf->fs_size, logical_length, ch->name); cio_error_set(ch, CIO_ERR_BAD_FILE_SIZE); @@ -271,21 +509,40 @@ static int cio_file_format_check(struct cio_chunk *ch, /* Get checksum stored in the mmap */ p = (unsigned char *) cio_file_st_get_hash(cf->map); + stored_crc = get_header_u32(cf->map, 2); /* Calculate content checksum */ cio_file_calculate_checksum(cf, &crc); /* Compare */ - crc_check = cio_crc32_finalize(crc); - crc_check = htonl(crc_check); - - if (memcmp(p, &crc_check, sizeof(crc_check)) != 0) { - cio_log_info(ch->ctx, "[cio file] invalid crc32 at %s/%s", - ch->name, cf->path); - cio_error_set(ch, CIO_ERR_BAD_CHECKSUM); - - return -1; + crc_check = finalize_checksum_value(crc); + + if (stored_crc != crc_check) { + ret = recover_unfinalized_checksum(ch, cf, p, crc); + if (ret == CIO_FALSE) { + ret = recover_shadow_header(ch, cf); + if (ret == CIO_FALSE) { + ret = recover_legacy_wide_checksum(ch, cf, crc, + crc_check); + } + + if (ret == CIO_FALSE) { + cio_log_info(ch->ctx, "[cio file] invalid crc32 at %s/%s", + ch->name, cf->path); + cio_error_set(ch, CIO_ERR_BAD_CHECKSUM); + + return -1; + } + + return 0; + } } + /* + * A valid primary tuple is authoritative. Older writers leave the + * shadow padding untouched, and without a generation marker a + * different shadow length cannot establish which tuple is newer. + * Recovering it here could discard an append or undo a truncation. + */ cf->crc_cur = crc; } @@ -1044,9 +1301,6 @@ int cio_file_write(struct cio_chunk *ch, const void *buf, size_t count) if (count == 0) { if (cf->crc_reset) { - cf->taint_flag = CIO_TRUE; - cio_file_st_set_content_len(cf->map, cf->data_size); - if (ch->ctx->options.flags & CIO_CHECKSUM) { cf->crc_cur = cio_crc32_init(); cio_file_calculate_checksum(cf, &cf->crc_cur); @@ -1054,6 +1308,11 @@ int cio_file_write(struct cio_chunk *ch, const void *buf, size_t count) cf->crc_reset = CIO_FALSE; cf->synced = CIO_FALSE; + cf->taint_flag = CIO_TRUE; + + if (!(ch->ctx->options.flags & CIO_CHECKSUM)) { + cio_file_st_set_content_len(cf->map, cf->data_size); + } } return 0; @@ -1104,15 +1363,6 @@ int cio_file_write(struct cio_chunk *ch, const void *buf, size_t count) old_size, new_size); } - /* If crc_reset was toggled we know that data_size was - * modified by cio_chunk_write_at which means we need - * to update the header before we recalculate the checksum - */ - if (cf->crc_reset) { - cf->taint_flag = CIO_TRUE; - cio_file_st_set_content_len(cf->map, cf->data_size); - } - if (ch->ctx->options.flags & CIO_CHECKSUM) { update_checksum(cf, (unsigned char *) buf, count); } @@ -1123,10 +1373,12 @@ int cio_file_write(struct cio_chunk *ch, const void *buf, size_t count) cf->data_size += count; cf->synced = CIO_FALSE; - cio_file_st_set_content_len(cf->map, cf->data_size); - cf->taint_flag = CIO_TRUE; + if (!(ch->ctx->options.flags & CIO_CHECKSUM)) { + cio_file_st_set_content_len(cf->map, cf->data_size); + } + return 0; } @@ -1221,6 +1473,7 @@ int cio_file_sync(struct cio_chunk *ch) size_t desired_size; size_t file_size; size_t av_size; + uint32_t crc; struct cio_file *cf; if (ch == NULL) { @@ -1296,12 +1549,16 @@ int cio_file_sync(struct cio_chunk *ch) } } - /* Persist the logical length after appends, truncation or rollback. */ - cio_file_st_set_content_len(cf->map, cf->data_size); - /* Finalize CRC32 checksum */ if (ch->ctx->options.flags & CIO_CHECKSUM) { - finalize_checksum(cf); + crc = finalize_checksum_value(cf->crc_cur); + write_shadow_header(cf, crc); + cio_file_st_set_content_len(cf->map, (uint32_t) cf->data_size); + write_checksum(cf, crc); + } + else { + /* Persist the logical length after appends, truncation or rollback. */ + cio_file_st_set_content_len(cf->map, cf->data_size); } /* Commit changes to disk */ @@ -1408,9 +1665,8 @@ void cio_file_scan_dump(struct cio_ctx *ctx, struct cio_stream *st) int ret; int meta_len; int set_down = CIO_FALSE; - char *p; crc_t crc; - crc_t crc_fs; + uint32_t crc_fs; char tmp[PATH_MAX]; struct mk_list *head; struct cio_chunk *ch; @@ -1431,10 +1687,7 @@ void cio_file_scan_dump(struct cio_ctx *ctx, struct cio_stream *st) snprintf(tmp, sizeof(tmp) -1, "%s/%s", st->name, ch->name); meta_len = cio_file_st_get_meta_len(cf->map); - p = cio_file_st_get_hash(cf->map); - - memcpy(&crc_fs, p, sizeof(crc_fs)); - crc_fs = ntohl(crc_fs); + crc_fs = get_header_u32(cf->map, 2); printf(" %-60s", tmp); diff --git a/tests/fs.c b/tests/fs.c index 8549e08..56fbed3 100644 --- a/tests/fs.c +++ b/tests/fs.c @@ -21,7 +21,10 @@ #include #include #endif +#include +#include #include +#include #include #include @@ -34,6 +37,7 @@ #include #include #include +#include #include "cio_tests_internal.h" @@ -51,6 +55,51 @@ static int log_cb(struct cio_ctx *ctx, int level, const char *file, int line, return 0; } +static void set_test_header_u32(char *map, int offset, uint32_t value) +{ + uint8_t *buffer; + + buffer = (uint8_t *) &map[offset]; + + buffer[0] = (uint8_t) ((value & 0xFF000000) >> 24); + buffer[1] = (uint8_t) ((value & 0x00FF0000) >> 16); + buffer[2] = (uint8_t) ((value & 0x0000FF00) >> 8); + buffer[3] = (uint8_t) ((value & 0x000000FF) >> 0); +} + +static uint32_t get_test_header_u32(char *map, int offset) +{ + uint8_t *buffer; + + buffer = (uint8_t *) &map[offset]; + + return (((uint32_t) buffer[0]) << 24) | + (((uint32_t) buffer[1]) << 16) | + (((uint32_t) buffer[2]) << 8) | + (((uint32_t) buffer[3]) << 0); +} + +static struct cio_chunk *find_stream_chunk(struct cio_stream *stream, + const char *name) +{ + struct mk_list *head; + struct cio_chunk *chunk; + + if (stream == NULL || name == NULL) { + return NULL; + } + + mk_list_foreach(head, &stream->chunks) { + chunk = mk_list_entry(head, struct cio_chunk, _head); + + if (strcmp(chunk->name, name) == 0) { + return chunk; + } + } + + return NULL; +} + /* Test API generating files to the file system and then scanning them back */ static void test_fs_write() { @@ -721,56 +770,66 @@ void test_issue_write_at() cf = (struct cio_file *) chunk->backend; TEST_CHECK(cf->map == NULL); TEST_CHECK(cf->fd <= 0); + cio_destroy(ctx); } - -void test_fs_up_down_up_append() +static void test_fs_recover_unfinalized_checksum() { int ret; int err; + char *hash; + char *out_buf; + size_t out_size; + crc_t crc_unfinalized; struct cio_ctx *ctx; + struct cio_file *cf; struct cio_chunk *chunk; struct cio_stream *stream; struct cio_options cio_opts; - - void *out_buf; - size_t out_size; + char line[] = "recoverable checksum payload\n"; cio_utils_recursive_delete(CIO_ENV); - /* create Chunk I/O context */ cio_options_init(&cio_opts); cio_opts.root_path = CIO_ENV; cio_opts.log_cb = log_cb; - cio_opts.log_level = CIO_LOG_DEBUG; + cio_opts.log_level = CIO_LOG_INFO; cio_opts.flags = CIO_CHECKSUM; - /* Create a temporal storage */ ctx = cio_create(&cio_opts); - stream = cio_stream_create(ctx, "cio", CIO_STORE_FS); - chunk = cio_chunk_open(ctx, stream, "c", CIO_OPEN, 1000, &err); + TEST_CHECK(ctx != NULL); + + stream = cio_stream_create(ctx, "test_recover_crc", CIO_STORE_FS); + TEST_CHECK(stream != NULL); + + chunk = cio_chunk_open(ctx, stream, "test", CIO_OPEN, 1000, &err); TEST_CHECK(chunk != NULL); if (!chunk) { - printf("cannot open chunk\n"); exit(1); } - ret = cio_chunk_get_content_copy(chunk, &out_buf, &out_size); + ret = cio_chunk_write(chunk, line, strlen(line)); TEST_CHECK(ret == CIO_OK); - TEST_CHECK(memcmp(out_buf, "", 1) == 0); - TEST_CHECK(out_size == 0); - free(out_buf); - ret = cio_chunk_write(chunk, "line 1\n", 7); + ret = cio_chunk_sync(chunk); TEST_CHECK(ret == CIO_OK); - ret = cio_chunk_get_content_copy(chunk, &out_buf, &out_size); + cf = (struct cio_file *) chunk->backend; + crc_unfinalized = cf->crc_cur; + hash = cio_file_hash(cf); + /* + * Reproduce the legacy on-disk layout exactly: the old writer stored the + * non finalized crc with memcpy(map + 2, &crc, sizeof(crc)). Using the full + * native width here means this test exercises the correct bytes on both + * little- and big-endian targets (on big-endian 64-bit the significant + * bytes land at offset 6..9, not 2..5). + */ + memcpy(hash, &crc_unfinalized, sizeof(crc_unfinalized)); + + ret = cio_file_native_sync(cf, CIO_FULL_SYNC); TEST_CHECK(ret == CIO_OK); - TEST_CHECK(memcmp(out_buf, "line 1\n", 7+1) == 0); - TEST_CHECK(out_size == 7); - free(out_buf); ret = cio_chunk_down(chunk); TEST_CHECK(ret == CIO_OK); @@ -778,216 +837,2065 @@ void test_fs_up_down_up_append() ret = cio_chunk_up(chunk); TEST_CHECK(ret == CIO_OK); - ret = cio_chunk_get_content_copy(chunk, &out_buf, &out_size); + ret = cio_chunk_get_content_copy(chunk, (void **) &out_buf, &out_size); TEST_CHECK(ret == CIO_OK); - TEST_CHECK(memcmp(out_buf, "line 1\n", 7+1) == 0); - TEST_CHECK(out_size == 7); + TEST_CHECK(out_size == strlen(line)); + TEST_CHECK(memcmp(out_buf, line, out_size) == 0); free(out_buf); - /* append */ - ret = cio_chunk_write(chunk, "line 2\n", 7); - TEST_CHECK(ret == CIO_OK); - ret = cio_chunk_down(chunk); TEST_CHECK(ret == CIO_OK); ret = cio_chunk_up(chunk); TEST_CHECK(ret == CIO_OK); - ret = cio_chunk_get_content_copy(chunk, &out_buf, &out_size); - TEST_CHECK(ret == CIO_OK); - TEST_CHECK(memcmp(out_buf, "line 1\nline 2\n", 7*2+1) == 0); - TEST_CHECK(out_size == 7*2); - free(out_buf); - cio_destroy(ctx); } -static void test_deep_hierarchy() +static void test_fs_shadow_checksum_recovers_primary_header() { - int i; int ret; int err; - int len; - char line[] = "this is a test line\n"; + char *hash; + char *out_buf; + size_t out_size; struct cio_ctx *ctx; + struct cio_file *cf; struct cio_chunk *chunk; struct cio_stream *stream; struct cio_options cio_opts; + char line[] = "shadow checksum payload\n"; - cio_utils_recursive_delete("tmp"); + cio_utils_recursive_delete(CIO_ENV); - /* Create a temporal storage */ cio_options_init(&cio_opts); - cio_opts.root_path = "tmp/deep/log/dir"; + cio_opts.root_path = CIO_ENV; cio_opts.log_cb = log_cb; - cio_opts.log_level = CIO_LOG_DEBUG; - cio_opts.flags = 0; + cio_opts.log_level = CIO_LOG_INFO; + cio_opts.flags = CIO_CHECKSUM; ctx = cio_create(&cio_opts); - stream = cio_stream_create(ctx, "test", CIO_STORE_FS); - chunk = cio_chunk_open(ctx, stream, "c", CIO_OPEN, 1000, &err); + TEST_CHECK(ctx != NULL); + + stream = cio_stream_create(ctx, "test_shadow_crc", CIO_STORE_FS); + TEST_CHECK(stream != NULL); + + chunk = cio_chunk_open(ctx, stream, "test", CIO_OPEN, 1000, &err); TEST_CHECK(chunk != NULL); if (!chunk) { - printf("cannot open chunk\n"); exit(1); } - len = strlen(line); - for (i = 0; i < 1000; i++) { - ret = cio_chunk_write(chunk, line, len); - TEST_CHECK(ret == CIO_OK); - - ret = cio_chunk_down(chunk); - TEST_CHECK(ret == CIO_OK); + ret = cio_chunk_write(chunk, line, strlen(line)); + TEST_CHECK(ret == CIO_OK); - ret = cio_chunk_up(chunk); - TEST_CHECK(ret == CIO_OK); - } + ret = cio_chunk_sync(chunk); + TEST_CHECK(ret == CIO_OK); - cio_destroy(ctx); -} + cf = (struct cio_file *) chunk->backend; + hash = cio_file_hash(cf); + hash[0] ^= 0xff; -static void truncate_file(struct cio_file *chunk_file, - size_t new_file_size, - int remove_content_length) -{ - int result; + ret = cio_file_native_sync(cf, CIO_FULL_SYNC); + TEST_CHECK(ret == CIO_OK); - result = cio_file_native_open(chunk_file); - TEST_CHECK(result == CIO_OK); + ret = cio_chunk_down(chunk); + TEST_CHECK(ret == CIO_OK); - result = cio_file_native_map(chunk_file, - chunk_file->page_size); - TEST_CHECK(result == CIO_OK); + ret = cio_chunk_up(chunk); + TEST_CHECK(ret == CIO_OK); - if (remove_content_length) { - chunk_file->map[CIO_FILE_CONTENT_LENGTH_OFFSET + 0] = 0; - chunk_file->map[CIO_FILE_CONTENT_LENGTH_OFFSET + 1] = 0; - chunk_file->map[CIO_FILE_CONTENT_LENGTH_OFFSET + 2] = 0; - chunk_file->map[CIO_FILE_CONTENT_LENGTH_OFFSET + 3] = 0; - } + ret = cio_chunk_get_content_copy(chunk, (void **) &out_buf, &out_size); + TEST_CHECK(ret == CIO_OK); + TEST_CHECK(out_size == strlen(line)); + TEST_CHECK(memcmp(out_buf, line, out_size) == 0); + free(out_buf); - result = cio_file_native_unmap(chunk_file); - TEST_CHECK(result == CIO_OK); + ret = cio_chunk_down(chunk); + TEST_CHECK(ret == CIO_OK); - result = cio_file_native_resize(chunk_file, new_file_size); - TEST_CHECK(result == 0); + ret = cio_chunk_up(chunk); + TEST_CHECK(ret == CIO_OK); - result = cio_file_native_close(chunk_file); - TEST_CHECK(result == CIO_OK); + cio_destroy(ctx); } -static void test_legacy_core(int trigger_checksum_error) +static void test_fs_shadow_checksum_recovers_read_only() { - struct cio_options cio_opts; - char *in_data; - size_t in_size; + int ret; + int err; + FILE *fp; + char *hash; + char *out_buf; + size_t out_size; + struct cio_ctx *ctx; + struct cio_file *cf; + struct cio_chunk *chunk; struct cio_stream *stream; - struct cio_chunk *chunk; - size_t delta; - struct cio_ctx *ctx; - int ret; + struct cio_options cio_opts; + unsigned char corrupted_hash[4]; + unsigned char stored_hash[4]; + char path[1024]; + char line[] = "read-only shadow checksum payload\n"; - /* delete any previous temporary content directory */ cio_utils_recursive_delete(CIO_ENV); - /* - * Load sample data file and with the same content through multiple write - * operations generating other files. - */ - ret = cio_utils_read_file(CIO_FILE_400KB, &in_data, &in_size); - TEST_CHECK(ret == 0); - if (ret == -1) { - exit(EXIT_FAILURE); - } - /* create Chunk I/O context */ cio_options_init(&cio_opts); cio_opts.root_path = CIO_ENV; cio_opts.log_cb = log_cb; - cio_opts.log_level = CIO_LOG_DEBUG; + cio_opts.log_level = CIO_LOG_INFO; cio_opts.flags = CIO_CHECKSUM; - /* Create a temporal storage */ ctx = cio_create(&cio_opts); + TEST_CHECK(ctx != NULL); - stream = cio_stream_create(ctx, "test-legacy", CIO_STORE_FS); + stream = cio_stream_create(ctx, "test_shadow_read_only", CIO_STORE_FS); + TEST_CHECK(stream != NULL); - /* do not force a maximum of chunks up, we want to test writing overhead */ - cio_set_max_chunks_up(ctx, 1); + chunk = cio_chunk_open(ctx, stream, "test", CIO_OPEN, 1000, &err); + TEST_CHECK(chunk != NULL); + if (!chunk) { + exit(1); + } - chunk = cio_chunk_open(ctx, - stream, - "test_chunk", - CIO_OPEN, - 1000, - &ret); + ret = cio_chunk_write(chunk, line, strlen(line)); + TEST_CHECK(ret == CIO_OK); - ret = cio_chunk_write(chunk, in_data, 128); - TEST_CHECK(ret == 0); + ret = cio_chunk_sync(chunk); + TEST_CHECK(ret == CIO_OK); - ret = cio_chunk_down(chunk); + cf = (struct cio_file *) chunk->backend; + hash = cio_file_hash(cf); + hash[0] ^= 0xff; + memcpy(corrupted_hash, hash, sizeof(corrupted_hash)); + + ret = cio_file_native_sync(cf, CIO_FULL_SYNC); TEST_CHECK(ret == CIO_OK); - delta = CIO_FILE_HEADER_MIN; + snprintf(path, sizeof(path), "%s/%s/%s", CIO_ENV, + "test_shadow_read_only", "test"); - if (trigger_checksum_error) { - delta++; - } + cio_destroy(ctx); - truncate_file((struct cio_file *) chunk->backend, - 128 + delta, - CIO_TRUE); + cio_options_init(&cio_opts); - ret = cio_chunk_up(chunk); + cio_opts.root_path = CIO_ENV; + cio_opts.log_cb = log_cb; + cio_opts.log_level = CIO_LOG_INFO; + cio_opts.flags = CIO_OPEN_RD | CIO_CHECKSUM; - if (trigger_checksum_error) { - TEST_CHECK(ret != CIO_OK); - } - else { - TEST_CHECK(ret == CIO_OK); + ctx = cio_create(&cio_opts); + TEST_CHECK(ctx != NULL); + + stream = cio_stream_create(ctx, "test_shadow_read_only", CIO_STORE_FS); + TEST_CHECK(stream != NULL); + + chunk = cio_chunk_open(ctx, stream, "test", CIO_OPEN_RD, 0, &err); + TEST_CHECK(chunk != NULL); + if (!chunk) { + exit(1); } + ret = cio_chunk_get_content_copy(chunk, (void **) &out_buf, &out_size); + TEST_CHECK(ret == CIO_OK); + TEST_CHECK(out_size == strlen(line)); + TEST_CHECK(memcmp(out_buf, line, out_size) == 0); + free(out_buf); + cio_destroy(ctx); - free(in_data); -} + fp = fopen(path, "rb+"); + TEST_CHECK(fp != NULL); + if (!fp) { + exit(1); + } -void test_legacy_success() -{ - test_legacy_core(CIO_FALSE); -} + ret = fseek(fp, 2, SEEK_SET); + TEST_CHECK(ret == 0); -void test_legacy_failure() -{ - test_legacy_core(CIO_TRUE); + ret = fread(stored_hash, 1, sizeof(stored_hash), fp); + TEST_CHECK(ret == sizeof(stored_hash)); + + ret = fclose(fp); + TEST_CHECK(ret == 0); + + TEST_CHECK(memcmp(stored_hash, corrupted_hash, sizeof(stored_hash)) == 0); } -/* - * Test case: Prevent unsigned underflow when writing large metadata to a - * chunk with small initial allocation. - * - * This test specifically validates the fix for the bug where calculating - * content_av = alloc_size - CIO_FILE_HEADER_MIN - size could underflow - * when size is large, causing the resize check to be skipped and leading - * to out-of-bounds writes. - * - * Scenario: - * - Create chunk with minimal initial size (e.g., ~100 bytes page-aligned) - * - Write small metadata (10 bytes) and small content (20 bytes) - * - Write large metadata (80 bytes) that would cause underflow: - * old code: content_av = 100 - 24 - 80 = -4 (wraps to huge unsigned) - * - Verify resize happens correctly and no buffer overrun occurs - */ -static void test_metadata_unsigned_underflow() +static void test_fs_shadow_checksum_recovers_primary_length() { int ret; int err; - char *meta_buf; - int meta_len; - void *content_buf; + char *out_buf; + size_t out_size; + struct cio_ctx *ctx; + struct cio_file *cf; + struct cio_chunk *chunk; + struct cio_stream *stream; + struct cio_options cio_opts; + char line[] = "shadow checksum payload\n"; + char tail[] = "after recovery\n"; + + cio_utils_recursive_delete(CIO_ENV); + + cio_options_init(&cio_opts); + + cio_opts.root_path = CIO_ENV; + cio_opts.log_cb = log_cb; + cio_opts.log_level = CIO_LOG_INFO; + cio_opts.flags = CIO_CHECKSUM; + + ctx = cio_create(&cio_opts); + TEST_CHECK(ctx != NULL); + + stream = cio_stream_create(ctx, "test_shadow_len", CIO_STORE_FS); + TEST_CHECK(stream != NULL); + + chunk = cio_chunk_open(ctx, stream, "test", CIO_OPEN, 1000, &err); + TEST_CHECK(chunk != NULL); + if (!chunk) { + exit(1); + } + + ret = cio_chunk_write(chunk, line, strlen(line)); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_sync(chunk); + TEST_CHECK(ret == CIO_OK); + + cf = (struct cio_file *) chunk->backend; + cio_file_st_set_content_len(cf->map, 5); + + ret = cio_file_native_sync(cf, CIO_FULL_SYNC); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_down(chunk); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_up(chunk); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_get_content_copy(chunk, (void **) &out_buf, &out_size); + TEST_CHECK(ret == CIO_OK); + TEST_CHECK(out_size == strlen(line)); + TEST_CHECK(memcmp(out_buf, line, out_size) == 0); + free(out_buf); + + ret = cio_chunk_write(chunk, tail, strlen(tail)); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_sync(chunk); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_down(chunk); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_up(chunk); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_get_content_copy(chunk, (void **) &out_buf, &out_size); + TEST_CHECK(ret == CIO_OK); + TEST_CHECK(out_size == strlen(line) + strlen(tail)); + TEST_CHECK(memcmp(out_buf, line, strlen(line)) == 0); + TEST_CHECK(memcmp(out_buf + strlen(line), tail, strlen(tail)) == 0); + free(out_buf); + + cio_destroy(ctx); +} + +static void test_fs_valid_primary_precedes_newer_shadow() +{ + int ret; + int err; + char *out_buf; + size_t out_size; + crc_t crc; + struct cio_ctx *ctx; + struct cio_file *cf; + struct cio_chunk *chunk; + struct cio_stream *stream; + struct cio_options cio_opts; + char line[] = "shadow checksum payload\n"; + char tail[] = "after recovery\n"; + + cio_utils_recursive_delete(CIO_ENV); + + cio_options_init(&cio_opts); + + cio_opts.root_path = CIO_ENV; + cio_opts.log_cb = log_cb; + cio_opts.log_level = CIO_LOG_INFO; + cio_opts.flags = CIO_CHECKSUM; + + ctx = cio_create(&cio_opts); + TEST_CHECK(ctx != NULL); + + stream = cio_stream_create(ctx, "test_shadow_stale", CIO_STORE_FS); + TEST_CHECK(stream != NULL); + + chunk = cio_chunk_open(ctx, stream, "test", CIO_OPEN, 1000, &err); + TEST_CHECK(chunk != NULL); + if (!chunk) { + exit(1); + } + + ret = cio_chunk_write(chunk, line, strlen(line)); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_sync(chunk); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_write(chunk, tail, strlen(tail)); + TEST_CHECK(ret == CIO_OK); + + /* Both tuples validate: without a generation marker the primary wins. */ + cf = (struct cio_file *) chunk->backend; + cf->crc_cur = cio_crc32_init(); + cio_file_calculate_checksum(cf, &crc); + cf->crc_cur = crc; + + set_test_header_u32(cf->map, + CIO_FILE_SHADOW_CONTENT_LENGTH_OFFSET, + (uint32_t) cf->data_size); + set_test_header_u32(cf->map, + CIO_FILE_SHADOW_HASH_OFFSET, + (uint32_t) cio_crc32_finalize(crc)); + + ret = cio_file_native_sync(cf, CIO_FULL_SYNC); + TEST_CHECK(ret == CIO_OK); + + cf->synced = CIO_TRUE; + + ret = cio_chunk_down(chunk); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_up(chunk); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_get_content_copy(chunk, (void **) &out_buf, &out_size); + TEST_CHECK(ret == CIO_OK); + TEST_CHECK(out_size == strlen(line)); + TEST_CHECK(memcmp(out_buf, line, strlen(line)) == 0); + free(out_buf); + + cio_destroy(ctx); +} + +/* Older writers update the primary tuple but leave shadow padding untouched. */ +static void check_stale_shadow_after_legacy_write(int truncate, int read_only) +{ + int ret; + int err; + char *out_buf; + size_t out_size; + size_t expected_size; + unsigned char shadow[8]; + unsigned char header[CIO_FILE_HEADER_MIN]; + struct cio_options opts; + struct cio_ctx *ctx; + struct cio_stream *stream; + struct cio_chunk *chunk; + struct cio_file *cf; + char head[] = "original records\n"; + char tail[] = "appended records\n"; + + cio_utils_recursive_delete(CIO_ENV); + cio_options_init(&opts); + opts.root_path = CIO_ENV; + opts.flags = CIO_CHECKSUM; + ctx = cio_create(&opts); + if (!TEST_CHECK(ctx != NULL)) { + exit(1); + } + stream = cio_stream_create(ctx, "legacy_writer", CIO_STORE_FS); + if (!TEST_CHECK(stream != NULL)) { + exit(1); + } + chunk = cio_chunk_open(ctx, stream, "test", CIO_OPEN, 1000, &err); + if (!TEST_CHECK(chunk != NULL)) { + exit(1); + } + ret = cio_chunk_write(chunk, head, strlen(head)); + if (!TEST_CHECK(ret == CIO_OK)) { + exit(1); + } + if (truncate) { + ret = cio_chunk_write(chunk, tail, strlen(tail)); + if (!TEST_CHECK(ret == CIO_OK)) { + exit(1); + } + } + ret = cio_chunk_sync(chunk); + if (!TEST_CHECK(ret == CIO_OK)) { + exit(1); + } + cf = chunk->backend; + memcpy(shadow, cf->map + CIO_FILE_SHADOW_CONTENT_LENGTH_OFFSET, sizeof(shadow)); + + if (truncate) { + ret = cio_chunk_write_at(chunk, strlen(head), NULL, 0); + expected_size = strlen(head); + } + else { + ret = cio_chunk_write(chunk, tail, strlen(tail)); + expected_size = strlen(head) + strlen(tail); + } + if (!TEST_CHECK(ret == CIO_OK)) { + exit(1); + } + ret = cio_chunk_sync(chunk); + if (!TEST_CHECK(ret == CIO_OK)) { + exit(1); + } + + /* Reproduce an old writer's valid primary with the previous shadow tuple. */ + memcpy(cf->map + CIO_FILE_SHADOW_CONTENT_LENGTH_OFFSET, shadow, sizeof(shadow)); + memcpy(header, cf->map, sizeof(header)); + ret = cio_file_native_sync(cf, CIO_FULL_SYNC); + if (!TEST_CHECK(ret == CIO_OK)) { + exit(1); + } + cio_destroy(ctx); + + cio_options_init(&opts); + opts.root_path = CIO_ENV; + opts.flags = CIO_CHECKSUM | (read_only ? CIO_OPEN_RD : 0); + ctx = cio_create(&opts); + if (!TEST_CHECK(ctx != NULL)) { + exit(1); + } + stream = cio_stream_create(ctx, "legacy_writer", CIO_STORE_FS); + if (!TEST_CHECK(stream != NULL)) { + exit(1); + } + chunk = cio_chunk_open(ctx, stream, "test", + read_only ? CIO_OPEN_RD : CIO_OPEN, 0, &err); + if (!TEST_CHECK(chunk != NULL)) { + exit(1); + } + cf = chunk->backend; + TEST_CHECK(memcmp(cf->map, header, sizeof(header)) == 0); + ret = cio_chunk_get_content_copy(chunk, (void **) &out_buf, &out_size); + if (!TEST_CHECK(ret == CIO_OK)) { + exit(1); + } + TEST_CHECK(out_size == expected_size); + if (out_size == expected_size) { + TEST_CHECK(memcmp(out_buf, head, strlen(head)) == 0); + if (!truncate) { + TEST_CHECK(memcmp(out_buf + strlen(head), tail, strlen(tail)) == 0); + } + } + free(out_buf); + cio_destroy(ctx); +} + +static void test_fs_valid_primary_with_stale_shadow_append() +{ + check_stale_shadow_after_legacy_write(CIO_FALSE, CIO_FALSE); +} + +static void test_fs_valid_primary_with_stale_shadow_truncate() +{ + check_stale_shadow_after_legacy_write(CIO_TRUE, CIO_FALSE); +} + +static void test_fs_valid_primary_with_stale_shadow_read_only_append() +{ + check_stale_shadow_after_legacy_write(CIO_FALSE, CIO_TRUE); +} + +static void test_fs_valid_primary_with_stale_shadow_read_only_truncate() +{ + check_stale_shadow_after_legacy_write(CIO_TRUE, CIO_TRUE); +} + +static void test_fs_write_at_zero_truncates_synced_checksum() +{ + int ret; + int err; + char *out_buf; + size_t out_size; + struct cio_ctx *ctx; + struct cio_file *cf; + struct cio_chunk *chunk; + struct cio_stream *stream; + struct cio_options cio_opts; + char head[] = "valid records\n"; + char stale[] = "stale records\n"; + char next[] = "next records\n"; + + cio_utils_recursive_delete(CIO_ENV); + + cio_options_init(&cio_opts); + + cio_opts.root_path = CIO_ENV; + cio_opts.log_cb = log_cb; + cio_opts.log_level = CIO_LOG_INFO; + cio_opts.flags = CIO_CHECKSUM; + + ctx = cio_create(&cio_opts); + TEST_CHECK(ctx != NULL); + + stream = cio_stream_create(ctx, "test_write_at_zero_crc", CIO_STORE_FS); + TEST_CHECK(stream != NULL); + + chunk = cio_chunk_open(ctx, stream, "test", CIO_OPEN, 1000, &err); + TEST_CHECK(chunk != NULL); + if (!chunk) { + exit(1); + } + + ret = cio_chunk_write(chunk, head, strlen(head)); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_write(chunk, stale, strlen(stale)); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_sync(chunk); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_write_at(chunk, strlen(head), NULL, 0); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_down(chunk); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_up(chunk); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_get_content_copy(chunk, (void **) &out_buf, &out_size); + TEST_CHECK(ret == CIO_OK); + TEST_CHECK(out_size == strlen(head)); + TEST_CHECK(memcmp(out_buf, head, out_size) == 0); + free(out_buf); + + ret = cio_chunk_write(chunk, next, strlen(next)); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_sync(chunk); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_down(chunk); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_up(chunk); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_get_content_copy(chunk, (void **) &out_buf, &out_size); + TEST_CHECK(ret == CIO_OK); + TEST_CHECK(out_size == strlen(head) + strlen(next)); + TEST_CHECK(memcmp(out_buf, head, strlen(head)) == 0); + TEST_CHECK(memcmp(out_buf + strlen(head), next, strlen(next)) == 0); + free(out_buf); + + cf = (struct cio_file *) chunk->backend; + TEST_CHECK(cf->map[CIO_FILE_LEGACY_CRC_PADDING_OFFSET + 0] == 0); + TEST_CHECK(cf->map[CIO_FILE_LEGACY_CRC_PADDING_OFFSET + 1] == 0); + TEST_CHECK(cf->map[CIO_FILE_LEGACY_CRC_PADDING_OFFSET + 2] == 0); + TEST_CHECK(cf->map[CIO_FILE_LEGACY_CRC_PADDING_OFFSET + 3] == 0); + + cio_destroy(ctx); +} + +static void test_fs_legacy_checksum_without_shadow_header() +{ + int ret; + int err; + char *out_buf; + size_t out_size; + struct cio_ctx *ctx; + struct cio_file *cf; + struct cio_chunk *chunk; + struct cio_stream *stream; + struct cio_options cio_opts; + char line[] = "legacy checksum payload\n"; + + cio_utils_recursive_delete(CIO_ENV); + + cio_options_init(&cio_opts); + + cio_opts.root_path = CIO_ENV; + cio_opts.log_cb = log_cb; + cio_opts.log_level = CIO_LOG_INFO; + cio_opts.flags = CIO_CHECKSUM; + + ctx = cio_create(&cio_opts); + TEST_CHECK(ctx != NULL); + + stream = cio_stream_create(ctx, "test_legacy_crc", CIO_STORE_FS); + TEST_CHECK(stream != NULL); + + chunk = cio_chunk_open(ctx, stream, "test", CIO_OPEN, 1000, &err); + TEST_CHECK(chunk != NULL); + if (!chunk) { + exit(1); + } + + ret = cio_chunk_write(chunk, line, strlen(line)); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_sync(chunk); + TEST_CHECK(ret == CIO_OK); + + cf = (struct cio_file *) chunk->backend; + TEST_CHECK(cf->map[CIO_FILE_LEGACY_CRC_PADDING_OFFSET + 0] == 0); + TEST_CHECK(cf->map[CIO_FILE_LEGACY_CRC_PADDING_OFFSET + 1] == 0); + TEST_CHECK(cf->map[CIO_FILE_LEGACY_CRC_PADDING_OFFSET + 2] == 0); + TEST_CHECK(cf->map[CIO_FILE_LEGACY_CRC_PADDING_OFFSET + 3] == 0); + + memset(cf->map + CIO_FILE_SHADOW_CONTENT_LENGTH_OFFSET, 0, 4); + memset(cf->map + CIO_FILE_SHADOW_HASH_OFFSET, 0, 4); + + ret = cio_file_native_sync(cf, CIO_FULL_SYNC); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_down(chunk); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_up(chunk); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_get_content_copy(chunk, (void **) &out_buf, &out_size); + TEST_CHECK(ret == CIO_OK); + TEST_CHECK(out_size == strlen(line)); + TEST_CHECK(memcmp(out_buf, line, out_size) == 0); + free(out_buf); + + cio_destroy(ctx); +} + +/* + * Forward compatibility: a chunk written in the legacy on-disk format (no shadow + * tuple, primary CRC + content length only) must be loaded correctly by the + * current code after a full restart (fresh context + cio_load), and the legacy + * chunk must NOT be rewritten/decorated with a shadow tuple just by reading it. + */ +static void test_fs_old_format_loads_via_restart() +{ + int ret; + int err; + char *buf; + size_t buf_size; + struct cio_ctx *ctx; + struct cio_file *cf; + struct cio_chunk *chunk; + struct cio_stream *stream; + struct cio_options cio_opts; + char line[] = "legacy on-disk chunk payload\n"; + + cio_utils_recursive_delete(CIO_ENV); + + cio_options_init(&cio_opts); + + cio_opts.root_path = CIO_ENV; + cio_opts.log_cb = log_cb; + cio_opts.log_level = CIO_LOG_INFO; + cio_opts.flags = CIO_OPEN | CIO_CHECKSUM; + + ctx = cio_create(&cio_opts); + TEST_CHECK(ctx != NULL); + if (!ctx) { + exit(1); + } + + stream = cio_stream_create(ctx, "test_old_format", CIO_STORE_FS); + TEST_CHECK(stream != NULL); + + chunk = cio_chunk_open(ctx, stream, "chunk.0", CIO_OPEN, 1000, &err); + TEST_CHECK(chunk != NULL); + if (!chunk) { + exit(1); + } + + ret = cio_chunk_write(chunk, line, strlen(line)); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_sync(chunk); + TEST_CHECK(ret == CIO_OK); + + /* + * Downgrade the header to the legacy layout: the primary CRC (bytes 2..5) + * and the content length (bytes 10..13) are already byte-identical to what + * old writers produced; the only legacy difference is the absence of the + * shadow tuple, so we zero bytes 14..21. + */ + cf = (struct cio_file *) chunk->backend; + memset(cf->map + CIO_FILE_SHADOW_CONTENT_LENGTH_OFFSET, 0, 4); + memset(cf->map + CIO_FILE_SHADOW_HASH_OFFSET, 0, 4); + + ret = cio_file_native_sync(cf, CIO_FULL_SYNC); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_down(chunk); + TEST_CHECK(ret == CIO_OK); + + cio_destroy(ctx); + + /* Restart: a fresh context discovers the legacy chunk on disk */ + cio_options_init(&cio_opts); + + cio_opts.root_path = CIO_ENV; + cio_opts.log_cb = log_cb; + cio_opts.log_level = CIO_LOG_INFO; + cio_opts.flags = CIO_OPEN | CIO_CHECKSUM; + + ctx = cio_create(&cio_opts); + TEST_CHECK(ctx != NULL); + if (!ctx) { + exit(1); + } + + ret = cio_load(ctx, NULL); + TEST_CHECK(ret == CIO_OK); + + stream = cio_stream_get(ctx, "test_old_format"); + TEST_CHECK(stream != NULL); + chunk = find_stream_chunk(stream, "chunk.0"); + TEST_CHECK(chunk != NULL); + if (!chunk) { + exit(1); + } + + if (cio_chunk_is_up(chunk) == CIO_FALSE) { + ret = cio_chunk_up_force(chunk); + TEST_CHECK(ret == CIO_OK); + } + + ret = cio_chunk_get_content(chunk, &buf, &buf_size); + TEST_CHECK(ret == CIO_OK); + TEST_CHECK(buf_size == strlen(line)); + TEST_CHECK(memcmp(buf, line, strlen(line)) == 0); + + /* + * Reading a valid legacy chunk must not have synthesized a shadow tuple: + * bytes 14..21 stay zero until the chunk is actually re-synced. + */ + cf = (struct cio_file *) chunk->backend; + TEST_CHECK(get_test_header_u32(cf->map, + CIO_FILE_SHADOW_CONTENT_LENGTH_OFFSET) == 0); + TEST_CHECK(get_test_header_u32(cf->map, CIO_FILE_SHADOW_HASH_OFFSET) == 0); + + cio_destroy(ctx); +} + +/* + * Shadow recovery must reject a tuple whose content length does not fit in the + * file: a corrupted primary CRC combined with an oversized shadow length must + * surface as CIO_CORRUPTED, never a silent accept of out-of-bounds content. + */ +static void check_shadow_rejects_oversized_length(int maximum_length) +{ + int ret; + int err; + char *hash; + struct cio_ctx *ctx; + struct cio_file *cf; + struct cio_chunk *chunk; + struct cio_stream *stream; + struct cio_options cio_opts; + char line[] = "shadow oversized payload\n"; + + cio_utils_recursive_delete(CIO_ENV); + + cio_options_init(&cio_opts); + + cio_opts.root_path = CIO_ENV; + cio_opts.log_cb = log_cb; + cio_opts.log_level = CIO_LOG_INFO; + cio_opts.flags = CIO_CHECKSUM; + + ctx = cio_create(&cio_opts); + TEST_CHECK(ctx != NULL); + + stream = cio_stream_create(ctx, "test_shadow_oversized", CIO_STORE_FS); + TEST_CHECK(stream != NULL); + + chunk = cio_chunk_open(ctx, stream, "test", CIO_OPEN, 1000, &err); + TEST_CHECK(chunk != NULL); + if (!chunk) { + exit(1); + } + + ret = cio_chunk_write(chunk, line, strlen(line)); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_sync(chunk); + TEST_CHECK(ret == CIO_OK); + + /* Corrupt the primary CRC so recovery must fall through to the shadow */ + cf = (struct cio_file *) chunk->backend; + hash = cio_file_hash(cf); + hash[0] ^= 0xff; + + /* Make the shadow content length larger than the file */ + set_test_header_u32(cf->map, CIO_FILE_SHADOW_CONTENT_LENGTH_OFFSET, + maximum_length ? UINT32_MAX : (uint32_t) (cf->fs_size + 4096)); + if (maximum_length) { + /* On 32-bit size_t, UINT32_MAX + the two metadata bytes wraps to 1. */ + set_test_header_u32(cf->map, CIO_FILE_SHADOW_HASH_OFFSET, + (uint32_t) cio_crc32_finalize(cio_crc32_update( + cio_crc32_init(), + cf->map + CIO_FILE_CONTENT_OFFSET, 1))); + } + + ret = cio_file_native_sync(cf, CIO_FULL_SYNC); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_down(chunk); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_up(chunk); + TEST_CHECK(ret == CIO_CORRUPTED); + TEST_CHECK(cio_error_get(chunk) == CIO_ERR_BAD_CHECKSUM); + + cf = (struct cio_file *) chunk->backend; + TEST_CHECK(cf->map == NULL); + + cio_destroy(ctx); +} + +static void test_fs_shadow_rejects_oversized_length() +{ + check_shadow_rejects_oversized_length(CIO_FALSE); +} + +static void test_fs_shadow_rejects_maximum_length() +{ + check_shadow_rejects_oversized_length(CIO_TRUE); +} + +/* + * Shadow recovery must not mask real content corruption: if a content byte is + * altered, neither the primary nor the shadow CRC validate against it and the + * chunk must be reported as corrupted. + */ +static void test_fs_shadow_rejects_corrupt_content() +{ + int ret; + int err; + struct cio_ctx *ctx; + struct cio_file *cf; + struct cio_chunk *chunk; + struct cio_stream *stream; + struct cio_options cio_opts; + char line[] = "shadow corrupt content payload\n"; + + cio_utils_recursive_delete(CIO_ENV); + + cio_options_init(&cio_opts); + + cio_opts.root_path = CIO_ENV; + cio_opts.log_cb = log_cb; + cio_opts.log_level = CIO_LOG_INFO; + cio_opts.flags = CIO_CHECKSUM; + + ctx = cio_create(&cio_opts); + TEST_CHECK(ctx != NULL); + + stream = cio_stream_create(ctx, "test_shadow_corrupt", CIO_STORE_FS); + TEST_CHECK(stream != NULL); + + chunk = cio_chunk_open(ctx, stream, "test", CIO_OPEN, 1000, &err); + TEST_CHECK(chunk != NULL); + if (!chunk) { + exit(1); + } + + ret = cio_chunk_write(chunk, line, strlen(line)); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_sync(chunk); + TEST_CHECK(ret == CIO_OK); + + /* + * Flip the first content byte. Both the primary and the (still valid) + * shadow CRC were computed over the original content, so neither matches + * the altered data. + */ + cf = (struct cio_file *) chunk->backend; + cf->map[CIO_FILE_HEADER_MIN] ^= 0xff; + + ret = cio_file_native_sync(cf, CIO_FULL_SYNC); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_down(chunk); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_up(chunk); + TEST_CHECK(ret == CIO_CORRUPTED); + TEST_CHECK(cio_error_get(chunk) == CIO_ERR_BAD_CHECKSUM); + + cf = (struct cio_file *) chunk->backend; + TEST_CHECK(cf->map == NULL); + + cio_destroy(ctx); +} + +/* + * When both the primary and the shadow CRC fields are damaged (even with intact + * content), recovery must fail closed rather than fabricate an accept. + */ +static void test_fs_shadow_rejects_both_tuples_corrupt() +{ + int ret; + int err; + char *hash; + struct cio_ctx *ctx; + struct cio_file *cf; + struct cio_chunk *chunk; + struct cio_stream *stream; + struct cio_options cio_opts; + char line[] = "shadow both corrupt payload\n"; + + cio_utils_recursive_delete(CIO_ENV); + + cio_options_init(&cio_opts); + + cio_opts.root_path = CIO_ENV; + cio_opts.log_cb = log_cb; + cio_opts.log_level = CIO_LOG_INFO; + cio_opts.flags = CIO_CHECKSUM; + + ctx = cio_create(&cio_opts); + TEST_CHECK(ctx != NULL); + + stream = cio_stream_create(ctx, "test_shadow_both", CIO_STORE_FS); + TEST_CHECK(stream != NULL); + + chunk = cio_chunk_open(ctx, stream, "test", CIO_OPEN, 1000, &err); + TEST_CHECK(chunk != NULL); + if (!chunk) { + exit(1); + } + + ret = cio_chunk_write(chunk, line, strlen(line)); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_sync(chunk); + TEST_CHECK(ret == CIO_OK); + + /* Corrupt the primary CRC and the shadow CRC, leaving content intact */ + cf = (struct cio_file *) chunk->backend; + hash = cio_file_hash(cf); + hash[0] ^= 0xff; + cf->map[CIO_FILE_SHADOW_HASH_OFFSET] ^= 0xff; + + ret = cio_file_native_sync(cf, CIO_FULL_SYNC); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_down(chunk); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_up(chunk); + TEST_CHECK(ret == CIO_CORRUPTED); + TEST_CHECK(cio_error_get(chunk) == CIO_ERR_BAD_CHECKSUM); + + cf = (struct cio_file *) chunk->backend; + TEST_CHECK(cf->map == NULL); + + cio_destroy(ctx); +} + +/* + * Wire-format contract + rollback safety. + * + * Pins the exact on-disk header bytes a freshly synced checksum chunk produces, + * computed independently from the documented format (CRC32 of the content + * section, finalized, stored big-endian at offset 2): + * + * - bytes 2..5 : finalized CRC32, big-endian + * - bytes 6..9 : kept zero + * - bytes 10..13: content length, big-endian + * + * Rollback safety scope: an OLDER chunkio binary that reads these chunks after + * a downgrade compares sizeof(crc_t) bytes starting at offset 2. + * - little-endian 64-bit (x86_64, aarch64) and big-endian 32-bit readers: + * compatible. They expect the significant CRC bytes at offset 2 and zeros + * filling the rest, which is exactly what we write, so the chunk is + * accepted. This covers every architecture Fluent Bit ships on. + * - big-endian 64-bit readers (s390x, ppc64be): NOT rollback compatible. With + * an 8-byte crc_t those builds expected the significant bytes at offset 6..9 + * and zeros at 2..5 -- the mirror of the canonical layout. The canonical + * format was deliberately chosen to be portable across architectures + * (forward compatible: see fs_legacy_big_endian_wide_checksum_migrates, + * which lets the current code READ such legacy chunks), at the cost of + * downgrade compatibility on big-endian 64-bit specifically. + * + * If any of the asserted bytes drift, old little-endian binaries would reject + * chunks written by new ones. + */ +static void test_fs_format_contract_new_chunk_layout() +{ + int ret; + int err; + crc_t crc; + uint32_t expected_crc; + uint8_t region[2 + 64]; + uint8_t legacy_field[8]; + size_t region_len; + struct cio_ctx *ctx; + struct cio_file *cf; + struct cio_chunk *chunk; + struct cio_stream *stream; + struct cio_options cio_opts; + char line[] = "wire format contract payload\n"; + + cio_utils_recursive_delete(CIO_ENV); + + cio_options_init(&cio_opts); + + cio_opts.root_path = CIO_ENV; + cio_opts.log_cb = log_cb; + cio_opts.log_level = CIO_LOG_INFO; + cio_opts.flags = CIO_CHECKSUM; + + ctx = cio_create(&cio_opts); + TEST_CHECK(ctx != NULL); + + stream = cio_stream_create(ctx, "test_contract", CIO_STORE_FS); + TEST_CHECK(stream != NULL); + + chunk = cio_chunk_open(ctx, stream, "test", CIO_OPEN, 1000, &err); + TEST_CHECK(chunk != NULL); + if (!chunk) { + exit(1); + } + + ret = cio_chunk_write(chunk, line, strlen(line)); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_sync(chunk); + TEST_CHECK(ret == CIO_OK); + + /* + * Independently compute the CRC the way the format documents it: over the + * content section which begins with the 2-byte metadata-length field + * (zero here, no metadata) followed by the user content. + */ + region[0] = 0x00; + region[1] = 0x00; + memcpy(region + 2, line, strlen(line)); + region_len = 2 + strlen(line); + + crc = cio_crc32_init(); + crc = cio_crc32_update(crc, region, region_len); + expected_crc = (uint32_t) cio_crc32_finalize(crc); + + cf = (struct cio_file *) chunk->backend; + + /* + * Assert the raw bytes are stored most-significant-byte first, independent + * of host endianness. These per-byte checks fail on a little-endian host if + * the implementation were ever changed to a native memcpy (which would put + * the least-significant byte at offset 2), so they guard portability even + * when CI only runs on little-endian. + */ + TEST_CHECK((uint8_t) cf->map[2] == (uint8_t) ((expected_crc >> 24) & 0xFF)); + TEST_CHECK((uint8_t) cf->map[3] == (uint8_t) ((expected_crc >> 16) & 0xFF)); + TEST_CHECK((uint8_t) cf->map[4] == (uint8_t) ((expected_crc >> 8) & 0xFF)); + TEST_CHECK((uint8_t) cf->map[5] == (uint8_t) ((expected_crc >> 0) & 0xFF)); + + /* Primary CRC at bytes 2..5, big-endian */ + TEST_CHECK(get_test_header_u32(cf->map, 2) == expected_crc); + + /* Legacy CRC padding bytes 6..9 must stay zero (little-endian rollback) */ + TEST_CHECK(get_test_header_u32(cf->map, + CIO_FILE_LEGACY_CRC_PADDING_OFFSET) == 0); + + /* Content length at bytes 10..13, big-endian (most-significant byte first) */ + TEST_CHECK((uint8_t) cf->map[CIO_FILE_CONTENT_LENGTH_OFFSET + 0] == + (uint8_t) (((uint32_t) strlen(line) >> 24) & 0xFF)); + TEST_CHECK((uint8_t) cf->map[CIO_FILE_CONTENT_LENGTH_OFFSET + 1] == + (uint8_t) (((uint32_t) strlen(line) >> 16) & 0xFF)); + TEST_CHECK((uint8_t) cf->map[CIO_FILE_CONTENT_LENGTH_OFFSET + 2] == + (uint8_t) (((uint32_t) strlen(line) >> 8) & 0xFF)); + TEST_CHECK((uint8_t) cf->map[CIO_FILE_CONTENT_LENGTH_OFFSET + 3] == + (uint8_t) (((uint32_t) strlen(line) >> 0) & 0xFF)); + TEST_CHECK(get_test_header_u32(cf->map, CIO_FILE_CONTENT_LENGTH_OFFSET) == + (uint32_t) strlen(line)); + + /* + * Emulate the exact comparison an old 64-bit reader performed: + * memcmp(map + 2, &crc_check, sizeof(crc_t)) where crc_check held the + * big-endian CRC in its low 4 bytes and zero in the upper 4. The new chunk + * must satisfy that 8-byte comparison. + */ + legacy_field[0] = (uint8_t) ((expected_crc & 0xFF000000) >> 24); + legacy_field[1] = (uint8_t) ((expected_crc & 0x00FF0000) >> 16); + legacy_field[2] = (uint8_t) ((expected_crc & 0x0000FF00) >> 8); + legacy_field[3] = (uint8_t) ((expected_crc & 0x000000FF) >> 0); + legacy_field[4] = 0; + legacy_field[5] = 0; + legacy_field[6] = 0; + legacy_field[7] = 0; + TEST_CHECK(memcmp(cf->map + 2, legacy_field, sizeof(legacy_field)) == 0); + + cio_destroy(ctx); +} + +/* + * Golden legacy fixture: a chunk whose header is constructed by hand in the + * pre-shadow on-disk format (no shadow tuple), independent of the current + * writer, must be discovered and read correctly after a restart. This guards + * the millions of chunks already on disk from prior chunkio releases. + */ +static void test_fs_golden_legacy_chunk_loads() +{ + int ret; + int err; + FILE *fp; + char *buf; + size_t buf_size; + crc_t crc; + uint32_t crc_final; + uint8_t header[CIO_FILE_HEADER_MIN]; + uint8_t region[2 + 64]; + size_t region_len; + char path[PATH_MAX]; + struct cio_ctx *ctx; + struct cio_chunk *chunk; + struct cio_stream *stream; + struct cio_options cio_opts; + char line[] = "golden legacy chunk payload\n"; + + cio_utils_recursive_delete(CIO_ENV); + + cio_options_init(&cio_opts); + + cio_opts.root_path = CIO_ENV; + cio_opts.log_cb = log_cb; + cio_opts.log_level = CIO_LOG_INFO; + cio_opts.flags = CIO_OPEN | CIO_CHECKSUM; + + ctx = cio_create(&cio_opts); + TEST_CHECK(ctx != NULL); + if (!ctx) { + exit(1); + } + + /* Create a real chunk so the stream directory and naming exist */ + stream = cio_stream_create(ctx, "test_golden", CIO_STORE_FS); + TEST_CHECK(stream != NULL); + + chunk = cio_chunk_open(ctx, stream, "chunk.0", CIO_OPEN, 1000, &err); + TEST_CHECK(chunk != NULL); + if (!chunk) { + exit(1); + } + + ret = cio_chunk_write(chunk, "placeholder", 11); + TEST_CHECK(ret == CIO_OK); + ret = cio_chunk_sync(chunk); + TEST_CHECK(ret == CIO_OK); + + snprintf(path, sizeof(path), "%s/%s/%s", CIO_ENV, "test_golden", "chunk.0"); + + ret = cio_chunk_down(chunk); + TEST_CHECK(ret == CIO_OK); + + cio_destroy(ctx); + + /* + * Overwrite the chunk file with a hand-built legacy image: + * [0..1] 0xC1 0x00 magic + * [2..5] finalized CRC32 BE over (2-byte meta len + content) + * [6..9] 0 legacy CRC padding + * [10..13] content length BE + * [14..21] 0 no shadow tuple (legacy) + * [22..23] 0 metadata length + * [24..] content + */ + region[0] = 0x00; + region[1] = 0x00; + memcpy(region + 2, line, strlen(line)); + region_len = 2 + strlen(line); + + crc = cio_crc32_init(); + crc = cio_crc32_update(crc, region, region_len); + crc_final = (uint32_t) cio_crc32_finalize(crc); + + memset(header, 0, sizeof(header)); + header[0] = 0xc1; + header[1] = 0x00; + header[2] = (uint8_t) ((crc_final & 0xFF000000) >> 24); + header[3] = (uint8_t) ((crc_final & 0x00FF0000) >> 16); + header[4] = (uint8_t) ((crc_final & 0x0000FF00) >> 8); + header[5] = (uint8_t) ((crc_final & 0x000000FF) >> 0); + /* content length at offset 10, big-endian */ + header[10] = (uint8_t) (((uint32_t) strlen(line) & 0xFF000000) >> 24); + header[11] = (uint8_t) (((uint32_t) strlen(line) & 0x00FF0000) >> 16); + header[12] = (uint8_t) (((uint32_t) strlen(line) & 0x0000FF00) >> 8); + header[13] = (uint8_t) (((uint32_t) strlen(line) & 0x000000FF) >> 0); + + fp = fopen(path, "wb"); + TEST_CHECK(fp != NULL); + if (!fp) { + exit(1); + } + TEST_CHECK(fwrite(header, 1, sizeof(header), fp) == sizeof(header)); + TEST_CHECK(fwrite(line, 1, strlen(line), fp) == strlen(line)); + fclose(fp); + + /* Restart and discover the hand-built legacy chunk */ + cio_options_init(&cio_opts); + + cio_opts.root_path = CIO_ENV; + cio_opts.log_cb = log_cb; + cio_opts.log_level = CIO_LOG_INFO; + cio_opts.flags = CIO_OPEN | CIO_CHECKSUM; + + ctx = cio_create(&cio_opts); + TEST_CHECK(ctx != NULL); + if (!ctx) { + exit(1); + } + + ret = cio_load(ctx, NULL); + TEST_CHECK(ret == CIO_OK); + + stream = cio_stream_get(ctx, "test_golden"); + TEST_CHECK(stream != NULL); + chunk = find_stream_chunk(stream, "chunk.0"); + TEST_CHECK(chunk != NULL); + if (!chunk) { + exit(1); + } + + if (cio_chunk_is_up(chunk) == CIO_FALSE) { + ret = cio_chunk_up_force(chunk); + TEST_CHECK(ret == CIO_OK); + } + + ret = cio_chunk_get_content(chunk, &buf, &buf_size); + TEST_CHECK(ret == CIO_OK); + TEST_CHECK(buf_size == strlen(line)); + TEST_CHECK(memcmp(buf, line, strlen(line)) == 0); + + cio_destroy(ctx); +} + +/* + * Cross-endianness upgrade: an older 64-bit BIG-ENDIAN build stored the + * finalized crc as a full 8-byte crc_t, landing the significant bytes at + * offset 6..9 with the canonical slot (offset 2..5) left zero. This test + * reproduces that exact on-disk layout (in an endianness-independent way, so it + * runs on any host), then verifies the current code recovers the content and + * migrates the chunk into the canonical portable layout. + */ +static void test_fs_legacy_big_endian_wide_checksum_migrates() +{ + int ret; + int err; + char *out_buf; + size_t out_size; + uint32_t real_crc; + struct cio_ctx *ctx; + struct cio_file *cf; + struct cio_chunk *chunk; + struct cio_stream *stream; + struct cio_options cio_opts; + char line[] = "legacy big-endian wide crc payload\n"; + + cio_utils_recursive_delete(CIO_ENV); + + cio_options_init(&cio_opts); + + cio_opts.root_path = CIO_ENV; + cio_opts.log_cb = log_cb; + cio_opts.log_level = CIO_LOG_INFO; + cio_opts.flags = CIO_CHECKSUM; + + ctx = cio_create(&cio_opts); + TEST_CHECK(ctx != NULL); + + stream = cio_stream_create(ctx, "test_legacy_be64", CIO_STORE_FS); + TEST_CHECK(stream != NULL); + + chunk = cio_chunk_open(ctx, stream, "test", CIO_OPEN, 1000, &err); + TEST_CHECK(chunk != NULL); + if (!chunk) { + exit(1); + } + + ret = cio_chunk_write(chunk, line, strlen(line)); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_sync(chunk); + TEST_CHECK(ret == CIO_OK); + + cf = (struct cio_file *) chunk->backend; + + /* + * Rewrite the header into the legacy 64-bit big-endian layout: + * - move the finalized crc from the canonical slot (2..5) to 6..9 + * - zero the canonical slot + * - drop the shadow tuple (it did not exist in that era) + */ + real_crc = get_test_header_u32(cf->map, 2); + set_test_header_u32(cf->map, CIO_FILE_LEGACY_CRC_PADDING_OFFSET, real_crc); + set_test_header_u32(cf->map, 2, 0); + memset(cf->map + CIO_FILE_SHADOW_CONTENT_LENGTH_OFFSET, 0, 4); + memset(cf->map + CIO_FILE_SHADOW_HASH_OFFSET, 0, 4); + + ret = cio_file_native_sync(cf, CIO_FULL_SYNC); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_down(chunk); + TEST_CHECK(ret == CIO_OK); + + /* Bring it up read/write: recovery + migration to canonical layout */ + ret = cio_chunk_up(chunk); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_get_content_copy(chunk, (void **) &out_buf, &out_size); + TEST_CHECK(ret == CIO_OK); + TEST_CHECK(out_size == strlen(line)); + TEST_CHECK(memcmp(out_buf, line, out_size) == 0); + free(out_buf); + + /* After migration the crc must live in the canonical slot, padding zeroed */ + cf = (struct cio_file *) chunk->backend; + TEST_CHECK(get_test_header_u32(cf->map, 2) == real_crc); + TEST_CHECK(get_test_header_u32(cf->map, + CIO_FILE_LEGACY_CRC_PADDING_OFFSET) == 0); + + /* And it must keep validating through a normal restart cycle */ + ret = cio_chunk_down(chunk); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_up(chunk); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_get_content_copy(chunk, (void **) &out_buf, &out_size); + TEST_CHECK(ret == CIO_OK); + TEST_CHECK(out_size == strlen(line)); + TEST_CHECK(memcmp(out_buf, line, out_size) == 0); + free(out_buf); + + cio_destroy(ctx); +} + +/* + * Read-only counterpart of the legacy big-endian recovery: opening such a + * chunk read-only must recover the content in memory but must NOT rewrite the + * header (no migration to the canonical layout). The on-disk bytes 2..9 must be + * byte-for-byte identical before and after the read-only open. + */ +static void test_fs_legacy_big_endian_wide_checksum_read_only() +{ + int ret; + int err; + FILE *fp; + char *out_buf; + size_t out_size; + uint32_t real_crc; + unsigned char before[8]; + unsigned char after[8]; + char path[1024]; + struct cio_ctx *ctx; + struct cio_file *cf; + struct cio_chunk *chunk; + struct cio_stream *stream; + struct cio_options cio_opts; + char line[] = "legacy big-endian read-only payload\n"; + + cio_utils_recursive_delete(CIO_ENV); + + cio_options_init(&cio_opts); + + cio_opts.root_path = CIO_ENV; + cio_opts.log_cb = log_cb; + cio_opts.log_level = CIO_LOG_INFO; + cio_opts.flags = CIO_CHECKSUM; + + ctx = cio_create(&cio_opts); + TEST_CHECK(ctx != NULL); + + stream = cio_stream_create(ctx, "test_legacy_be64_ro", CIO_STORE_FS); + TEST_CHECK(stream != NULL); + + chunk = cio_chunk_open(ctx, stream, "test", CIO_OPEN, 1000, &err); + TEST_CHECK(chunk != NULL); + if (!chunk) { + exit(1); + } + + ret = cio_chunk_write(chunk, line, strlen(line)); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_sync(chunk); + TEST_CHECK(ret == CIO_OK); + + /* Reproduce the legacy 64-bit big-endian layout (crc at 6..9, no shadow) */ + cf = (struct cio_file *) chunk->backend; + real_crc = get_test_header_u32(cf->map, 2); + set_test_header_u32(cf->map, CIO_FILE_LEGACY_CRC_PADDING_OFFSET, real_crc); + set_test_header_u32(cf->map, 2, 0); + memset(cf->map + CIO_FILE_SHADOW_CONTENT_LENGTH_OFFSET, 0, 4); + memset(cf->map + CIO_FILE_SHADOW_HASH_OFFSET, 0, 4); + + ret = cio_file_native_sync(cf, CIO_FULL_SYNC); + TEST_CHECK(ret == CIO_OK); + + snprintf(path, sizeof(path), "%s/%s/%s", CIO_ENV, + "test_legacy_be64_ro", "test"); + + cio_destroy(ctx); + + /* Snapshot on-disk bytes 2..9 before the read-only open */ + fp = fopen(path, "rb"); + TEST_CHECK(fp != NULL); + if (!fp) { + exit(1); + } + TEST_CHECK(fseek(fp, 2, SEEK_SET) == 0); + TEST_CHECK(fread(before, 1, sizeof(before), fp) == sizeof(before)); + fclose(fp); + + /* Open read-only: content recovered in memory, nothing persisted */ + cio_options_init(&cio_opts); + + cio_opts.root_path = CIO_ENV; + cio_opts.log_cb = log_cb; + cio_opts.log_level = CIO_LOG_INFO; + cio_opts.flags = CIO_OPEN_RD | CIO_CHECKSUM; + + ctx = cio_create(&cio_opts); + TEST_CHECK(ctx != NULL); + + stream = cio_stream_create(ctx, "test_legacy_be64_ro", CIO_STORE_FS); + TEST_CHECK(stream != NULL); + + chunk = cio_chunk_open(ctx, stream, "test", CIO_OPEN_RD, 0, &err); + TEST_CHECK(chunk != NULL); + if (!chunk) { + exit(1); + } + + ret = cio_chunk_get_content_copy(chunk, (void **) &out_buf, &out_size); + TEST_CHECK(ret == CIO_OK); + TEST_CHECK(out_size == strlen(line)); + TEST_CHECK(memcmp(out_buf, line, out_size) == 0); + free(out_buf); + + cio_destroy(ctx); + + /* On-disk header must be untouched: still legacy layout, not migrated */ + fp = fopen(path, "rb"); + TEST_CHECK(fp != NULL); + if (!fp) { + exit(1); + } + TEST_CHECK(fseek(fp, 2, SEEK_SET) == 0); + TEST_CHECK(fread(after, 1, sizeof(after), fp) == sizeof(after)); + fclose(fp); + + TEST_CHECK(memcmp(before, after, sizeof(before)) == 0); + /* + * after[] starts at file offset 2, so after[0..3] is the canonical slot + * (offsets 2..5) and after[4..7] is the legacy slot (offsets 6..9). The + * canonical slot must still be empty and the crc must still live in the + * legacy slot -- i.e. no migration happened on a read-only open. + */ + TEST_CHECK(get_test_header_u32((char *) after, 0) == 0); + TEST_CHECK(get_test_header_u32((char *) after, 4) == real_crc); +} + + +void test_fs_up_down_up_append() +{ + int ret; + int err; + struct cio_ctx *ctx; + struct cio_chunk *chunk; + struct cio_stream *stream; + struct cio_options cio_opts; + + void *out_buf; + size_t out_size; + + cio_utils_recursive_delete(CIO_ENV); + + /* create Chunk I/O context */ + cio_options_init(&cio_opts); + + cio_opts.root_path = CIO_ENV; + cio_opts.log_cb = log_cb; + cio_opts.log_level = CIO_LOG_DEBUG; + cio_opts.flags = CIO_CHECKSUM; + + /* Create a temporal storage */ + ctx = cio_create(&cio_opts); + stream = cio_stream_create(ctx, "cio", CIO_STORE_FS); + chunk = cio_chunk_open(ctx, stream, "c", CIO_OPEN, 1000, &err); + TEST_CHECK(chunk != NULL); + if (!chunk) { + printf("cannot open chunk\n"); + exit(1); + } + + ret = cio_chunk_get_content_copy(chunk, &out_buf, &out_size); + TEST_CHECK(ret == CIO_OK); + TEST_CHECK(memcmp(out_buf, "", 1) == 0); + TEST_CHECK(out_size == 0); + free(out_buf); + + ret = cio_chunk_write(chunk, "line 1\n", 7); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_get_content_copy(chunk, &out_buf, &out_size); + TEST_CHECK(ret == CIO_OK); + TEST_CHECK(memcmp(out_buf, "line 1\n", 7+1) == 0); + TEST_CHECK(out_size == 7); + free(out_buf); + + ret = cio_chunk_down(chunk); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_up(chunk); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_get_content_copy(chunk, &out_buf, &out_size); + TEST_CHECK(ret == CIO_OK); + TEST_CHECK(memcmp(out_buf, "line 1\n", 7+1) == 0); + TEST_CHECK(out_size == 7); + free(out_buf); + + /* append */ + ret = cio_chunk_write(chunk, "line 2\n", 7); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_down(chunk); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_up(chunk); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_get_content_copy(chunk, &out_buf, &out_size); + TEST_CHECK(ret == CIO_OK); + TEST_CHECK(memcmp(out_buf, "line 1\nline 2\n", 7*2+1) == 0); + TEST_CHECK(out_size == 7*2); + free(out_buf); + + cio_destroy(ctx); +} + +static void test_deep_hierarchy() +{ + int i; + int ret; + int err; + int len; + char line[] = "this is a test line\n"; + struct cio_ctx *ctx; + struct cio_chunk *chunk; + struct cio_stream *stream; + struct cio_options cio_opts; + + cio_utils_recursive_delete("tmp"); + + /* Create a temporal storage */ + cio_options_init(&cio_opts); + + cio_opts.root_path = "tmp/deep/log/dir"; + cio_opts.log_cb = log_cb; + cio_opts.log_level = CIO_LOG_DEBUG; + cio_opts.flags = 0; + + ctx = cio_create(&cio_opts); + stream = cio_stream_create(ctx, "test", CIO_STORE_FS); + chunk = cio_chunk_open(ctx, stream, "c", CIO_OPEN, 1000, &err); + TEST_CHECK(chunk != NULL); + if (!chunk) { + printf("cannot open chunk\n"); + exit(1); + } + + len = strlen(line); + for (i = 0; i < 1000; i++) { + ret = cio_chunk_write(chunk, line, len); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_down(chunk); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_up(chunk); + TEST_CHECK(ret == CIO_OK); + } + + cio_destroy(ctx); +} + +static void truncate_file(struct cio_file *chunk_file, + size_t new_file_size, + int remove_content_length) +{ + int result; + + result = cio_file_native_open(chunk_file); + TEST_CHECK(result == CIO_OK); + + result = cio_file_native_map(chunk_file, + chunk_file->page_size); + TEST_CHECK(result == CIO_OK); + + if (remove_content_length) { + chunk_file->map[CIO_FILE_CONTENT_LENGTH_OFFSET + 0] = 0; + chunk_file->map[CIO_FILE_CONTENT_LENGTH_OFFSET + 1] = 0; + chunk_file->map[CIO_FILE_CONTENT_LENGTH_OFFSET + 2] = 0; + chunk_file->map[CIO_FILE_CONTENT_LENGTH_OFFSET + 3] = 0; + + chunk_file->map[CIO_FILE_SHADOW_CONTENT_LENGTH_OFFSET + 0] = 0; + chunk_file->map[CIO_FILE_SHADOW_CONTENT_LENGTH_OFFSET + 1] = 0; + chunk_file->map[CIO_FILE_SHADOW_CONTENT_LENGTH_OFFSET + 2] = 0; + chunk_file->map[CIO_FILE_SHADOW_CONTENT_LENGTH_OFFSET + 3] = 0; + chunk_file->map[CIO_FILE_SHADOW_HASH_OFFSET + 0] = 0; + chunk_file->map[CIO_FILE_SHADOW_HASH_OFFSET + 1] = 0; + chunk_file->map[CIO_FILE_SHADOW_HASH_OFFSET + 2] = 0; + chunk_file->map[CIO_FILE_SHADOW_HASH_OFFSET + 3] = 0; + } + + result = cio_file_native_unmap(chunk_file); + TEST_CHECK(result == CIO_OK); + + result = cio_file_native_resize(chunk_file, new_file_size); + TEST_CHECK(result == 0); + + result = cio_file_native_close(chunk_file); + TEST_CHECK(result == CIO_OK); +} + +static void test_legacy_core(int trigger_checksum_error) +{ + struct cio_options cio_opts; + char *in_data; + size_t in_size; + struct cio_stream *stream; + struct cio_chunk *chunk; + size_t delta; + struct cio_ctx *ctx; + int ret; + + /* delete any previous temporary content directory */ + cio_utils_recursive_delete(CIO_ENV); + /* + * Load sample data file and with the same content through multiple write + * operations generating other files. + */ + ret = cio_utils_read_file(CIO_FILE_400KB, &in_data, &in_size); + TEST_CHECK(ret == 0); + if (ret == -1) { + exit(EXIT_FAILURE); + } + + /* create Chunk I/O context */ + cio_options_init(&cio_opts); + + cio_opts.root_path = CIO_ENV; + cio_opts.log_cb = log_cb; + cio_opts.log_level = CIO_LOG_DEBUG; + cio_opts.flags = CIO_CHECKSUM; + + /* Create a temporal storage */ + ctx = cio_create(&cio_opts); + + stream = cio_stream_create(ctx, "test-legacy", CIO_STORE_FS); + + /* do not force a maximum of chunks up, we want to test writing overhead */ + cio_set_max_chunks_up(ctx, 1); + + chunk = cio_chunk_open(ctx, + stream, + "test_chunk", + CIO_OPEN, + 1000, + &ret); + + ret = cio_chunk_write(chunk, in_data, 128); + TEST_CHECK(ret == 0); + + ret = cio_chunk_down(chunk); + TEST_CHECK(ret == CIO_OK); + + delta = CIO_FILE_HEADER_MIN; + + if (trigger_checksum_error) { + delta++; + } + + truncate_file((struct cio_file *) chunk->backend, + 128 + delta, + CIO_TRUE); + + ret = cio_chunk_up(chunk); + + if (trigger_checksum_error) { + TEST_CHECK(ret != CIO_OK); + } + else { + TEST_CHECK(ret == CIO_OK); + } + + cio_destroy(ctx); + + free(in_data); +} + +void test_legacy_success() +{ + test_legacy_core(CIO_FALSE); +} + +void test_legacy_failure() +{ + test_legacy_core(CIO_TRUE); +} + +/* + * Exercise the filesystem/checksum call pattern used by Fluent Bit storage: + * create a chunk with metadata, reload it with cio_load(), bring it up for + * backlog processing, lock/down it, truncate the valid content prefix through + * cio_chunk_write_at(..., NULL, 0), reload again, append, and copy to a DLQ + * style stream. + */ +static void test_fs_fluent_bit_storage_lifecycle() +{ + int ret; + int err; + int meta_len; + char *buf; + char *meta_buf; + void *copy_buf; + size_t buf_size; + size_t copy_size; + size_t valid_size; + ssize_t content_size; + ssize_t real_size; + struct stat st; + struct cio_ctx *ctx; + struct cio_chunk *chunk; + struct cio_chunk *dlq_chunk; + struct cio_file *cf; + struct cio_stream *stream; + struct cio_stream *dlq_stream; + struct cio_options cio_opts; + char path[1024]; + char meta[] = "event_type=logs;tag=fb.test"; + char valid[] = "record one\nrecord two\n"; + char stale[] = "partial trailing record"; + char next[] = "record three\n"; + + valid_size = strlen(valid); + copy_buf = NULL; + + cio_utils_recursive_delete(CIO_ENV); + + cio_options_init(&cio_opts); + cio_opts.root_path = CIO_ENV; + cio_opts.log_cb = log_cb; + cio_opts.log_level = CIO_LOG_INFO; + cio_opts.flags = CIO_OPEN | CIO_CHECKSUM | CIO_FULL_SYNC | CIO_TRIM_FILES; + + ctx = cio_create(&cio_opts); + TEST_CHECK(ctx != NULL); + if (!ctx) { + exit(1); + } + + cio_set_max_chunks_up(ctx, 1); + + stream = cio_stream_create(ctx, "fluent-bit", CIO_STORE_FS); + TEST_CHECK(stream != NULL); + + chunk = cio_chunk_open(ctx, stream, "chunk.0", CIO_OPEN, 64, &err); + TEST_CHECK(chunk != NULL); + if (!chunk) { + exit(1); + } + + ret = cio_meta_write(chunk, meta, strlen(meta)); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_write(chunk, valid, strlen(valid)); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_write(chunk, stale, strlen(stale)); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_sync(chunk); + TEST_CHECK(ret == CIO_OK); + + snprintf(path, sizeof(path), "%s/%s/%s", CIO_ENV, "fluent-bit", "chunk.0"); + ret = stat(path, &st); + TEST_CHECK(ret == 0); + real_size = cio_chunk_get_real_size(chunk); + TEST_CHECK(real_size == st.st_size); + + content_size = cio_chunk_get_content_size(chunk); + TEST_CHECK(content_size == (ssize_t) (strlen(valid) + strlen(stale))); + + ret = cio_chunk_down(chunk); + TEST_CHECK(ret == CIO_OK); + + cio_destroy(ctx); + + cio_options_init(&cio_opts); + cio_opts.root_path = CIO_ENV; + cio_opts.log_cb = log_cb; + cio_opts.log_level = CIO_LOG_INFO; + cio_opts.flags = CIO_OPEN | CIO_CHECKSUM | CIO_FULL_SYNC | CIO_TRIM_FILES; + + ctx = cio_create(&cio_opts); + TEST_CHECK(ctx != NULL); + if (!ctx) { + exit(1); + } + + cio_set_max_chunks_up(ctx, 1); + + ret = cio_load(ctx, NULL); + TEST_CHECK(ret == CIO_OK); + + stream = cio_stream_get(ctx, "fluent-bit"); + TEST_CHECK(stream != NULL); + chunk = find_stream_chunk(stream, "chunk.0"); + TEST_CHECK(chunk != NULL); + if (!chunk) { + exit(1); + } + + if (cio_chunk_is_up(chunk) == CIO_TRUE) { + ret = cio_chunk_down(chunk); + TEST_CHECK(ret == CIO_OK); + } + + if (cio_chunk_is_up(chunk) == CIO_FALSE) { + ret = cio_chunk_up_force(chunk); + TEST_CHECK(ret == CIO_OK); + } + + ret = cio_meta_read(chunk, &meta_buf, &meta_len); + TEST_CHECK(ret == CIO_OK); + TEST_CHECK(meta_len == (int) strlen(meta)); + TEST_CHECK(memcmp(meta_buf, meta, strlen(meta)) == 0); + + ret = cio_chunk_get_content(chunk, &buf, &buf_size); + TEST_CHECK(ret == CIO_OK); + TEST_CHECK(buf_size == strlen(valid) + strlen(stale)); + TEST_CHECK(memcmp(buf, valid, strlen(valid)) == 0); + TEST_CHECK(memcmp(buf + strlen(valid), stale, strlen(stale)) == 0); + + ret = cio_chunk_lock(chunk); + TEST_CHECK(ret == CIO_OK); + TEST_CHECK(cio_chunk_is_locked(chunk) == CIO_TRUE); + + ret = cio_chunk_down(chunk); + TEST_CHECK(ret == CIO_OK); + + if (cio_chunk_is_up(chunk) == CIO_FALSE) { + ret = cio_chunk_up_force(chunk); + TEST_CHECK(ret == CIO_OK); + } + + ret = cio_chunk_write_at(chunk, (off_t) valid_size, NULL, 0); + TEST_CHECK(ret == CIO_OK); + + content_size = cio_chunk_get_content_size(chunk); + TEST_CHECK(content_size == (ssize_t) valid_size); + + ret = cio_chunk_down(chunk); + TEST_CHECK(ret == CIO_OK); + + cio_destroy(ctx); + + cio_options_init(&cio_opts); + cio_opts.root_path = CIO_ENV; + cio_opts.log_cb = log_cb; + cio_opts.log_level = CIO_LOG_INFO; + cio_opts.flags = CIO_OPEN | CIO_CHECKSUM | CIO_FULL_SYNC | CIO_TRIM_FILES; + + ctx = cio_create(&cio_opts); + TEST_CHECK(ctx != NULL); + if (!ctx) { + exit(1); + } + + ret = cio_load(ctx, NULL); + TEST_CHECK(ret == CIO_OK); + + stream = cio_stream_get(ctx, "fluent-bit"); + TEST_CHECK(stream != NULL); + chunk = find_stream_chunk(stream, "chunk.0"); + TEST_CHECK(chunk != NULL); + if (!chunk) { + exit(1); + } + + if (cio_chunk_is_up(chunk) == CIO_FALSE) { + ret = cio_chunk_up_force(chunk); + TEST_CHECK(ret == CIO_OK); + } + + ret = cio_chunk_get_content_copy(chunk, ©_buf, ©_size); + TEST_CHECK(ret == CIO_OK); + TEST_CHECK(copy_size == strlen(valid)); + TEST_CHECK(memcmp(copy_buf, valid, strlen(valid)) == 0); + free(copy_buf); + copy_buf = NULL; + + ret = cio_chunk_write(chunk, next, strlen(next)); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_sync(chunk); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_down(chunk); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_up_force(chunk); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_get_content_copy(chunk, ©_buf, ©_size); + TEST_CHECK(ret == CIO_OK); + TEST_CHECK(copy_size == strlen(valid) + strlen(next)); + TEST_CHECK(memcmp(copy_buf, valid, strlen(valid)) == 0); + TEST_CHECK(memcmp((char *) copy_buf + strlen(valid), next, strlen(next)) == 0); + + dlq_stream = cio_stream_create(ctx, "rejected", CIO_STORE_FS); + TEST_CHECK(dlq_stream != NULL); + + dlq_chunk = cio_chunk_open(ctx, dlq_stream, "fb.test_500_out.flb", + CIO_OPEN, copy_size, &err); + TEST_CHECK(dlq_chunk != NULL); + if (!dlq_chunk) { + free(copy_buf); + exit(1); + } + + ret = cio_chunk_write(dlq_chunk, copy_buf, copy_size); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_sync(dlq_chunk); + TEST_CHECK(ret == CIO_OK); + + cio_chunk_close(dlq_chunk, CIO_FALSE); + free(copy_buf); + + cf = (struct cio_file *) chunk->backend; + TEST_CHECK(cf->map[CIO_FILE_LEGACY_CRC_PADDING_OFFSET + 0] == 0); + TEST_CHECK(cf->map[CIO_FILE_LEGACY_CRC_PADDING_OFFSET + 1] == 0); + TEST_CHECK(cf->map[CIO_FILE_LEGACY_CRC_PADDING_OFFSET + 2] == 0); + TEST_CHECK(cf->map[CIO_FILE_LEGACY_CRC_PADDING_OFFSET + 3] == 0); + + if (cio_chunk_is_locked(chunk) == CIO_TRUE) { + ret = cio_chunk_unlock(chunk); + TEST_CHECK(ret == CIO_OK); + } + + cio_destroy(ctx); +} + +static void test_fs_fluent_bit_delete_irrecoverable_on_load() +{ + int ret; + int err; + struct stat st; + struct cio_ctx *ctx; + struct cio_chunk *chunk; + struct cio_stream *stream; + struct cio_options cio_opts; + char path[1024]; + char payload[] = "irrecoverable payload"; + + cio_utils_recursive_delete(CIO_ENV); + + cio_options_init(&cio_opts); + cio_opts.root_path = CIO_ENV; + cio_opts.log_cb = log_cb; + cio_opts.log_level = CIO_LOG_INFO; + cio_opts.flags = CIO_OPEN | CIO_CHECKSUM; + + ctx = cio_create(&cio_opts); + TEST_CHECK(ctx != NULL); + if (!ctx) { + exit(1); + } + + stream = cio_stream_create(ctx, "fluent-bit-bad", CIO_STORE_FS); + TEST_CHECK(stream != NULL); + + chunk = cio_chunk_open(ctx, stream, "bad.0", CIO_OPEN, 64, &err); + TEST_CHECK(chunk != NULL); + if (!chunk) { + exit(1); + } + + ret = cio_chunk_write(chunk, payload, strlen(payload)); + TEST_CHECK(ret == CIO_OK); + + ret = cio_chunk_sync(chunk); + TEST_CHECK(ret == CIO_OK); + + snprintf(path, sizeof(path), "%s/%s/%s", CIO_ENV, "fluent-bit-bad", "bad.0"); + + ret = cio_chunk_down(chunk); + TEST_CHECK(ret == CIO_OK); + + truncate_file((struct cio_file *) chunk->backend, 1, CIO_FALSE); + + cio_destroy(ctx); + + cio_options_init(&cio_opts); + cio_opts.root_path = CIO_ENV; + cio_opts.log_cb = log_cb; + cio_opts.log_level = CIO_LOG_INFO; + cio_opts.flags = CIO_OPEN | CIO_CHECKSUM | CIO_DELETE_IRRECOVERABLE; + + ctx = cio_create(&cio_opts); + TEST_CHECK(ctx != NULL); + if (!ctx) { + exit(1); + } + + ret = cio_load(ctx, NULL); + TEST_CHECK(ret == CIO_OK); + + ret = stat(path, &st); + TEST_CHECK(ret == -1); + + stream = cio_stream_get(ctx, "fluent-bit-bad"); + TEST_CHECK(stream != NULL); + TEST_CHECK(find_stream_chunk(stream, "bad.0") == NULL); + + cio_destroy(ctx); +} + +/* + * Test case: Prevent unsigned underflow when writing large metadata to a + * chunk with small initial allocation. + * + * This test specifically validates the fix for the bug where calculating + * content_av = alloc_size - CIO_FILE_HEADER_MIN - size could underflow + * when size is large, causing the resize check to be skipped and leading + * to out-of-bounds writes. + * + * Scenario: + * - Create chunk with minimal initial size (e.g., ~100 bytes page-aligned) + * - Write small metadata (10 bytes) and small content (20 bytes) + * - Write large metadata (80 bytes) that would cause underflow: + * old code: content_av = 100 - 24 - 80 = -4 (wraps to huge unsigned) + * - Verify resize happens correctly and no buffer overrun occurs + */ +static void test_metadata_unsigned_underflow() +{ + int ret; + int err; + char *meta_buf; + int meta_len; + void *content_buf; size_t content_size; struct cio_ctx *ctx; struct cio_chunk *chunk; @@ -1074,9 +2982,9 @@ static void test_metadata_unsigned_underflow() TEST_CHECK(ret == CIO_OK); TEST_CHECK(content_size == strlen(content_data)); TEST_CHECK(memcmp(content_buf, content_data, strlen(content_data)) == 0); + free(content_buf); /* Step 6: Sync to disk to ensure persistence */ - free(content_buf); ret = cio_chunk_sync(chunk); TEST_CHECK(ret == CIO_OK); @@ -1111,6 +3019,28 @@ TEST_LIST = { {"issue_51", test_issue_51}, {"issue_flb_2025", test_issue_flb_2025}, {"issue_write_at", test_issue_write_at}, + {"fs_recover_unfinalized_checksum", test_fs_recover_unfinalized_checksum}, + {"fs_shadow_checksum_recovers_primary_header", test_fs_shadow_checksum_recovers_primary_header}, + {"fs_shadow_checksum_recovers_read_only", test_fs_shadow_checksum_recovers_read_only}, + {"fs_shadow_checksum_recovers_primary_length", test_fs_shadow_checksum_recovers_primary_length}, + {"fs_valid_primary_precedes_newer_shadow", test_fs_valid_primary_precedes_newer_shadow}, + {"fs_valid_primary_with_stale_shadow_read_only_truncate", test_fs_valid_primary_with_stale_shadow_read_only_truncate}, + {"fs_valid_primary_with_stale_shadow_read_only_append", test_fs_valid_primary_with_stale_shadow_read_only_append}, + {"fs_valid_primary_with_stale_shadow_truncate", test_fs_valid_primary_with_stale_shadow_truncate}, + {"fs_valid_primary_with_stale_shadow_append", test_fs_valid_primary_with_stale_shadow_append}, + {"fs_write_at_zero_truncates_synced_checksum", test_fs_write_at_zero_truncates_synced_checksum}, + {"fs_legacy_checksum_without_shadow_header", test_fs_legacy_checksum_without_shadow_header}, + {"fs_old_format_loads_via_restart", test_fs_old_format_loads_via_restart}, + {"fs_shadow_rejects_oversized_length", test_fs_shadow_rejects_oversized_length}, + {"fs_shadow_rejects_maximum_length", test_fs_shadow_rejects_maximum_length}, + {"fs_shadow_rejects_corrupt_content", test_fs_shadow_rejects_corrupt_content}, + {"fs_shadow_rejects_both_tuples_corrupt", test_fs_shadow_rejects_both_tuples_corrupt}, + {"fs_format_contract_new_chunk_layout", test_fs_format_contract_new_chunk_layout}, + {"fs_golden_legacy_chunk_loads", test_fs_golden_legacy_chunk_loads}, + {"fs_legacy_big_endian_wide_checksum_migrates", test_fs_legacy_big_endian_wide_checksum_migrates}, + {"fs_legacy_big_endian_wide_checksum_read_only", test_fs_legacy_big_endian_wide_checksum_read_only}, + {"fs_fluent_bit_storage_lifecycle", test_fs_fluent_bit_storage_lifecycle}, + {"fs_fluent_bit_delete_irrecoverable_on_load", test_fs_fluent_bit_delete_irrecoverable_on_load}, {"fs_up_down_up_append", test_fs_up_down_up_append}, {"fs_deep_hierachy", test_deep_hierarchy}, {"legacy_success", test_legacy_success},