Feature/buffer manager

.gitignore

@@ -4,3 +4,4 @@ compile_commands.json
 *.db
 *.wal
 .claude
+next_steps.txt

.obsidian/appearance.json

@@ -1,3 +1,3 @@
 {
-  "theme": "moonstone"
+  "theme": "obsidian"
 }

.obsidian/graph.json

@@ -0,0 +1,22 @@
+{
+  "collapse-filter": false,
+  "search": "design-docs",
+  "showTags": false,
+  "showAttachments": false,
+  "hideUnresolved": false,
+  "showOrphans": true,
+  "collapse-color-groups": false,
+  "colorGroups": [],
+  "collapse-display": true,
+  "showArrow": false,
+  "textFadeMultiplier": 0,
+  "nodeSizeMultiplier": 1,
+  "lineSizeMultiplier": 1,
+  "collapse-forces": true,
+  "centerStrength": 0.518713248970312,
+  "repelStrength": 10,
+  "linkStrength": 1,
+  "linkDistance": 250,
+  "scale": 0.8914715268792586,
+  "close": false
+}

.obsidian/workspace.json

@@ -4,21 +4,21 @@
     "type": "split",
     "children": [
       {
-        "id": "19615ae5e5dc0062",
+        "id": "f8b803ce0789e6ab",
         "type": "tabs",
         "children": [
           {
-            "id": "b27174596c7c49e7",
+            "id": "213624f19b44c94b",
             "type": "leaf",
             "state": {
               "type": "markdown",
               "state": {
-                "file": "design-docs/DD-001-storage-file-layout.md",
+                "file": "design-docs/DD-002-buffer-pool-manager.md",
                 "mode": "preview",
                 "source": false
               },
               "icon": "lucide-file",
-              "title": "DD-001-storage-file-layout"
+              "title": "DD-002-buffer-pool-manager"
             }
           }
         ]
@@ -41,7 +41,9 @@
               "type": "file-explorer",
               "state": {
                 "sortOrder": "alphabetical",
-                "autoReveal": false
+                "autoReveal": false,
+                "showSearch": false,
+                "searchQuery": ""
               },
               "icon": "lucide-folder-closed",
               "title": "Files"
@@ -78,8 +80,7 @@
       }
     ],
     "direction": "horizontal",
-    "width": 300,
-    "collapsed": true
+    "width": 300
   },
   "right": {
     "id": "4ce9e65978c028c9",
@@ -185,23 +186,30 @@
       "bases:Create new base": false
     }
   },
-  "active": "b27174596c7c49e7",
+  "active": "213624f19b44c94b",
   "lastOpenFiles": [
-    "build/debug/Testing/Temporary/CTestCheckpoint.txt",
-    "build/debug/Testing/Temporary/LastTest.log.tmp",
-    "build/debug/CMakeFiles/disk_manager_test.dir/link.d",
-    "build/debug/CMakeFiles/disk_manager_test.dir/test/storage/disk_manager_test.cpp.o.d",
-    "test/storage/disk_manager_test.cpp.tmp.9657.43c7f8735f29",
-    "test/storage/disk_manager_test.cpp.tmp.9657.bdd712030b93",
     "build/debug/CMakeFiles/kernsql.dir/link.d",
-    "build/debug/stu9JUzF",
-    "build/debug/st9efrCo",
-    "build/debug/CMakeFiles/kernsql_lib.dir/src/storage/disk_manager.cpp.o.d",
-    "build/debug/CMakeCache.txt.tmp00436",
-    "assets/slotted_page.png",
-    "assets/storage_file_layout.png",
+    "build/debug/stt8jRT2",
+    "build/debug/stTk6Uc9",
+    "build/debug/CMakeFiles/kernsql_lib.dir/src/buffer/buffer_pool_manager.cpp.o.d",
+    "src/buffer/buffer_pool_manager.cpp.tmp.16764.7acecd0a7cef",
+    "src/buffer/buffer_pool_manager.cpp.tmp.16764.6d12f137211c",
+    "build/debug/stB1ka1L",
+    "build/debug/stMIkJFI",
+    "build/debug/stCuaJp3",
+    "build/debug/stI9cO3C",
+    "build/debug/CMakeFiles/kernsql_lib.dir/src/buffer/page_guard.cpp.o.d",
+    "design-docs/DD-003-threading-model.md",
+    "design-docs/DD-002-buffer-pool-manager.md",
+    "design-docs/DD-002-buffer-manager.md",
+    "design-docs/notes/Buffer-Pool-Manager.md",
+    "design-docs.md",
+    "build/asan/_deps/googletest-src/docs/reference/assertions.md",
+    "design-docs/notes/Concurrency-Control.md",
     "design-docs/notes/Indexes.md",
     "design-docs/DD-001-storage-file-layout.md",
+    "assets/slotted_page.png",
+    "assets/storage_file_layout.png",
     "Untitled.canvas",
     "README.md"
   ]

CMakeLists.txt

@@ -6,9 +6,14 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS OFF)
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)   # for clangd
 
-add_library(kernsql_lib STATIC src/common/logger.cpp src/storage/disk_manager.cpp)
+add_library(kernsql_lib STATIC
+  src/common/logger.cpp
+  src/storage/disk_manager.cpp
+  src/buffer/buffer_pool_manager.cpp
+  src/buffer/page_guard.cpp)
 target_include_directories(kernsql_lib PUBLIC src)
-target_compile_options(kernsql_lib PUBLIC -Wall -Wextra -Wpedantic -Wshadow -Wconversion)
+target_compile_options(kernsql_lib PUBLIC -Wall -Wextra -Wpedantic -Wshadow -Wconversion
+  -Werror=format)
 
 add_executable(kernsql src/shell/main.cpp)
 target_link_libraries(kernsql PRIVATE kernsql_lib)
@@ -26,5 +31,7 @@ foreach(t ${TEST_SRCS})
   get_filename_component(name ${t} NAME_WE)
   add_executable(${name} ${t})
   target_link_libraries(${name} PRIVATE kernsql_lib GTest::gtest_main)
-  gtest_discover_tests(${name})
+  # A dropped condvar notify makes a concurrency test HANG rather than fail, and a hung test
+  # blocks the run forever instead of reporting. Generous because TSan is roughly 10x slower.
+  gtest_discover_tests(${name} PROPERTIES TIMEOUT 120)
 endforeach()

README.md

@@ -3,7 +3,7 @@
 </p>
 
 <p align="center">
-  A minimal SQL database built from scratch in C++20 on Linux —
+  A minimal SQL database built from scratch in C++23 on Linux —
   storage engine, transactions, and query engine, no shortcuts.
 </p>
 
@@ -19,51 +19,109 @@ It is a learning-in-public project. The goal is not to compete with SQLite; it i
 understand how real databases (Postgres in particular) work by building one, and to leave
 behind a codebase clean enough that others can learn from it.
 
-## Architecture at a glance
+The emphasis is **depth over surface area**. The SQL dialect is deliberately tiny; the
+concurrency underneath it is not. Every design decision is recorded in
+[`design-docs/`](design-docs/) with the alternatives that were rejected and why — those
+documents are the most useful thing in this repository.
 
-KernSQL follows a Postgres-flavored design:
+## Status
+
+| Component | State |
+|---|---|
+| `DiskManager` — single file, 4 KB pages, persistent free list, thread-safe | done, tested |
+| `PageHeader` — 32-byte self-describing header (self page id, format version) | done |
+| `Replacer` — CLOCK sweep with capped usage counts | done, tested |
+| `BufferPoolManager` — sharded page table, frame state machine, RAII page guards | code complete, **tests pending** |
+| Shell — REPL over the buffer pool, one command per operation | done |
+| Slotted pages + heap file | next |
+| B+tree with latch crabbing | planned |
+| Catalog | planned |
+| Parser, binder, executor | planned |
+| Two-phase locking + deadlock detection | planned |
+
+## Architecture
 
 | Layer | Design |
 |---|---|
-| Storage | Single-file, page-based (4 KB), slotted pages, heap files |
-| Caching | Buffer pool with LRU-K replacement, pin/latch semantics |
-| Indexing | B+tree with latch crabbing for concurrent access |
-| Concurrency control | MVCC with snapshot isolation — in-heap version chains (`xmin`/`xmax`), first-updater-wins |
-| Durability | Redo-only write-ahead log, checkpoints, crash recovery, vacuum |
+| Storage | Single-file, page-based (4 KB), self-describing page headers, slotted pages, heap files |
+| Caching | Buffer pool: 16-way sharded page table, CLOCK-sweep eviction, pin/latch separation, RAII guards |
+| Indexing | B+tree with latch crabbing, fixed-width integer keys |
+| Concurrency control | Strict two-phase locking — row-level shared/exclusive locks, wait-for-graph deadlock detection |
+| Durability | Explicit `Shutdown()` — flush, then `fsync`. No write-ahead log (see non-goals) |
 | SQL front-end | Hand-written lexer and recursive-descent parser, binder with type checking |
-| Execution | Volcano (iterator) model, rule-based planner with predicate pushdown and index selection |
+| Execution | Volcano (iterator) model, fixed execution strategy — no cost model |
 | Interface | Interactive shell, thread-per-session concurrency |
 
-A key consequence of this combination: **rollback writes nothing to the heap.** Aborted
-transactions simply become invisible — the same property that makes redo-only logging
-sufficient in Postgres.
+Two decisions shape everything else:
+
+**Thread-per-session.** One thread carries a statement down the entire stack and back; layers
+are a code decomposition, never a scheduling one. Every layer below is therefore synchronous
+and blocking by construction ([DD-003](design-docs/DD-003-threading-model.md)).
+
+**Latches are not locks.** Latches protect physical structures for nanoseconds and are ordered
+to prevent deadlock; locks protect logical database state for the length of a transaction and
+deadlock is detected and resolved. Conflating them is the classic error, and the distinction is
+load-bearing throughout the codebase.
 
-## Features (v1 scope)
+## Scope (v1)
 
 **SQL surface**
 
-- `CREATE TABLE` / `DROP TABLE`
+- `CREATE TABLE`
 - `INSERT`, `UPDATE`, `DELETE`
-- `SELECT` with `WHERE`, `ORDER BY`, `LIMIT`, `INNER JOIN`
-- `GROUP BY` with `COUNT` / `SUM` / `AVG` (stretch)
+- `SELECT` with `WHERE` and a single `INNER JOIN`
 - `BEGIN` / `COMMIT` / `ROLLBACK`
-- Types: `BIGINT`, `VARCHAR(n)`, `BOOLEAN` — with `NULL` support
+- Types: `INT`, `VARCHAR(n)`, with `NULL` support
 
 **Engine guarantees**
 
-- Snapshot isolation for all transactions; readers never block writers
-- Crash safety: committed data survives `kill -9` (WAL replay on restart)
-- Concurrent sessions with correct latching throughout — the full test suite runs under
+- Serializable isolation via strict 2PL; deadlocks are detected and one transaction is aborted
+- Concurrent sessions with correct latching throughout — the test suite runs under
   ThreadSanitizer and AddressSanitizer in CI
-- System catalog stored in the database itself, as regular tables
+- Durability on clean shutdown
+
+## Non-goals
+
+Deliberately out of scope. These are decisions, not omissions — each is recorded with its
+reasoning in the relevant design doc.
+
+**No write-ahead log, and therefore no crash recovery.** Durability comes from an explicit
+`Shutdown()` that flushes and `fsync`s. A `kill -9` loses every dirty page in the buffer pool,
+and because a transaction's pages are not flushed atomically, a crash can leave the database
+*structurally* inconsistent — a half-applied B+tree split, not merely missing recent writes.
+This is the single largest simplification in the project. It is deferred rather than unexamined:
+adding a WAL would change the buffer pool's flush sequence (a page could no longer be written
+before the log record describing it) and would demote the shutdown flush from the durability
+mechanism to a restart-time optimization.
+
+**No MVCC.** Concurrency control is 2PL, so readers block writers and writers block readers.
+MVCC was rejected on scope: it is a tuple-format decision (version chains or an undo log) plus
+a mandatory garbage collector, and it reaches into layers this project builds later. 2PL is
+additive to what already exists.
 
-## Non-goals (v1)
+**No query optimizer.** No cost model, no statistics, no join ordering. An index is used if and
+only if the predicate is on the primary key.
 
-Deliberately out of scope, so the core stays finishable and understandable:
+**No variable-length index keys**, no prefix compression, no hash join, no overflow pages —
+tuples larger than a page are rejected rather than split.
 
-distributed anything · cost-based optimization · subqueries, CTEs, views, triggers,
-foreign keys · `ALTER TABLE` · floating-point types · authentication · network wire
-protocol.
+**No page checksums.** Space is reserved in the page header; the self page id catches
+misdirected reads, but not bit rot.
+
+**Not portable across architectures.** Page headers are `memcpy`'d, so the file format is
+host-endian and host-ABI.
+
+Also out of scope: distributed anything · subqueries, CTEs, views, triggers, foreign keys ·
+`ALTER TABLE`, `DROP TABLE` · aggregates, `GROUP BY`, `ORDER BY` · floating-point types ·
+authentication · a network wire protocol.
+
+## Design docs
+
+The reasoning behind each component, including rejected alternatives:
+
+- [DD-001 — Storage file layout](design-docs/DD-001-storage-file-layout.md)
+- [DD-002 — Buffer pool: concurrency and latching](design-docs/DD-002-buffer-pool-manager.md)
+- [DD-003 — Threading and execution model](design-docs/DD-003-threading-model.md)
 
 ## Project layout
 
@@ -76,7 +134,7 @@ assets/         logo and branding
 
 ## Building
 
-Requires GCC 13+ (or Clang 17+), CMake ≥ 3.25, Ninja.
+Requires Clang 17+ with libc++ (the sanitizer presets pin it), CMake ≥ 3.25, Ninja.
 
 ```bash
 cmake --preset debug && cmake --build --preset debug   # fast development build
@@ -89,10 +147,17 @@ cmake --preset tsan && cmake --build --preset tsan && ctest --preset tsan   # ra
 CI runs the ASan and TSan suites plus a clang-format check on every pull request; `main`
 only moves through green pipelines.
 
-## Status
+Then drive the engine by hand:
 
-🚧 Early days — foundation phase. Progress, design decisions, and write-ups are tracked in
-[`design-docs/`](design-docs/) as each component lands, feature by feature, branch by branch.
+```bash
+./build/debug/kernsql mydb.db
+kernsql> new
+new: allocated page 2
+kernsql> write 2 hello
+kernsql> read 2
+read: page 2 -> "hello"
+kernsql> quit
+```
 
 ## References