From 080fecf6538a5b9ce942ed421f5a031460ff3f8c Mon Sep 17 00:00:00 2001 From: jcbl1 Date: Fri, 23 Oct 2026 09:30:02 +0000 Subject: [PATCH] perf(db): index expiry, recency and chunk lookup columns Migration 008 adds: - filecodes(expired_at) for expiry scans and admin expiry stats - filecodes(created_at) for the default recency ordering and today/yesterday counters - uploadchunk(upload_id, chunk_index) for chunk status checks and merges; its prefix also covers upload_id-only deletes (previously unindexed) Models mirror the same metadata for the generate_schemas test path. Upgrade benchmark: 50k filecodes + 20k chunks -> 30ms index build; scans become index searches. Refs NOTES batch 4. --- apps/base/migrations/migrations_008.py | 26 +++++++++ apps/base/models.py | 13 ++++- tests/test_migration_indexes.py | 81 ++++++++++++++++++++++++++ tests/test_migration_runner.py | 18 +++--- 4 files changed, 126 insertions(+), 12 deletions(-) create mode 100644 apps/base/migrations/migrations_008.py create mode 100644 tests/test_migration_indexes.py diff --git a/apps/base/migrations/migrations_008.py b/apps/base/migrations/migrations_008.py new file mode 100644 index 000000000..2405f827a --- /dev/null +++ b/apps/base/migrations/migrations_008.py @@ -0,0 +1,26 @@ +from tortoise import connections + +# 高频查询字段索引。真实部署的 schema 全部由迁移建立 +# （init_db 不调用 generate_schemas），因此索引必须写在迁移里； +# apps/base/models.py 中的 index/Meta 仅用于测试的 generate_schemas 路径。 +# +# - filecodes(expired_at)：过期扫描（delete_expire_files）与后台过期统计。 +# - filecodes(created_at)：后台默认倒序列表、首页最近记录、今日/昨日统计。 +# - uploadchunk(upload_id, chunk_index)：分片状态检查与合并；其前缀同时 +# 覆盖仅按 upload_id 的删除/清理查询（原先没有对应索引）。 +# 已知缺口：clean_incomplete_uploads 的 chunk_index=-1 AND created_at str: + _, rows = await conn.execute_query(f"EXPLAIN QUERY PLAN {sql}") + return " | ".join(str(row["detail"]) for row in rows) + + +@pytest.mark.asyncio +class TestMigrationIndexes: + async def test_indexes_exist(self, migrated_db): + _, rows = await migrated_db.execute_query( + "SELECT name FROM sqlite_master WHERE type='index'" + ) + names = {row["name"] for row in rows} + assert INDEXES <= names + + async def test_expiry_scan_uses_index(self, migrated_db): + plan = await _plan( + migrated_db, + "SELECT id FROM filecodes WHERE expired_at < '2024-01-01 00:00:00'", + ) + assert "idx_filecodes_expired_at" in plan + + async def test_recent_listing_uses_created_at_index(self, migrated_db): + plan = await _plan( + migrated_db, + "SELECT id FROM filecodes ORDER BY created_at DESC LIMIT 8", + ) + assert "idx_filecodes_created_at" in plan + + async def test_chunk_lookup_uses_composite_index(self, migrated_db): + plan = await _plan( + migrated_db, + "SELECT chunk_index FROM uploadchunk " + "WHERE upload_id = 'abc' AND chunk_index = 3", + ) + assert "idx_uploadchunk_upload_chunk" in plan + + async def test_chunk_cleanup_uses_upload_id_prefix(self, migrated_db): + plan = await _plan( + migrated_db, + "DELETE FROM uploadchunk WHERE upload_id = 'abc'", + ) + assert "idx_uploadchunk_upload_chunk" in plan diff --git a/tests/test_migration_runner.py b/tests/test_migration_runner.py index 6b4556178..7b51a444a 100644 --- a/tests/test_migration_runner.py +++ b/tests/test_migration_runner.py @@ -1,5 +1,5 @@ """Migration runner coverage: discovery order, registration, idempotency, -and failure propagation. Runs the REAL migrations (001-007) against an +and failure propagation. Runs the REAL migrations (001-008) against an in-memory database — the same code path a deployment upgrade takes. """ import pytest @@ -8,7 +8,7 @@ from core.database import execute_migrations from tests.helpers import MEMORY_DB_CONFIG, close_db -EXPECTED = [f"migrations_{i:03d}.py" for i in range(1, 8)] +EXPECTED = [f"migrations_{i:03d}.py" for i in range(1, 9)] @pytest.fixture @@ -60,15 +60,15 @@ async def test_pre_registered_subset_skips_those(self, fresh_db): """部分已登记（模拟中途升级）：只执行缺口，顺序保持。""" conn = Tortoise.get_connection("default") await conn.execute_query("DELETE FROM migrates") - # 预登记 006/007（模拟已执行过全部迁移的库），验证 runner 只跑缺口 - # 001-005 且跳过已登记的 006/007（跳过语义 = 不再执行其 DDL） - for name in EXPECTED[-2:]: + # 预登记 006/007/008（模拟已执行过末尾迁移的库），验证 runner 只跑缺口 + # 001-005 且跳过已登记的 006/007/008（跳过语义 = 不再执行其 DDL） + for name in EXPECTED[-3:]: await conn.execute_query( "INSERT INTO migrates (migration_file) VALUES (?)", [name] ) await execute_migrations() registered = await _registered() - # 预登记的 006/007 占据登记序前列——断言用集合：7 个全登记、无重复 + # 预登记的 006/007/008 占据登记序前列——断言用集合：全部登记、无重复 assert sorted(registered) == sorted(EXPECTED) assert len(registered) == len(EXPECTED) tables = { @@ -107,7 +107,7 @@ async def test_failure_propagates_and_halts(self, fresh_db, monkeypatch): """迁移执行失败必须向上抛（启动失败可见），不能静默吞掉。""" import importlib - # 构造一个会失败的执行缺口：007 未登记，且其模块 migrate 抛错 + # 构造一个会失败的执行缺口：008 未登记，且其模块 migrate 抛错 conn = Tortoise.get_connection("default") await conn.execute_query("DELETE FROM migrates") for name in EXPECTED[:-1]: @@ -117,7 +117,7 @@ async def test_failure_propagates_and_halts(self, fresh_db, monkeypatch): real_import = importlib.import_module def fake_import(name): - if name.endswith("migrations_007"): + if name.endswith("migrations_008"): class FakeMod: @staticmethod async def migrate(): @@ -130,4 +130,4 @@ async def migrate(): with pytest.raises(RuntimeError, match="boom"): await execute_migrations() # 失败的迁移不得登记 - assert "migrations_007.py" not in await _registered() + assert "migrations_008.py" not in await _registered()