From 993ed4da2d55d8f53d82e12cc93428df316b45e8 Mon Sep 17 00:00:00 2001 From: hylin Date: Thu, 13 Aug 2026 14:23:45 +0800 Subject: [PATCH] ci: separate hermetic unit tests from online e2e --- .github/workflows/pull_request.yml | 10 ++-- Makefile | 9 ++- pyproject.toml | 1 + tests/README.md | 42 +++++++++++++ tests/e2e/test_dataset.py | 58 ++++++++++++++++++ tests/{ => unit}/test_adbpg.py | 0 tests/{ => unit}/test_aws_opensearch_cli.py | 0 tests/{ => unit}/test_case_runner_reuse.py | 0 tests/{ => unit}/test_cli_note.py | 0 .../test_cloud_cold_latency_case.py | 0 tests/{ => unit}/test_cloud_payload_case.py | 0 tests/{ => unit}/test_cloud_payload_search.py | 0 tests/{ => unit}/test_dataset.py | 59 +------------------ tests/{ => unit}/test_db_client_resolution.py | 0 tests/{ => unit}/test_fts_cases.py | 0 tests/{ => unit}/test_fts_cli_user_control.py | 0 tests/{ => unit}/test_fts_dataset.py | 0 tests/{ => unit}/test_fts_filter_runner.py | 8 ++- tests/{ => unit}/test_fts_format_results.py | 0 tests/{ => unit}/test_fts_metrics.py | 0 tests/{ => unit}/test_lancedb_config.py | 0 tests/{ => unit}/test_milvus_zilliz_cli.py | 0 tests/{ => unit}/test_oss_opensearch_fts.py | 0 tests/{ => unit}/test_utils.py | 0 24 files changed, 122 insertions(+), 65 deletions(-) create mode 100644 tests/README.md create mode 100644 tests/e2e/test_dataset.py rename tests/{ => unit}/test_adbpg.py (100%) rename tests/{ => unit}/test_aws_opensearch_cli.py (100%) rename tests/{ => unit}/test_case_runner_reuse.py (100%) rename tests/{ => unit}/test_cli_note.py (100%) rename tests/{ => unit}/test_cloud_cold_latency_case.py (100%) rename tests/{ => unit}/test_cloud_payload_case.py (100%) rename tests/{ => unit}/test_cloud_payload_search.py (100%) rename tests/{ => unit}/test_dataset.py (80%) rename tests/{ => unit}/test_db_client_resolution.py (100%) rename tests/{ => unit}/test_fts_cases.py (100%) rename tests/{ => unit}/test_fts_cli_user_control.py (100%) rename tests/{ => unit}/test_fts_dataset.py (100%) rename tests/{ => unit}/test_fts_filter_runner.py (96%) rename tests/{ => unit}/test_fts_format_results.py (100%) rename tests/{ => unit}/test_fts_metrics.py (100%) rename tests/{ => unit}/test_lancedb_config.py (100%) rename tests/{ => unit}/test_milvus_zilliz_cli.py (100%) rename tests/{ => unit}/test_oss_opensearch_fts.py (100%) rename tests/{ => unit}/test_utils.py (100%) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index e9de2c3b9..1f898ab06 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -7,8 +7,8 @@ on: - vdbbench_* jobs: - build: - name: Run Python Tests + unit-test: + name: Hermetic unit tests strategy: matrix: python-version: [3.11, 3.12] @@ -30,12 +30,12 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -e ".[test]" + python -m pip install -e ".[test]" - name: Run coding checks run: | make lint - - name: Test with pytest + - name: Run hermetic unit tests run: | - make unittest + make unit-test diff --git a/Makefile b/Makefile index ef8207c55..62bb6f130 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,10 @@ -unittest: - PYTHONPATH=`pwd` python3 -m pytest tests/test_dataset.py::TestDataSet::test_download_small -svv +unit-test: + python -m pytest --disable-socket tests/unit + +unittest: unit-test + +e2e-test: + python -m pytest tests/e2e -svv format: PYTHONPATH=`pwd` python3 -m black vectordb_bench diff --git a/pyproject.toml b/pyproject.toml index acd30275b..c4ad8ae4e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,6 +52,7 @@ test = [ "black", "ruff", "pytest", + "pytest-socket", ] restful = [ "flask" ] qdrant = [ "qdrant-client" ] diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 000000000..e54bcdd51 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,42 @@ +# Test layout + +The test suite is being separated by execution requirements. + +## Hermetic unit tests + +`tests/unit/` is the allowlisted local suite. Tests in this directory must be +deterministic and must not use network sockets, credentials, live databases, +containers, or downloaded datasets. + +Install only the project and test extra, then run: + +```shell +pip install -e ".[test]" +make unit-test +``` + +`make unittest` is a compatibility alias. The unit target uses +`pytest-socket` to disable network access and is the suite run by pull-request +CI. + +## Online E2E tests + +`tests/e2e/` contains tests that download datasets or require external +services. Run them explicitly with: + +```shell +make e2e-test +``` + +The dataset E2E tests require network access to S3 and/or Aliyun OSS and can +download large Cohere or LAION datasets. Other E2E tests may require provider +SDK extras, credentials, containers, or a locally running database; document +those prerequisites with the test when migrating it. + +## Legacy tests + +Test modules directly under `tests/` are legacy and are not yet classified. +They are intentionally outside both Make targets: do not assume they are +hermetic, and do not move them into `tests/unit/` until they pass with only +`.[test]` and with network sockets disabled. New tests must go directly into +`tests/unit/` or `tests/e2e/`. diff --git a/tests/e2e/test_dataset.py b/tests/e2e/test_dataset.py new file mode 100644 index 000000000..bb5088a78 --- /dev/null +++ b/tests/e2e/test_dataset.py @@ -0,0 +1,58 @@ +import logging +import os +import time + +import pytest + +from vectordb_bench.backend.data_source import DatasetSource +from vectordb_bench.backend.dataset import Dataset + +log = logging.getLogger("vectordb_bench") +pytestmark = pytest.mark.integration + + +class TestDataSet: + def test_iter_cohere(self): + cohere_10m = Dataset.COHERE.manager(10_000_000) + cohere_10m.prepare() + + before = time.time() + for batch in cohere_10m: + log.debug(batch.head(1)) + + duration = time.time() - before + log.warning("iter through cohere_10m cost=%smin", duration / 60) + + def test_iter_laion(self): + laion_100m = Dataset.LAION.manager(100_000_000) + laion_100m.prepare(source=DatasetSource.AliyunOSS) + + before = time.time() + for batch in laion_100m: + log.debug(batch.head(1)) + + duration = time.time() - before + log.warning("iter through laion_100m cost=%smin", duration / 60) + + def test_download_small(self): + openai_50k = Dataset.OPENAI.manager(50_000) + files = [ + "test.parquet", + "neighbors.parquet", + "neighbors_head_1p.parquet", + "neighbors_tail_1p.parquet", + ] + + file_path = openai_50k.data_dir.joinpath("test.parquet") + DatasetSource.S3.reader().read( + openai_50k.data.dir_name.lower(), + files=files, + local_ds_root=openai_50k.data_dir, + ) + + os.remove(file_path) + DatasetSource.AliyunOSS.reader().read( + openai_50k.data.dir_name.lower(), + files=files, + local_ds_root=openai_50k.data_dir, + ) diff --git a/tests/test_adbpg.py b/tests/unit/test_adbpg.py similarity index 100% rename from tests/test_adbpg.py rename to tests/unit/test_adbpg.py diff --git a/tests/test_aws_opensearch_cli.py b/tests/unit/test_aws_opensearch_cli.py similarity index 100% rename from tests/test_aws_opensearch_cli.py rename to tests/unit/test_aws_opensearch_cli.py diff --git a/tests/test_case_runner_reuse.py b/tests/unit/test_case_runner_reuse.py similarity index 100% rename from tests/test_case_runner_reuse.py rename to tests/unit/test_case_runner_reuse.py diff --git a/tests/test_cli_note.py b/tests/unit/test_cli_note.py similarity index 100% rename from tests/test_cli_note.py rename to tests/unit/test_cli_note.py diff --git a/tests/test_cloud_cold_latency_case.py b/tests/unit/test_cloud_cold_latency_case.py similarity index 100% rename from tests/test_cloud_cold_latency_case.py rename to tests/unit/test_cloud_cold_latency_case.py diff --git a/tests/test_cloud_payload_case.py b/tests/unit/test_cloud_payload_case.py similarity index 100% rename from tests/test_cloud_payload_case.py rename to tests/unit/test_cloud_payload_case.py diff --git a/tests/test_cloud_payload_search.py b/tests/unit/test_cloud_payload_search.py similarity index 100% rename from tests/test_cloud_payload_search.py rename to tests/unit/test_cloud_payload_search.py diff --git a/tests/test_dataset.py b/tests/unit/test_dataset.py similarity index 80% rename from tests/test_dataset.py rename to tests/unit/test_dataset.py index 3e5d469a7..53633cb7c 100644 --- a/tests/test_dataset.py +++ b/tests/unit/test_dataset.py @@ -8,7 +8,6 @@ from vectordb_bench import config from vectordb_bench.backend import dataset as dataset_module from vectordb_bench.backend.clients import MetricType -from vectordb_bench.backend.data_source import DatasetSource from vectordb_bench.backend.dataset import CustomDataset, Dataset, DatasetManager from vectordb_bench.backend.filter import LabelFilter, NewIntFilter, non_filter @@ -187,8 +186,8 @@ def _write_vector_fixture(data_dir): class TestDataSet: def test_iter_dataset(self): - for ds in Dataset: - log.info(ds) + for dataset in Dataset: + log.info(dataset) def test_cohere(self): cohere = Dataset.COHERE.get(100_000) @@ -201,57 +200,3 @@ def test_cohere(self): def test_cohere_error(self): with pytest.raises(ValidationError): Dataset.COHERE.get(9999) - - def test_iter_cohere(self): - cohere_10m = Dataset.COHERE.manager(10_000_000) - cohere_10m.prepare() - - import time - - before = time.time() - for i in cohere_10m: - log.debug(i.head(1)) - - dur_iter = time.time() - before - log.warning(f"iter through cohere_10m cost={dur_iter/60}min") - - # pytest -sv tests/test_dataset.py::TestDataSet::test_iter_laion - def test_iter_laion(self): - laion_100m = Dataset.LAION.manager(100_000_000) - from vectordb_bench.backend.data_source import DatasetSource - - laion_100m.prepare(source=DatasetSource.AliyunOSS) - - import time - - before = time.time() - for i in laion_100m: - log.debug(i.head(1)) - - dur_iter = time.time() - before - log.warning(f"iter through laion_100m cost={dur_iter/60}min") - - def test_download_small(self): - openai_50k = Dataset.OPENAI.manager(50_000) - files = [ - "test.parquet", - "neighbors.parquet", - "neighbors_head_1p.parquet", - "neighbors_tail_1p.parquet", - ] - - file_path = openai_50k.data_dir.joinpath("test.parquet") - import os - - DatasetSource.S3.reader().read( - openai_50k.data.dir_name.lower(), - files=files, - local_ds_root=openai_50k.data_dir, - ) - - os.remove(file_path) - DatasetSource.AliyunOSS.reader().read( - openai_50k.data.dir_name.lower(), - files=files, - local_ds_root=openai_50k.data_dir, - ) diff --git a/tests/test_db_client_resolution.py b/tests/unit/test_db_client_resolution.py similarity index 100% rename from tests/test_db_client_resolution.py rename to tests/unit/test_db_client_resolution.py diff --git a/tests/test_fts_cases.py b/tests/unit/test_fts_cases.py similarity index 100% rename from tests/test_fts_cases.py rename to tests/unit/test_fts_cases.py diff --git a/tests/test_fts_cli_user_control.py b/tests/unit/test_fts_cli_user_control.py similarity index 100% rename from tests/test_fts_cli_user_control.py rename to tests/unit/test_fts_cli_user_control.py diff --git a/tests/test_fts_dataset.py b/tests/unit/test_fts_dataset.py similarity index 100% rename from tests/test_fts_dataset.py rename to tests/unit/test_fts_dataset.py diff --git a/tests/test_fts_filter_runner.py b/tests/unit/test_fts_filter_runner.py similarity index 96% rename from tests/test_fts_filter_runner.py rename to tests/unit/test_fts_filter_runner.py index 50803ae19..071cbedb4 100644 --- a/tests/test_fts_filter_runner.py +++ b/tests/unit/test_fts_filter_runner.py @@ -1,4 +1,5 @@ import threading +from types import SimpleNamespace from vectordb_bench.backend.cases import CaseLabel from vectordb_bench.backend.data_source import DatasetSource @@ -48,7 +49,11 @@ class Case: dataset = Dataset() filters = filter_obj - config_obj = type("Config", (), {"stages": [TaskStage.LOAD]})() + config_obj = type( + "Config", + (), + {"stages": [TaskStage.LOAD], "case_config": SimpleNamespace(k=10)}, + )() runner = CaseRunner.construct(ca=Case(), config=config_obj, dataset_source=DatasetSource.S3) init_calls = [] monkeypatch.setattr(CaseRunner, "init_db", lambda self, drop_old=True: init_calls.append(drop_old)) @@ -70,6 +75,7 @@ class Config: db = type("DbConfig", (), {"init_cls": Db})() db_config = type("Config", (), {"to_dict": lambda self: {}})() db_case_config = object() + case_config = SimpleNamespace(k=10) for filters in (non_filter, NewIntFilter(filter_rate=0.5, int_field="filter_id", int_value=50)): case = type( diff --git a/tests/test_fts_format_results.py b/tests/unit/test_fts_format_results.py similarity index 100% rename from tests/test_fts_format_results.py rename to tests/unit/test_fts_format_results.py diff --git a/tests/test_fts_metrics.py b/tests/unit/test_fts_metrics.py similarity index 100% rename from tests/test_fts_metrics.py rename to tests/unit/test_fts_metrics.py diff --git a/tests/test_lancedb_config.py b/tests/unit/test_lancedb_config.py similarity index 100% rename from tests/test_lancedb_config.py rename to tests/unit/test_lancedb_config.py diff --git a/tests/test_milvus_zilliz_cli.py b/tests/unit/test_milvus_zilliz_cli.py similarity index 100% rename from tests/test_milvus_zilliz_cli.py rename to tests/unit/test_milvus_zilliz_cli.py diff --git a/tests/test_oss_opensearch_fts.py b/tests/unit/test_oss_opensearch_fts.py similarity index 100% rename from tests/test_oss_opensearch_fts.py rename to tests/unit/test_oss_opensearch_fts.py diff --git a/tests/test_utils.py b/tests/unit/test_utils.py similarity index 100% rename from tests/test_utils.py rename to tests/unit/test_utils.py