From c99785f6cf8d53d3b6c3769d6c657a9267192c9b Mon Sep 17 00:00:00 2001 From: Dan LaManna Date: Tue, 6 Oct 2026 11:13:01 -0400 Subject: [PATCH] Add pytest-benchmark benchmarks against a fixed synthetic dataset --- .github/workflows/benchmarks.yml | 79 ++++ .gitignore | 1 + benchmarks/README.md | 26 ++ benchmarks/__init__.py | 0 benchmarks/conftest.py | 156 +++++++ benchmarks/dataset.py | 678 +++++++++++++++++++++++++++++++ benchmarks/history.py | 96 +++++ benchmarks/index.html | 145 +++++++ benchmarks/test_collections.py | 33 ++ benchmarks/test_exports.py | 25 ++ benchmarks/test_images.py | 64 +++ benchmarks/test_lesions.py | 20 + benchmarks/test_pages.py | 39 ++ benchmarks/test_search.py | 30 ++ benchmarks/test_search_index.py | 25 ++ benchmarks/test_writes.py | 93 +++++ benchmarks/test_zip_download.py | 28 ++ isic/settings/benchmark.py | 30 ++ pyproject.toml | 11 +- tox.ini | 12 + uv.lock | 26 ++ 21 files changed, 1616 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/benchmarks.yml create mode 100644 benchmarks/README.md create mode 100644 benchmarks/__init__.py create mode 100644 benchmarks/conftest.py create mode 100644 benchmarks/dataset.py create mode 100644 benchmarks/history.py create mode 100644 benchmarks/index.html create mode 100644 benchmarks/test_collections.py create mode 100644 benchmarks/test_exports.py create mode 100644 benchmarks/test_images.py create mode 100644 benchmarks/test_lesions.py create mode 100644 benchmarks/test_pages.py create mode 100644 benchmarks/test_search.py create mode 100644 benchmarks/test_search_index.py create mode 100644 benchmarks/test_writes.py create mode 100644 benchmarks/test_zip_download.py create mode 100644 isic/settings/benchmark.py diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml new file mode 100644 index 000000000..eed126387 --- /dev/null +++ b/.github/workflows/benchmarks.yml @@ -0,0 +1,79 @@ +name: benchmarks +on: + push: + branches: + - master +permissions: + # Each run pushes its results to the gh-pages branch. + contents: write +# Each run adds to the history on gh-pages, so runs can't overlap. +concurrency: benchmarks +jobs: + benchmark: + runs-on: ubuntu-24.04 + services: + postgres: + image: pgvector/pgvector:0.8.6-pg18 + env: + POSTGRES_PASSWORD: postgres + options: >- + --health-cmd "pg_isready --username postgres" + --health-start-period 30s + --health-start-interval 2s + ports: + - 5432:5432 + elasticsearch: + image: elasticsearch:9.5.2 + env: + ES_JAVA_OPTS: "-Xms250m -Xmx750m" + discovery.type: single-node + xpack.security.enabled: "true" + ELASTIC_PASSWORD: elastic + options: >- + --health-cmd "curl --fail --user elastic:elastic http://localhost:9200/" + --health-start-period 30s + --health-start-interval 2s + ports: + - 9200:9200 + env: + DJANGO_DATABASE_URL: postgres://postgres:postgres@localhost:5432/django + DJANGO_ISIC_ELASTICSEARCH_URL: http://elastic:elastic@localhost:9200 + # The same as ci.yml. No benchmark connects to them, so this job has no services for them. + DJANGO_CELERY_BROKER_URL: amqp://localhost:5672/ + DJANGO_CACHE_URL: redis://localhost:6379/0 + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Install uv + uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 + - name: Increase maintenance_work_mem for pgvector + # The dataset rebuilds the imageembedding IVFFlat index; see ci.yml. + run: psql -c "ALTER SYSTEM SET maintenance_work_mem = '512MB'" -c "SELECT pg_reload_conf();" + env: + PGHOST: localhost + PGUSER: postgres + PGPASSWORD: postgres + - name: Run benchmarks + run: uv run --locked tox -e benchmark -- --benchmark-json=.benchmarks/ci.json + - name: Checkout gh-pages + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: gh-pages + path: gh-pages + - name: Record results + run: | + uv run --locked python benchmarks/history.py record .benchmarks/ci.json gh-pages/benchmarks/data.js + cp benchmarks/index.html gh-pages/benchmarks/ + - name: Push results + # A regression is still recorded, so the next run compares against it. + working-directory: gh-pages + run: | + # github-actions[bot]'s noreply address (41898282 is its user ID), as in actions/checkout's + # README, so GitHub attributes the commit to that bot. + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add benchmarks + git commit --message "Record benchmarks for $GITHUB_SHA" + git push + - name: Check for regressions + run: uv run --locked python benchmarks/history.py check gh-pages/benchmarks/data.js diff --git a/.gitignore b/.gitignore index 0f0f95f35..c8bf06246 100644 --- a/.gitignore +++ b/.gitignore @@ -68,6 +68,7 @@ pip-delete-this-directory.txt htmlcov/ .tox/ .nox/ +.benchmarks/ .coverage .coverage.* .cache diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 000000000..b7e401501 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,26 @@ +# Benchmarks + +[pytest-benchmark](https://pytest-benchmark.readthedocs.io) benchmarks against a fixed synthetic dataset (`dataset.py`), so results are comparable between runs and commits. + +They need the Postgres and Elasticsearch from `docker compose`. They use their own test database (`test_isic_benchmark`) and `benchmark-*` indices, so the development and test databases and indices are never touched. + +```sh +uv run tox -e benchmark # everything +uv run tox -e benchmark -- -k image_search # a subset +uv run tox -e benchmark -- --benchmark-save=before # save the run to .benchmarks/ +uv run tox -e benchmark -- --benchmark-compare # compare with the last saved run +uv run tox -e benchmark -- --benchmark-disable # run each benchmark once, without timing +``` + +- Each session starts from a fresh copy of the dataset. The first run for a schema generates about 100k images, which takes about 2 minutes, and keeps them in a `test_isic_benchmark_` database that later runs copy. Drop old ones with `DROP DATABASE test_isic_benchmark_`. +- Caching is disabled (`DummyCache`, cachalot off), so every request does its full work. +- Benchmarks that write data roll back each round. +- Benchmarks that use the `benchmark_with_memory` fixture also record the peak memory Python allocates, measured in a separate call. Memory allocated outside Python, such as by psycopg, isn't counted. +- CI (`.github/workflows/benchmarks.yml`) runs on each push to master. `history.py record` adds each benchmark's fastest round and peak memory to `benchmarks/data.js` on the `gh-pages` branch, and the job copies `index.html` next to it. Then `history.py check` fails the job when either is more than 50% higher than in the run before it. +- `index.html` charts `data.js`, grouped by module and test, at https://imagemarkup.github.io/isic/benchmarks/. To chart local runs: + + ```sh + uv run tox -e benchmark -- --benchmark-json=.benchmarks/run.json + uv run python benchmarks/history.py record .benchmarks/run.json .benchmarks/site/data.js + cp benchmarks/index.html .benchmarks/site/ && python -m http.server -d .benchmarks/site + ``` diff --git a/benchmarks/__init__.py b/benchmarks/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/benchmarks/conftest.py b/benchmarks/conftest.py new file mode 100644 index 000000000..664d91896 --- /dev/null +++ b/benchmarks/conftest.py @@ -0,0 +1,156 @@ +from collections.abc import Callable +import tracemalloc + +from django.conf import settings +from django.contrib.auth.models import User +from django.db import connection, transaction +from django.test import Client +from django.test.utils import setup_databases, teardown_databases +import pytest + +from benchmarks import dataset +from benchmarks.dataset import Viewer +from isic.core.dsl import parse_query +from isic.core.search import IMAGE_INDEX_MAPPINGS, LESION_INDEX_MAPPINGS, get_elasticsearch_client +from isic.core.tasks import sync_elasticsearch_indices_task + + +@pytest.fixture(scope="session") +def django_db_setup(django_test_environment, django_db_blocker): + # pytest-django only creates the test database for tests marked django_db, and the marker + # wraps each test in a transaction, which some views can't run in (e.g. setting the isolation + # level). so the database is created the same way, but for every benchmark, and copied from + # the generated dataset when there is one. + # migrations create the indices, and refuse to when they exist with other mappings, e.g. from + # a run of another commit. + client = get_elasticsearch_client() + for index in [ + settings.ISIC_ELASTICSEARCH_IMAGES_INDEX, + settings.ISIC_ELASTICSEARCH_LESIONS_INDEX, + ]: + client.indices.delete(index=index, ignore_unavailable=True) + with django_db_blocker.unblock(): + generated = dataset.template_exists() + if generated: + connection.settings_dict["TEST"]["TEMPLATE"] = dataset.template_name() + config = setup_databases( + verbosity=0, interactive=False, aliases={"default"}, serialized_aliases=set() + ) + if not generated: + dataset.generate() + yield + with django_db_blocker.unblock(): + teardown_databases(config, verbosity=0) + + +@pytest.fixture(scope="session") +def manifest(django_db_setup, django_db_blocker) -> dict: + with django_db_blocker.unblock(): + return dataset.manifest() + + +@pytest.fixture(autouse=True) +def _database(django_db_setup, django_db_blocker): + with django_db_blocker.unblock(): + yield + + +INDEX_SETTINGS = { + "index.number_of_replicas": 0, + # nothing writes during a run. an explicit interval also stops idle shards from making the + # next search wait for a refresh. + "index.refresh_interval": "-1", + # every request should do its full work + "index.requests.cache.enable": False, + "index.queries.cache.enabled": False, +} + + +@pytest.fixture(scope="session") +def search_index(django_db_setup, django_db_blocker): + """Index the dataset into the benchmark indices.""" + client = get_elasticsearch_client() + indices = { + settings.ISIC_ELASTICSEARCH_IMAGES_INDEX: IMAGE_INDEX_MAPPINGS, + settings.ISIC_ELASTICSEARCH_LESIONS_INDEX: LESION_INDEX_MAPPINGS, + } + for index, mappings in indices.items(): + client.indices.delete(index=index, ignore_unavailable=True) + client.indices.create(index=index, mappings=mappings, settings=INDEX_SETTINGS) + with django_db_blocker.unblock(): + sync_elasticsearch_indices_task() + client.indices.refresh(index=list(indices)) + # bulk indexing leaves segments that elasticsearch keeps merging in the background, which + # slows whichever benchmarks run first. merged up front, every run searches the same index. + client.indices.forcemerge(index=list(indices), max_num_segments=1) + client.indices.refresh(index=list(indices)) + + +def _fetch(client: Client, path: str) -> bytes: + parse_query.cache_clear() + response = client.get(path) + if response.status_code != 200: + raise AssertionError(f"{path}: {response.status_code}") + if response.streaming: + return b"".join(response.streaming_content) + return response.content + + +@pytest.fixture +def endpoint(manifest) -> Callable[..., Callable[[], bytes]]: + """ + Log in as a user and return a callable that requests a path. + + The path can be a function of the logged in client. It's requested once before returning. + """ + + def make(user: Viewer, path: str | Callable[[Client], str]) -> Callable[[], bytes]: + client = Client() + if user != "anonymous": + client.force_login(User.objects.get(pk=manifest["users"][user])) + if callable(path): + path = path(client) + _fetch(client, path) + return lambda: _fetch(client, path) + + return make + + +@pytest.fixture +def rolled_back() -> Callable[[Callable[[], object]], Callable[[], None]]: + """ + Wrap a function so each call's writes are rolled back. + + A benchmark calls the function many times, and each call has to start from the same data. + pytest-django's rollback only happens once, at the end of a test. + """ + + def wrap(fn): + def run(): + with transaction.atomic(): + fn() + transaction.set_rollback(True) + + return run + + return wrap + + +@pytest.fixture +def benchmark_with_memory(benchmark) -> Callable[[Callable[[], object]], None]: + """ + Benchmark a function, and record the peak memory Python allocates while calling it. + + Memory is measured in a separate call, since tracing allocations slows the function down. + """ + + def run(fn): + tracemalloc.start() + try: + fn() + benchmark.extra_info["peak_memory_mib"] = tracemalloc.get_traced_memory()[1] / 2**20 + finally: + tracemalloc.stop() + benchmark(fn) + + return run diff --git a/benchmarks/dataset.py b/benchmarks/dataset.py new file mode 100644 index 000000000..cef27e92c --- /dev/null +++ b/benchmarks/dataset.py @@ -0,0 +1,678 @@ +""" +A fixed synthetic dataset, generated from SEED so every commit is benchmarked against the same rows. + +Proportions loosely follow production: about 46% of images are public, one non-staff user has a +fifth of the images shared with them, and one collection holds three quarters of all images. +""" + +from datetime import UTC, datetime +import hashlib +from pathlib import Path +import random +from typing import Literal, get_args + +from django.conf import settings +from django.contrib.auth.models import User +from django.db import connection +import numpy as np + +import isic +from isic.core.models import Collection, Doi, Image, ImageEmbedding, IsicId +from isic.core.models.collection import CollectionImage, CollectionShare +from isic.core.models.image import ImageShare +from isic.ingest.models import Accession, AccessionStatus, Cohort, Contributor, Lesion, Patient +from isic.ingest.models.accession_review import AccessionReview +from isic.ingest.models.metadata_version import MetadataVersion +from isic.ingest.models.unstructured_metadata import UnstructuredMetadata +from isic.login.models import Profile +from isic.studies.models import ( + Annotation, + Question, + QuestionChoice, + Response, + Study, + StudyQuestion, + StudyTask, +) + +SEED = 1 +EPOCH = datetime(2016, 1, 1, tzinfo=UTC) + +NamedUser = Literal["staff", "shares", "owner", "user"] +NAMED_USERS: tuple[NamedUser, ...] = get_args(NamedUser) +# not logged in, or logged in as a named user +Viewer = Literal["anonymous"] | NamedUser +# the users most benchmarks run as +USERS: list[Viewer] = ["anonymous", "shares", "staff"] +ANNOTATORS = 10 +EXTRA_USERS = 200 + +PUBLISHED_ACCESSIONS = 100_000 +UNPUBLISHED_ACCESSIONS = 5_000 +IMAGE_SHARES = 20_000 +EMBEDDINGS = 2_000 +PINNED_IMAGES = 20 + +# (percentage of published accessions, fraction of them that is public) +COHORTS = [ + (40, 0.5), + (20, 0.0), + (10, 1.0), + (10, 0.5), + (5, 1.0), + (5, 0.0), + (4, 0.5), + (3, 1.0), + (2, 0.0), + (1, 1.0), +] +CONTRIBUTORS = 5 +OWNED_CONTRIBUTOR = 1 +INGEST_COHORT = "Benchmark Ingest" + +PATIENT_SIZES = [1, 1, 1, 2, 2, 3, 5, 8, 13, 30, 100] +LESION_SIZES = [1, 1, 2, 3, 5, 10] +SHOWCASE_PATIENT_SIZE = 300 +SHOWCASE_LESION_SIZE = 10 +SHOWCASE_ISIC_ID = "ISIC_0000000" +SHOWCASE_LESION_ID = "IL_0000000" + +COLLECTIONS = { + "all_public": "Benchmark All Public", + "large": "Benchmark Large", + "medium": "Benchmark Medium", + "small": "Benchmark Small", + "target": "Benchmark Target", +} +LARGE_COLLECTION_IMAGES = 75_000 +MEDIUM_COLLECTION_IMAGES = 10_000 +SMALL_COLLECTION_IMAGES = 1_000 +MISC_COLLECTIONS = 40 +MISC_COLLECTION_SIZES = [10, 50, 100, 500, 2_000] +DOI_SLUG = "benchmark-medium" + +PUBLIC_STUDY = "Benchmark Public Study" +PRIVATE_STUDY = "Benchmark Private Study" + +# the most common value combinations in production, as (weight, values) +DIAGNOSES = [ + (60, ("Benign", None, None, None, None)), + (12, ("Benign", "Benign melanocytic proliferations", "Nevus", None, None)), + (8, (None, None, None, None, None)), + ( + 5, + ( + "Malignant", + "Malignant adnexal epithelial proliferations - Follicular", + "Basal cell carcinoma", + None, + None, + ), + ), + ( + 4, + ( + "Malignant", + "Malignant melanocytic proliferations (Melanoma)", + "Melanoma, NOS", + None, + None, + ), + ), + ( + 3, + ( + "Benign", + "Benign melanocytic proliferations", + "Nevus", + "Nevus, Atypical, Dysplastic, or Clark", + "Nevus, Dysplastic", + ), + ), + (2, ("Benign", "Benign epidermal proliferations", "Seborrheic keratosis", None, None)), + ( + 2, + ("Benign", "Benign melanocytic proliferations", "Nevus", "Nevus, NOS, Compound", None), + ), + ( + 2, + ( + "Indeterminate", + "Indeterminate epidermal proliferations", + "Solar or actinic keratosis", + None, + None, + ), + ), + ( + 2, + ( + "Malignant", + "Malignant epidermal proliferations", + "Squamous cell carcinoma, NOS", + None, + None, + ), + ), +] +ANATOM_SITES = [ + (25, ("Lower extremity", None, None)), + (15, ("Trunk", "Anterior trunk", None)), + (13, ("Trunk", "Posterior trunk", "Upper back")), + (9, ("Upper extremity", "Upper arm", None)), + (8, ("Upper extremity", None, None)), + (8, ("Trunk", "Posterior trunk", "Mid back")), + (7, (None, None, None)), + (5, ("Trunk", "Posterior trunk", None)), +] +IMAGE_TYPES = [ + (60, ("TBP tile: close-up", None, "3D: XP")), + (15, ("TBP tile: close-up", None, "3D: white")), + (12, ("dermoscopic", None, None)), + (7, ("dermoscopic", "contact polarized", None)), + (4, ("dermoscopic", "contact non-polarized", None)), + (1, ("clinical: close-up", None, None)), + (1, ("TBP tile: overview", None, "3D: XP")), +] +CONFIRMATIONS = [ + (60, (None, None, None)), + (16, ("single contributor clinical assessment", False, None)), + (6, (None, False, True)), + (5, ("serial imaging showing no change", False, None)), + (4, (None, False, None)), + (3, ("histopathology", True, True)), + (2, ("single image expert consensus", False, None)), + (1, ("histopathology", True, False)), +] +SEXES = [(60, "male"), (30, "female"), (10, None)] +FITZPATRICK_SKIN_TYPES = [(95, None), (2, "II"), (1, "I"), (1, "III"), (1, "IV")] +LICENSES = [(46, "CC-BY"), (45, "CC-BY-NC"), (9, "CC-0")] +STATUSES = [ + (90, AccessionStatus.SUCCEEDED), + (5, AccessionStatus.FAILED), + (5, AccessionStatus.SKIPPED), +] + + +def template_name() -> str: + """ + Name the database that keeps a generated dataset. + + It's named after the migrations and this file, so it's only reused for the same schema and + data. + """ + return f"{connection.settings_dict['TEST']['NAME']}_{_dataset_key()}" + + +def template_exists() -> bool: + with connection._nodb_cursor() as cursor: # noqa: SLF001 + cursor.execute("SELECT 1 FROM pg_database WHERE datname = %s", [template_name()]) + return cursor.fetchone() is not None + + +def generate() -> None: + """Generate the dataset into the migrated test database and keep a copy of it.""" + _build() + connection.creation.clone_test_db(suffix=_dataset_key(), verbosity=0, keepdb=True) + + +def manifest() -> dict: + return { + "users": {name: User.objects.get(username=f"benchmark-{name}").pk for name in NAMED_USERS}, + "collections": { + key: Collection.objects.get(name=name).pk for key, name in COLLECTIONS.items() + }, + "doi_slug": DOI_SLUG, + "public_study": Study.objects.get(name=PUBLIC_STUDY).pk, + "ingest_cohort": Cohort.objects.get(name=INGEST_COHORT).pk, + } + + +def _dataset_key() -> str: + digest = hashlib.sha256(Path(__file__).read_bytes()) + for migration in sorted(Path(isic.__file__).parent.glob("*/migrations/*.py")): + digest.update(migration.read_bytes()) + return digest.hexdigest()[:16] + + +def _pick(rng: random.Random, choices: list[tuple[int, object]]): + return rng.choices([value for _, value in choices], [weight for weight, _ in choices])[0] + + +def _metadata(rng: random.Random) -> dict: + diagnosis = _pick(rng, DIAGNOSES) + anatom_site = _pick(rng, ANATOM_SITES) + image_type, dermoscopic_type, tbp_tile_type = _pick(rng, IMAGE_TYPES) + confirm_type, concomitant_biopsy, melanocytic = _pick(rng, CONFIRMATIONS) + return { + **{f"diagnosis_{i}": value for i, value in enumerate(diagnosis, start=1)}, + **{f"anatom_site_{i}": value for i, value in enumerate(anatom_site, start=1)}, + "image_type": image_type, + "dermoscopic_type": dermoscopic_type, + "tbp_tile_type": tbp_tile_type, + "diagnosis_confirm_type": confirm_type, + "concomitant_biopsy": concomitant_biopsy, + "melanocytic": melanocytic, + "sex": _pick(rng, SEXES), + "age": None if rng.random() < 0.1 else rng.randint(0, 85), + "fitzpatrick_skin_type": _pick(rng, FITZPATRICK_SKIN_TYPES), + "copyright_license": _pick(rng, LICENSES), + } + + +def _build() -> None: + rng = random.Random(SEED) + + users = _create_users() + staff = users["staff"] + cohorts, ingest_cohort = _create_cohorts(users) + published, unpublished = _create_accessions(rng, staff, cohorts, ingest_cohort) + images = _create_images(published, staff) + _create_reviews(rng, staff, published, unpublished) + _create_unstructured_metadata(rng, [*published, *unpublished]) + _create_metadata_versions(staff, published[0]) + _set_created_times() + + public_ids = [image.pk for image in images if image.public] + private_ids = [image.pk for image in images if not image.public] + collections = _create_collections(rng, users, [image.pk for image in images], public_ids) + _create_shares(rng, users, private_ids, collections) + _create_embeddings(public_ids[:EMBEDDINGS]) + _create_studies(rng, users, collections) + + with connection.cursor() as cursor: + cursor.execute("REINDEX TABLE core_imageembedding") + cursor.execute("REFRESH MATERIALIZED VIEW materialized_collection_counts") + cursor.execute("VACUUM ANALYZE") + + +def _create_users() -> dict[str, User]: + names = [ + *NAMED_USERS, + *(f"annotator-{i}" for i in range(ANNOTATORS)), + *(f"user-{i}" for i in range(EXTRA_USERS)), + ] + users = { + name: User.objects.create_user( + username=f"benchmark-{name}", + email=f"benchmark-{name}@example.com", + first_name="Benchmark", + last_name=name, + is_staff=name == "staff", + is_superuser=name == "staff", + ) + for name in names + } + Profile.objects.update(accepted_terms=EPOCH) + return users + + +def _create_cohorts(users: dict[str, User]) -> tuple[list[Cohort], Cohort]: + staff = users["staff"] + contributors = [ + Contributor.objects.create( + creator=staff, + institution_name=f"Benchmark Institution {i}", + legal_contact_info="Benchmark", + default_copyright_license="CC-BY", + default_attribution=f"Benchmark Institution {i}", + ) + for i in range(CONTRIBUTORS) + ] + contributors[OWNED_CONTRIBUTOR].owners.add(users["owner"]) + + def cohort(name: str, contributor: Contributor) -> Cohort: + return Cohort.objects.create( + contributor=contributor, + creator=staff, + name=name, + description=name, + default_copyright_license="CC-BY", + default_attribution=contributor.default_attribution, + ) + + cohorts = [ + cohort(f"Benchmark Cohort {i}", contributors[i % CONTRIBUTORS]) for i in range(len(COHORTS)) + ] + return cohorts, cohort(INGEST_COHORT, contributors[0]) + + +def _create_accessions( + rng: random.Random, staff: User, cohorts: list[Cohort], ingest_cohort: Cohort +) -> tuple[list[Accession], list[Accession]]: + patients: list[Patient] = [] + lesions: list[Lesion] = [] + published: list[Accession] = [] + + def accession(n: int, cohort: Cohort, **kwargs) -> Accession: + return Accession( + creator=staff, + cohort=cohort, + attribution=cohort.default_attribution, + original_blob=f"original/IMG_{n:07}.jpg", + original_blob_name=f"IMG_{n:07}.jpg", + original_blob_size=rng.randint(1_000_000, 20_000_000), + **_metadata(rng), + **kwargs, + ) + + def stored(n: int, *, public: bool) -> dict: + blob, thumbnail = f"images/ISIC_{n:07}.jpg", f"thumbnails/ISIC_{n:07}.jpg" + return { + "status": AccessionStatus.SUCCEEDED, + "blob": "" if public else blob, + "thumbnail_256": "" if public else thumbnail, + "sponsored_blob": blob if public else "", + "sponsored_thumbnail_256_blob": thumbnail if public else "", + "blob_size": rng.randint(200_000, 5_000_000), + "thumbnail_256_size": rng.randint(5_000, 30_000), + "width": 1024, + "height": 768, + } + + for cohort_index, (cohort, (percentage, public_fraction)) in enumerate( + zip(cohorts, COHORTS, strict=True) + ): + remaining = PUBLISHED_ACCESSIONS * percentage // 100 + showcase = cohort_index == 0 + + while remaining: + size = min(remaining, SHOWCASE_PATIENT_SIZE if showcase else rng.choice(PATIENT_SIZES)) + patient = None + if showcase or rng.random() < 0.75: + patient = Patient( + id=f"IP_{len(patients):07}", + cohort=cohort, + private_patient_id=f"patient-{len(patients)}", + ) + patients.append(patient) + + lesion, lesion_remaining = None, 0 + for _ in range(size): + if patient and not lesion_remaining: + lesion_remaining = ( + SHOWCASE_LESION_SIZE if showcase else rng.choice(LESION_SIZES) + ) + lesion = Lesion( + id=f"IL_{len(lesions):07}", + cohort=cohort, + private_lesion_id=f"lesion-{len(lesions)}", + ) + lesions.append(lesion) + lesion_remaining -= 1 + + n = len(published) + public = showcase or rng.random() < public_fraction + published.append( + accession(n, cohort, patient=patient, lesion=lesion, **stored(n, public=public)) + ) + + remaining -= size + showcase = False + + unpublished = [] + for n in range(len(published), len(published) + UNPUBLISHED_ACCESSIONS): + status = _pick(rng, STATUSES) + kwargs = ( + {**stored(n, public=False), "blob": f"images/IMG_{n:07}.jpg"} + if status == AccessionStatus.SUCCEEDED + else {"status": status} + ) + unpublished.append(accession(n, ingest_cohort, **kwargs)) + + Patient.objects.bulk_create(patients, batch_size=5_000) + Lesion.objects.bulk_create(lesions, batch_size=5_000) + Accession.objects.bulk_create([*published, *unpublished], batch_size=5_000) + return published, unpublished + + +def _create_images(published: list[Accession], staff: User) -> list[Image]: + isic_ids = IsicId.objects.bulk_create( + (IsicId(id=f"ISIC_{n:07}") for n in range(len(published))), batch_size=5_000 + ) + images = [ + Image( + accession=accession, + isic=isic_id, + creator=staff, + public=bool(accession.sponsored_blob), + ) + for accession, isic_id in zip(published, isic_ids, strict=True) + ] + for pinned, image in enumerate( + (image for image in images[SHOWCASE_PATIENT_SIZE:] if image.public), start=1 + ): + if pinned > PINNED_IMAGES: + break + image.pinned = pinned + return Image.objects.bulk_create(images, batch_size=5_000) + + +def _create_reviews( + rng: random.Random, staff: User, published: list[Accession], unpublished: list[Accession] +) -> None: + reviewed = [ + *((accession, True) for accession in published), + *( + (accession, rng.random() < 0.8) + for accession in unpublished + if accession.status == AccessionStatus.SUCCEEDED and rng.random() < 0.5 + ), + ] + AccessionReview.objects.bulk_create( + ( + AccessionReview(accession=accession, creator=staff, reviewed_at=EPOCH, value=value) + for accession, value in reviewed + ), + batch_size=5_000, + ) + + +def _create_unstructured_metadata(rng: random.Random, accessions: list[Accession]) -> None: + UnstructuredMetadata.objects.bulk_create( + ( + UnstructuredMetadata( + accession=accession, + value=( + {"device": f"device-{rng.randint(0, 9)}", "site": f"site-{rng.randint(0, 99)}"} + if rng.random() < 0.3 + else {} + ), + ) + for accession in accessions + ), + batch_size=5_000, + ) + + +def _create_metadata_versions(staff: User, accession: Accession) -> None: + MetadataVersion.objects.bulk_create( + MetadataVersion( + creator=staff, + accession=accession, + metadata={**accession.metadata, "age": 30 + version}, + unstructured_metadata={"revision": version}, + ) + for version in range(5) + ) + + +def _set_created_times() -> None: + with connection.cursor() as cursor: + cursor.execute( + "UPDATE ingest_accession SET created = %s + id * interval '1 minute'", [EPOCH] + ) + cursor.execute( + """ + UPDATE core_image SET created = a.created + interval '30 days' + FROM ingest_accession a WHERE a.id = core_image.accession_id + """ + ) + + +def _create_collections( + rng: random.Random, users: dict[str, User], image_ids: list[int], public_ids: list[int] +) -> dict[str, Collection]: + staff = users["staff"] + + def collection(name, creator, ids, *, public, pinned=False, locked=False) -> Collection: + created = Collection.objects.create( + creator=creator, name=name, public=public, pinned=pinned, locked=locked + ) + CollectionImage.objects.bulk_create( + (CollectionImage(collection=created, image_id=pk) for pk in ids), batch_size=5_000 + ) + return created + + collections = { + "all_public": collection( + COLLECTIONS["all_public"], staff, public_ids, public=True, pinned=True + ), + "large": collection( + COLLECTIONS["large"], + staff, + sorted(rng.sample(image_ids, LARGE_COLLECTION_IMAGES)), + public=False, + ), + "medium": collection( + COLLECTIONS["medium"], + staff, + sorted(rng.sample(public_ids, MEDIUM_COLLECTION_IMAGES)), + public=True, + locked=True, + ), + "small": collection( + COLLECTIONS["small"], + staff, + [public_ids[0], *sorted(rng.sample(public_ids[1:], SMALL_COLLECTION_IMAGES - 1))], + public=True, + ), + "target": collection(COLLECTIONS["target"], staff, [], public=False), + } + + creators = [staff, users["owner"], users["user"], users["shares"]] + for i in range(MISC_COLLECTIONS): + public = i % 2 == 0 + collections[f"misc-{i}"] = collection( + f"Benchmark Collection {i}", + creators[i % len(creators)], + sorted( + rng.sample(public_ids if public else image_ids, rng.choice(MISC_COLLECTION_SIZES)) + ), + public=public, + ) + + Doi.objects.create( + id=f"{settings.ISIC_DATACITE_DOI_PREFIX}/100000", + slug=DOI_SLUG, + collection=collections["medium"], + creator=staff, + ) + return collections + + +def _create_shares( + rng: random.Random, + users: dict[str, User], + private_ids: list[int], + collections: dict[str, Collection], +) -> None: + staff, grantee = users["staff"], users["shares"] + shared_collections = [collections["target"]] + [ + collection + for key, collection in collections.items() + if key.startswith("misc-") and int(key.removeprefix("misc-")) % 5 == 1 + ] + CollectionShare.objects.bulk_create( + CollectionShare(grantor=staff, grantee=grantee, collection=collection) + for collection in shared_collections + ) + + shared_ids = set(rng.sample(private_ids, IMAGE_SHARES)) + shared_ids.update( + CollectionImage.objects.filter( + collection__in=shared_collections, image__public=False + ).values_list("image_id", flat=True) + ) + ImageShare.objects.bulk_create( + (ImageShare(grantor=staff, grantee=grantee, image_id=pk) for pk in sorted(shared_ids)), + batch_size=5_000, + ) + + +def _create_embeddings(image_ids: list[int]) -> None: + vectors = np.random.default_rng(SEED).standard_normal((len(image_ids), 3584)) + vectors /= np.linalg.norm(vectors, axis=1, keepdims=True) + ImageEmbedding.objects.bulk_create( + ( + ImageEmbedding(image_id=pk, embedding=vector.astype(np.float16)) + for pk, vector in zip(image_ids, vectors, strict=True) + ), + batch_size=200, + ) + + +def _create_studies( + rng: random.Random, users: dict[str, User], collections: dict[str, Collection] +) -> None: + staff = users["staff"] + annotators = [users[f"annotator-{i}"] for i in range(ANNOTATORS)] + questions = [] + for prompt, choices in [ + ("Benchmark: Is this lesion benign or malignant?", ["Benign", "Malignant", "Unsure"]), + ("Benchmark: How confident are you?", ["Low", "Medium", "High"]), + ]: + question = Question.objects.create(prompt=prompt, type="select", official=False) + QuestionChoice.objects.bulk_create( + QuestionChoice(question=question, text=text) for text in choices + ) + questions.append((question, list(question.choices.all()))) + + def study(name, collection, annotators, *, public, annotated_fraction) -> None: + created = Study.objects.create( + creator=staff, + attribution="Benchmark", + name=name, + description=name, + collection=collection, + public=public, + ) + created.owners.add(staff) + StudyQuestion.objects.bulk_create( + StudyQuestion(study=created, question=question, order=order, required=True) + for order, (question, _) in enumerate(questions) + ) + image_ids = list(collection.images.order_by("pk").values_list("pk", flat=True)) + tasks = StudyTask.objects.bulk_create( + ( + StudyTask(study=created, annotator=annotator, image_id=pk) + for annotator in annotators + for pk in image_ids + ), + batch_size=5_000, + ) + annotations = Annotation.objects.bulk_create( + ( + Annotation( + study=created, + image_id=task.image_id, + annotator=task.annotator, + task=task, + start_time=EPOCH, + ) + for task in tasks + if rng.random() < annotated_fraction + ), + batch_size=5_000, + ) + Response.objects.bulk_create( + ( + Response(annotation=annotation, question=question, choice=rng.choice(choices)) + for annotation in annotations + for question, choices in questions + ), + batch_size=5_000, + ) + + study(PUBLIC_STUDY, collections["small"], annotators, public=True, annotated_fraction=0.5) + study(PRIVATE_STUDY, collections["misc-1"], annotators[:3], public=False, annotated_fraction=0) diff --git a/benchmarks/history.py b/benchmarks/history.py new file mode 100644 index 000000000..f72a56b9f --- /dev/null +++ b/benchmarks/history.py @@ -0,0 +1,96 @@ +""" +Keep the benchmark history that index.html charts. + + python benchmarks/history.py record REPORT DATA_JS # add a pytest-benchmark JSON report + python benchmarks/history.py check DATA_JS # fail if the last run regressed + +Each run keeps each benchmark's fastest round, which varies less between runs than the mean, and +the peak memory of the benchmarks that record it. +""" + +import json +from pathlib import Path +import subprocess +import sys +import time + +REPO_URL = "https://github.com/ImageMarkup/isic" +PREFIX = "window.BENCHMARK_DATA = " +# Shared runners are noisy, and each run is on a different runner than the run before it, so only +# large changes fail. +THRESHOLD = 1.5 + + +def load(path: Path) -> dict: + if not path.exists(): + return {"repoUrl": REPO_URL, "entries": {"Benchmark": []}} + return json.loads(path.read_text().removeprefix(PREFIX)) + + +def record(report_path: Path, data_path: Path) -> None: + report = json.loads(report_path.read_text()) + commit = report["commit_info"] + message = subprocess.run( + ["git", "show", "--no-patch", "--format=%B", commit["id"]], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + + data = load(data_path) + data["lastUpdate"] = time.time() * 1000 + data["entries"]["Benchmark"].append( + { + "commit": { + "id": commit["id"], + "message": message, + "timestamp": commit["time"], + "url": f"{data['repoUrl']}/commit/{commit['id']}", + }, + "benches": [ + { + "name": benchmark["fullname"], + "value": benchmark["stats"]["min"] * 1000, + "unit": "ms", + } + for benchmark in report["benchmarks"] + ] + + [ + { + "name": f"{benchmark['fullname']} peak memory", + "value": benchmark["extra_info"]["peak_memory_mib"], + "unit": "MiB", + } + for benchmark in report["benchmarks"] + if "peak_memory_mib" in benchmark["extra_info"] + ], + } + ) + data_path.parent.mkdir(parents=True, exist_ok=True) + data_path.write_text(PREFIX + json.dumps(data)) + + +def check(data_path: Path) -> None: + runs = load(data_path)["entries"]["Benchmark"] + if len(runs) < 2: + return + + previous = {bench["name"]: bench["value"] for bench in runs[-2]["benches"]} + regressions = [ + f"{bench['name']}: {previous[bench['name']]:.3g} -> {bench['value']:.3g} {bench['unit']}" + for bench in runs[-1]["benches"] + if bench["name"] in previous and bench["value"] > previous[bench["name"]] * THRESHOLD + ] + if regressions: + sys.exit( + f"More than {THRESHOLD:.0%} of the run before ({runs[-2]['commit']['id']}):\n" + + "\n".join(regressions) + ) + + +if len(sys.argv) == 4 and sys.argv[1] == "record": + record(Path(sys.argv[2]), Path(sys.argv[3])) +elif len(sys.argv) == 3 and sys.argv[1] == "check": + check(Path(sys.argv[2])) +else: + sys.exit(__doc__) diff --git a/benchmarks/index.html b/benchmarks/index.html new file mode 100644 index 000000000..a1c7f98c3 --- /dev/null +++ b/benchmarks/index.html @@ -0,0 +1,145 @@ + + + + + + ISIC benchmarks + + + + + +

ISIC benchmarks

+ + + + diff --git a/benchmarks/test_collections.py b/benchmarks/test_collections.py new file mode 100644 index 000000000..5aa27ce69 --- /dev/null +++ b/benchmarks/test_collections.py @@ -0,0 +1,33 @@ +from django.urls import reverse +import pytest + +from benchmarks.dataset import USERS + +# (collection, user) +DETAIL_PAGES = { + "small": ("small", "anonymous"), + "medium": ("medium", "anonymous"), + "all_public": ("all_public", "anonymous"), + "large": ("large", "staff"), +} + + +@pytest.mark.parametrize("user", USERS) +def test_collection_list(benchmark, endpoint, user): + benchmark(endpoint(user, f"{reverse('api:collection_list')}?limit=100")) + + +@pytest.mark.parametrize("page", DETAIL_PAGES) +def test_collection_detail_page(benchmark, endpoint, manifest, page): + collection, user = DETAIL_PAGES[page] + pk = manifest["collections"][collection] + benchmark(endpoint(user, reverse("core/collection-detail", args=[pk]))) + + +# 1k, 10k and 75k images +@pytest.mark.parametrize("collection", ["small", "medium", "large"]) +def test_collection_metadata_export(benchmark_with_memory, endpoint, manifest, collection): + pk = manifest["collections"][collection] + benchmark_with_memory( + endpoint("staff", reverse("core/collection-download-metadata", args=[pk])) + ) diff --git a/benchmarks/test_exports.py b/benchmarks/test_exports.py new file mode 100644 index 000000000..12d43991a --- /dev/null +++ b/benchmarks/test_exports.py @@ -0,0 +1,25 @@ +from django.urls import reverse + +from isic.core.models import Image +from isic.core.services import staff_image_metadata_csv +from isic.core.utils.csv import EscapingDictWriter +from isic.core.utils.http import Echo + + +def test_staff_image_metadata_csv(benchmark_with_memory): + """Generate the staff CSV of every image's metadata, as its task does, without uploading it.""" + + def generate(): + rows = staff_image_metadata_csv(qs=Image.objects.all()) + writer = EscapingDictWriter(Echo(), next(rows)) + writer.writeheader() + for row in rows: + writer.writerow(row) + + benchmark_with_memory(generate) + + +def test_study_responses_csv(benchmark_with_memory, endpoint, manifest): + """Download a study's responses. It has 10k, which is typical of production studies.""" + path = reverse("studies/study-download-responses", args=[manifest["public_study"]]) + benchmark_with_memory(endpoint("staff", path)) diff --git a/benchmarks/test_images.py b/benchmarks/test_images.py new file mode 100644 index 000000000..00e65d785 --- /dev/null +++ b/benchmarks/test_images.py @@ -0,0 +1,64 @@ +from urllib.parse import urlencode, urlsplit + +from django.test import Client +from django.urls import reverse +import pytest + +from benchmarks.dataset import SHOWCASE_ISIC_ID, USERS + +# listing and searching take their result counts from elasticsearch +pytestmark = pytest.mark.usefixtures("search_index") + +# the shapes of the slowest production searches. collections are named by their manifest key. +SEARCHES = { + "collection": {"collections": "all_public"}, + "large_collection": {"collections": "large"}, + "diagnosis": {"query": "diagnosis_1:Malignant"}, + "image_type": {"query": "image_type:dermoscopic"}, + "license_in_collection": {"query": "copyright_license:CC-BY", "collections": "all_public"}, + "rcm_case": {"query": "rcm_case_id:*"}, + "nested": { + "query": 'diagnosis_3:"Melanoma, NOS" OR (image_type:dermoscopic AND age_approx:[30 TO 50])' + }, + "pin_sort": {"query": "anatom_site_1:Trunk", "pin_sort": "true"}, +} + + +def _second_page(client: Client) -> str: + first_page = client.get(f"{reverse('api:image_search')}?limit=100").json() + next_url = urlsplit(first_page["next"]) + return f"{next_url.path}?{next_url.query}" + + +@pytest.mark.parametrize("user", USERS) +def test_image_list(benchmark, endpoint, user): + benchmark(endpoint(user, f"{reverse('api:image_list')}?limit=100")) + + +@pytest.mark.parametrize("user", USERS) +@pytest.mark.parametrize("search", SEARCHES) +def test_image_search(benchmark, endpoint, manifest, search, user): + query = {"limit": 100, **SEARCHES[search]} + if "collections" in query: + query["collections"] = manifest["collections"][query["collections"]] + benchmark(endpoint(user, f"{reverse('api:image_search')}?{urlencode(query)}")) + + +@pytest.mark.parametrize("user", USERS) +def test_image_search_second_page(benchmark, endpoint, user): + benchmark(endpoint(user, _second_page)) + + +@pytest.mark.parametrize("user", ["anonymous", "shares"]) +def test_image_detail_api(benchmark, endpoint, user): + benchmark(endpoint(user, reverse("api:image_detail", args=[SHOWCASE_ISIC_ID]))) + + +@pytest.mark.parametrize("user", USERS) +def test_image_detail_page(benchmark, endpoint, user): + benchmark(endpoint(user, reverse("core/image-detail", args=[SHOWCASE_ISIC_ID]))) + + +@pytest.mark.parametrize("user", ["shares", "staff"]) +def test_similar_images(benchmark, endpoint, user): + benchmark(endpoint(user, reverse("api:image_similar", args=[SHOWCASE_ISIC_ID]))) diff --git a/benchmarks/test_lesions.py b/benchmarks/test_lesions.py new file mode 100644 index 000000000..5fe3f21cd --- /dev/null +++ b/benchmarks/test_lesions.py @@ -0,0 +1,20 @@ +from django.urls import reverse +import pytest + +from benchmarks.dataset import SHOWCASE_LESION_ID, USERS + +# the lesion list takes its count from elasticsearch +pytestmark = pytest.mark.usefixtures("search_index") + + +@pytest.mark.parametrize("user", USERS) +def test_lesion_list(benchmark, endpoint, user): + benchmark(endpoint(user, f"{reverse('api:lesion_list')}?limit=100")) + + +def test_lesion_detail_api(benchmark, endpoint): + benchmark(endpoint("anonymous", reverse("api:lesion_detail", args=[SHOWCASE_LESION_ID]))) + + +def test_lesion_detail_page(benchmark, endpoint): + benchmark(endpoint("anonymous", reverse("core/lesion-detail", args=[SHOWCASE_LESION_ID]))) diff --git a/benchmarks/test_pages.py b/benchmarks/test_pages.py new file mode 100644 index 000000000..2bb34ade2 --- /dev/null +++ b/benchmarks/test_pages.py @@ -0,0 +1,39 @@ +from urllib.parse import urlencode + +from django.urls import reverse +import pytest + +# (url name, the manifest key of its argument) +PUBLIC_PAGES = { + "stats": ("stats/stats", None), + "doi": ("core/doi-detail", "doi_slug"), + "study": ("studies/study-detail", "public_study"), +} +STAFF_PAGES = { + "cohort_list": ("ingest/cohort-list", None), + "cohort_review": ("ingest/cohort-detail", "ingest_cohort"), + "embargoed_dashboard": ("core/embargoed-dashboard", None), + "admin_contributors": ("admin:ingest_contributor_changelist", None), +} + + +def _reverse(manifest: dict, name: str, key: str | None) -> str: + return reverse(name, args=[manifest[key]] if key else []) + + +@pytest.mark.parametrize("page", PUBLIC_PAGES) +def test_public_page(benchmark, endpoint, manifest, page): + benchmark(endpoint("anonymous", _reverse(manifest, *PUBLIC_PAGES[page]))) + + +@pytest.mark.parametrize("page", STAFF_PAGES) +def test_staff_page(benchmark, endpoint, manifest, page): + benchmark(endpoint("staff", _reverse(manifest, *STAFF_PAGES[page]))) + + +@pytest.mark.parametrize("user", ["anonymous", "staff"]) +@pytest.mark.parametrize("query", ["ISIC_00", "Benchmark", "Benign"]) +def test_quickfind(benchmark_with_memory, endpoint, query, user): + benchmark_with_memory( + endpoint(user, f"{reverse('api:quickfind')}?{urlencode({'query': query})}") + ) diff --git a/benchmarks/test_search.py b/benchmarks/test_search.py new file mode 100644 index 000000000..a0ee33191 --- /dev/null +++ b/benchmarks/test_search.py @@ -0,0 +1,30 @@ +from django.contrib.auth.models import AnonymousUser, User +import pytest + +from isic.core.dsl import django_parser, es_parser, parse_query +from isic.core.search import build_elasticsearch_query + +QUERIES = { + "simple": "diagnosis_1:Benign", + "boolean": "diagnosis_1:Malignant AND image_type:dermoscopic AND -sex:male", + "nested": ( + 'diagnosis_3:"Melanoma, NOS" OR (image_type:dermoscopic AND ' + "(age_approx:[30 TO 50] OR anatom_site_1:Trunk OR diagnosis_2:*proliferations))" + ), +} +PARSERS = {"django": django_parser, "elasticsearch": es_parser} + + +@pytest.mark.parametrize("parser", PARSERS) +@pytest.mark.parametrize("query", QUERIES) +def test_parse_query(benchmark, query, parser): + """Parse the search DSL without parse_query's cache.""" + benchmark(parse_query.__wrapped__, PARSERS[parser], QUERIES[query]) + + +@pytest.mark.parametrize("collections", ["none", "large"]) +@pytest.mark.parametrize("user", ["anonymous", "shares", "owner", "staff"]) +def test_build_elasticsearch_query(benchmark, manifest, user, collections): + user = AnonymousUser() if user == "anonymous" else User.objects.get(pk=manifest["users"][user]) + collection_ids = [manifest["collections"]["large"]] if collections == "large" else None + benchmark(build_elasticsearch_query, {}, user, collection_ids) diff --git a/benchmarks/test_search_index.py b/benchmarks/test_search_index.py new file mode 100644 index 000000000..dabe115e6 --- /dev/null +++ b/benchmarks/test_search_index.py @@ -0,0 +1,25 @@ +"""Endpoints answered by elasticsearch, against the indexed dataset.""" + +from urllib.parse import urlencode + +from django.urls import reverse +import pytest + +from benchmarks.dataset import USERS + +pytestmark = [ + pytest.mark.usefixtures("search_index"), + # elasticsearch's JVM is slower until it warms up + pytest.mark.benchmark(warmup=True, warmup_iterations=100), +] + + +@pytest.mark.parametrize("user", USERS) +def test_facets(benchmark, endpoint, user): + benchmark(endpoint(user, reverse("api:image_facets"))) + + +@pytest.mark.parametrize("user", USERS) +@pytest.mark.parametrize("query", ["", "diagnosis_1:Malignant"]) +def test_search_size(benchmark, endpoint, query, user): + benchmark(endpoint(user, f"{reverse('api:image_search_size')}?{urlencode({'query': query})}")) diff --git a/benchmarks/test_writes.py b/benchmarks/test_writes.py new file mode 100644 index 000000000..208ebec15 --- /dev/null +++ b/benchmarks/test_writes.py @@ -0,0 +1,93 @@ +import csv +import io + +from django.contrib.auth.models import User +import pytest + +from isic.core.models import Collection, Image +from isic.core.services.collection.image import add_images_to_collection +from isic.ingest.models import Accession, AccessionStatus, Cohort +from isic.ingest.services.accession import update_accession_metadata +from isic.ingest.utils.metadata import ( + validate_archive_consistency, + validate_csv_format_and_filenames, + validate_internal_consistency, +) + +CSV_ROWS = [1_000, 5_000] + + +def _metadata_row(i: int) -> dict[str, str]: + return { + "age": str(20 + i % 60), + "sex": "female" if i % 2 else "male", + "diagnosis": "Benign:Benign melanocytic proliferations:Nevus", + "anatom_site": "Trunk:Anterior trunk", + "patient_id": f"benchmark-patient-{i // 10}", + "lesion_id": f"benchmark-lesion-{i // 2}", + } + + +def _metadata_csv(manifest, rows) -> tuple[Cohort, list[dict[str, str]], str]: + """Return a metadata CSV for the 5k accession ingest cohort, as rows and as text.""" + cohort = Cohort.objects.get(pk=manifest["ingest_cohort"]) + filenames = ( + Accession.objects.filter(cohort=cohort) + .order_by("pk") + .values_list("original_blob_name", flat=True)[:rows] + ) + csv_rows = [{"filename": filename, **_metadata_row(i)} for i, filename in enumerate(filenames)] + csv_file = io.StringIO() + writer = csv.DictWriter(csv_file, fieldnames=list(csv_rows[0])) + writer.writeheader() + writer.writerows(csv_rows) + return cohort, csv_rows, csv_file.getvalue() + + +@pytest.mark.parametrize("images", [1_000, 10_000]) +def test_add_images_to_collection(benchmark, manifest, rolled_back, images): + """Add images to a collection that's shared, which also shares each image.""" + collection = Collection.objects.get(pk=manifest["collections"]["target"]) + qs = Image.objects.filter( + pk__in=list(Image.objects.order_by("pk").values_list("pk", flat=True)[:images]) + ) + benchmark(rolled_back(lambda: add_images_to_collection(collection=collection, qs=qs))) + + +@pytest.mark.parametrize("rows", [100, 1_000]) +def test_update_accession_metadata(benchmark_with_memory, manifest, rolled_back, rows): + """Apply a metadata CSV to unpublished accessions.""" + user = User.objects.get(pk=manifest["users"]["staff"]) + accession_ids = ( + Accession.objects.filter( + cohort_id=manifest["ingest_cohort"], status=AccessionStatus.SUCCEEDED + ) + .order_by("pk") + .values_list("pk", flat=True)[:rows] + ) + metadata = [(pk, _metadata_row(i)) for i, pk in enumerate(accession_ids)] + benchmark_with_memory( + rolled_back(lambda: update_accession_metadata(user=user, metadata=metadata)) + ) + + +@pytest.mark.parametrize("rows", CSV_ROWS) +def test_validate_csv_format_and_filenames(benchmark_with_memory, manifest, rows): + cohort, _, text = _metadata_csv(manifest, rows) + benchmark_with_memory( + lambda: validate_csv_format_and_filenames(csv.DictReader(io.StringIO(text)), cohort) + ) + + +@pytest.mark.parametrize("rows", CSV_ROWS) +def test_validate_internal_consistency(benchmark_with_memory, manifest, rows): + _, csv_rows, _ = _metadata_csv(manifest, rows) + benchmark_with_memory(lambda: validate_internal_consistency(csv_rows)) + + +@pytest.mark.parametrize("rows", CSV_ROWS) +def test_validate_archive_consistency(benchmark_with_memory, manifest, rows): + cohort, _, text = _metadata_csv(manifest, rows) + benchmark_with_memory( + lambda: validate_archive_consistency(csv.DictReader(io.StringIO(text)), cohort) + ) diff --git a/benchmarks/test_zip_download.py b/benchmarks/test_zip_download.py new file mode 100644 index 000000000..55620cd48 --- /dev/null +++ b/benchmarks/test_zip_download.py @@ -0,0 +1,28 @@ +from urllib.parse import urlencode + +from django.core.signing import TimestampSigner +from django.urls import reverse +import pytest + +FILES = { + "file-listing": "api:zip_download_listing", + "metadata-file": "api:zip_download_metadata_file", + "attribution-file": "api:zip_download_attribution_file", +} +# collections are named by their manifest key +SEARCHES = { + "medium_collection": {"query": "", "collections": ["medium"]}, + "malignant": {"query": "diagnosis_1:Malignant", "collections": None}, + "all_public": {"query": "", "collections": None}, +} + + +@pytest.mark.parametrize("search", SEARCHES) +@pytest.mark.parametrize("file", FILES) +def test_zip_download(benchmark_with_memory, endpoint, manifest, file, search): + """Request the files the zip server needs for an anonymous download.""" + token = {"user": None, **SEARCHES[search]} + if token["collections"]: + token["collections"] = [manifest["collections"][key] for key in token["collections"]] + query = urlencode({"token": TimestampSigner().sign_object(token)}) + benchmark_with_memory(endpoint("anonymous", f"{reverse(FILES[file])}?{query}")) diff --git a/isic/settings/benchmark.py b/isic/settings/benchmark.py new file mode 100644 index 000000000..1258fe464 --- /dev/null +++ b/isic/settings/benchmark.py @@ -0,0 +1,30 @@ +from .base import * + +SECRET_KEY = "insecure-secret" +ALLOWED_HOSTS = ["testserver"] + +# Kept apart from the database the test suite creates. +DATABASES["default"]["TEST"] = {"NAME": "test_isic_benchmark"} + +# Every request should do its full work. +CACHES = {"default": {"BACKEND": "django.core.cache.backends.dummy.DummyCache"}} +CACHALOT_ENABLED = False + +# A migration creates the indices, so keep them apart from the development indices. +ISIC_ELASTICSEARCH_IMAGES_INDEX = "benchmark-isic" +ISIC_ELASTICSEARCH_LESIONS_INDEX = "benchmark-isic-lesions" + +# Production storage backends. Unsigned S3 urls are built locally, so no credentials or network +# are needed. +AWS_S3_REGION_NAME = "us-east-1" +AWS_S3_ACCESS_KEY_ID = "benchmark" +AWS_S3_SECRET_ACCESS_KEY = "benchmark" +STORAGES["default"] = { + "BACKEND": "isic.core.storages.s3.IsicS3StaticStorage", + "OPTIONS": {"bucket_name": "isic-storage"}, +} +STORAGES["sponsored"] = { + "BACKEND": "isic.core.storages.s3.IsicS3StaticStorage", + "OPTIONS": {"bucket_name": "isic-archive"}, +} +STORAGES["staticfiles"]["BACKEND"] = "django.contrib.staticfiles.storage.StaticFilesStorage" diff --git a/pyproject.toml b/pyproject.toml index bec32fcc6..b3e985bf1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -118,6 +118,9 @@ profile = [ "memray==1.19.3", "pyinstrument==5.1.3", ] +benchmark = [ + "pytest-benchmark==5.3.0", +] [tool.hatch.build] only-include = [ @@ -200,9 +203,13 @@ ignore = [ "F403", # undefined-local-with-import-star "F405", # undefined-local-with-import-star-usage ] -"**/settings/{development,testing}.py" = [ +"**/settings/{benchmark,development,testing}.py" = [ "S105", # hardcoded-password-string ] +"benchmarks/**" = [ + "PLR0913", # too-many-arguments + "S", # flake8-bandit +] "**/tests/**" = [ "DJ007", # django-all-with-model-form "DJ008", # django-model-without-dunder-str @@ -254,6 +261,8 @@ plugins = [ django_settings_module = "isic.settings.testing" [tool.pytest.ini_options] +# benchmarks/ runs separately +testpaths = ["isic"] markers = [ "playwright: mark a test as using Playwright.", ] diff --git a/tox.ini b/tox.ini index a3d44435b..36a3570db 100644 --- a/tox.ini +++ b/tox.ini @@ -59,6 +59,18 @@ base = commands = pytest --count 5 --random-order {posargs} +[testenv:benchmark] +dependency_groups = + test + benchmark +set_env = + # the benchmarks import their shared modules from the repository root + PYTHONPATH = {toxinidir} +commands = + pytest benchmarks --ds=isic.settings.benchmark --no-cov \ + --benchmark-disable-gc --benchmark-group-by=func --benchmark-columns=min,median,iqr,rounds \ + {posargs} + [testenv:check-migrations] set_env = DJANGO_SETTINGS_MODULE = isic.settings.testing diff --git a/uv.lock b/uv.lock index 13df94afa..528f36fb3 100644 --- a/uv.lock +++ b/uv.lock @@ -1786,6 +1786,9 @@ development = [ ] [package.dev-dependencies] +benchmark = [ + { name = "pytest-benchmark" }, +] dev = [ { name = "isic", extra = ["development"] }, { name = "tox" }, @@ -1891,6 +1894,7 @@ requires-dist = [ provides-extras = ["development"] [package.metadata.requires-dev] +benchmark = [{ name = "pytest-benchmark", specifier = "==5.3.0" }] dev = [ { name = "isic", extras = ["development"] }, { name = "tox", specifier = "==4.58.0" }, @@ -2966,6 +2970,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl", hash = "sha256:1db8e35b67b3d218d818ae653e27f06c3aa420901fa7b081ca98cbedc874e0d0", size = 11842, upload-time = "2024-07-21T12:58:20.04Z" }, ] +[[package]] +name = "py-cpuinfo2" +version = "10.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/dc/97/a8b1ddada14c8280a047c0746f95cb05d94a31b1a331cea22bcdc2b2a82d/py_cpuinfo2-10.1.1.tar.gz", hash = "sha256:7861133863663f16e06eca63b12904ef100b5760415e92372dac0162799a4771", size = 100840, upload-time = "2026-03-25T21:49:40.797Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/23/0a/ba69d2dde1ae12ef1d389ea5a216384c5ff6ef7a1e7a48d1e9b6686f6790/py_cpuinfo2-10.1.1-py3-none-any.whl", hash = "sha256:adc53396bfb206e6498d078ec2ab407f85799ecd819584ac36a8f80a2d4d762d", size = 23791, upload-time = "2026-03-25T21:49:39.574Z" }, +] + [[package]] name = "pyarrow" version = "24.0.0" @@ -3301,6 +3314,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/98/1c/b00940ab9eb8ede7897443b771987f2f4a76f06be02f1b3f01eb7567e24a/pytest_base_url-2.1.0-py3-none-any.whl", hash = "sha256:3ad15611778764d451927b2a53240c1a7a591b521ea44cebfe45849d2d2812e6", size = 5302, upload-time = "2024-01-31T22:42:58.897Z" }, ] +[[package]] +name = "pytest-benchmark" +version = "5.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "py-cpuinfo2" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/63/8f/83a15e40dbc34a580ee56eb56983cae5394c6e94d50cf28fe268e457be25/pytest_benchmark-5.3.0.tar.gz", hash = "sha256:358444d4e89be901ee2b6404fb043ac3d7684002ad7f3563cc153fca6339c965", size = 375410, upload-time = "2026-08-23T17:45:08.891Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/eb/42/7e80f7cfa191e0a766d1de99b4661847415ad5db34f8209d81fd42175b59/pytest_benchmark-5.3.0-py3-none-any.whl", hash = "sha256:920ab1dfcffa718d49aa15ba144c7e357bda59216a0dc308016cc1c7236f719d", size = 48401, upload-time = "2026-08-23T17:45:07.094Z" }, +] + [[package]] name = "pytest-cov" version = "7.1.0"