Run the Python suite in parallel and stop vitest's double work

.devenv-version

@@ -1 +1 @@
-0.2.10
+0.2.11

.github/workflows/devenv-check.yml

@@ -54,7 +54,7 @@ on:
         type: boolean
         default: true
       retention-days:
-        description: Days to keep python-dist and the ci-passed marker.
+        description: Days to keep python-dist, the ci-passed marker and coverage.
         type: number
         default: 7
     outputs:
@@ -334,6 +334,17 @@ jobs:
           name: test-results-${{ strategy.job-index }}
           path: "**/test-results/pytest/*.xml"
 
+      # pytest-cov's HTML report. A project whose tests write none, such as
+      # one with pytest addopts of its own, has nothing to upload.
+      - name: Upload Coverage
+        if: matrix.junit && !cancelled() && steps.work.outcome != 'skipped'
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: coverage-${{ strategy.job-index }}
+          path: "**/test-results/coverage/"
+          retention-days: ${{ inputs.retention-days }}
+          if-no-files-found: ignore
+
       - name: Publish Test Report
         if: matrix.junit && !cancelled() && steps.upload-tests.outcome == 'success'
         uses: mikepenz/action-junit-report@b7b80d7ac5b1da46d6aaff9fb57ad50d8453bcb3 # v6.6.0

.gitignore

@@ -50,7 +50,6 @@ MANIFEST
 TODO.md
 __pycache__/
 __pypackages__/
-__snapshots__
 build
 celerybeat-schedule
 celerybeat.pid

frontend/package.json

@@ -12,8 +12,9 @@
     "devBuild": "vite build --mode development",
     "fix": "eslint_d --cache --fix . && prettier --write .",
     "lint": "eslint_d --cache . && prettier --check .",
-    "test": "vitest --reporter verbose --update",
-    "test:ci": "vitest run --reporter verbose --update"
+    "test": "vitest --reporter verbose",
+    "test:ci": "vitest run --reporter verbose",
+    "test:update": "vitest run --update"
   },
   "dependencies": {
     "@unhead/vue": "^3.4.3",

frontend/tests/unit/reader-nav-button.test.js

@@ -40,7 +40,12 @@ test("reader-nav-button", async () => {
   const readerStore = useReaderStore();
 
   // test initial state
-  expect(wrapper.html()).toMatchSnapshot();
+  expect(wrapper.element.tagName).toBe("A");
+  expect(wrapper.attributes("href")).toBe("/read/0?page=0");
+  expect(wrapper.attributes("title")).toBe("Page 0");
+  expect(wrapper.attributes("aria-disabled")).toBe("true");
+  expect(wrapper.classes()).toContain("readerNavButton");
+  expect(wrapper.classes()).toContain("readerNavButtonLeft");
   expect(wrapper.text()).toContain("0");
 
   // push new route

frontend/vite.config.js

@@ -207,12 +207,14 @@ const config = defineConfig(({ mode }) => {
     plugins: [
       vue(),
       vuetify({ autoImport: true }),
-      checker({
-        eslint: {
-          lintCommand: "eslint_d --cache .", // "./src/**/*.{js,vue}"',
-          useFlatConfig: true,
-        },
-      }),
+      // Lint runs ESLint over the frontend; vitest needn't run it again.
+      !IS_TEST_ENV &&
+        checker({
+          eslint: {
+            lintCommand: "eslint_d --cache .", // "./src/**/*.{js,vue}"',
+            useFlatConfig: true,
+          },
+        }),
       dynamicBase({
         publicPath: PUBLIC_PATH,
       }),

package.json

@@ -2,8 +2,8 @@
   "description": "linting for top level codex project & django backend",
   "type": "module",
   "scripts": {
-    "fix": "eslint_d --cache --fix . && prettier --write . && bin/prettier-nginx.sh --write",
-    "lint": "eslint_d --cache . && prettier --check ."
+    "fix": "eslint_d --cache --ignore-pattern frontend/ --fix . && prettier --write . && bin/prettier-nginx.sh --write",
+    "lint": "eslint_d --cache --ignore-pattern frontend/ . && prettier --check ."
   },
   "browserslist": [
     "> 1%",

pyproject.toml

@@ -108,6 +108,7 @@ test = [
   "pytest-asyncio~=1.0",
   "pytest-cov~=7.0",
   "pytest-django~=4.1",
+  "pytest-xdist~=3.8",
   "rnc2rng~=2.7",
 ]
 docs = ["mkdocs-minify-plugin~=0.8", "mkdocs~=1.6"]
@@ -243,10 +244,21 @@ omit = [
 [tool.pytest]
 minversion = "9.0"
 addopts = [
+  "-p",
+  "tests.pytest_config_dir",
+  "--numprocesses=auto",
+  "--dist=loadscope",
+  # Keep pytest-django's order, TestCase before TransactionTestCase, on every
+  # worker. loadscope otherwise sorts scopes by size, and a
+  # TransactionTestCase's flush then strips the migration-seeded rows from
+  # the TestCases that follow it on that worker.
+  "--no-loadscope-reorder",
   "--cov",
   "--cov-append",
   "--cov-report=html",
   "--cov-report=term",
+  "--durations=25",
+  "--durations-min=1.0",
   "--junit-xml=test-results/pytest/results.xml",
   "--strict-config",
   "--strict-markers",

tests/importer/test_basic.py

@@ -16,7 +16,6 @@
 from deepdiff import DeepDiff
 from django.core.cache import cache
 from django.test import TestCase
-from django.test.testcases import SerializeMixin
 from loguru import logger
 
 from codex.librarian.mp_queue import LIBRARIAN_QUEUE
@@ -946,9 +945,7 @@ def run_full_import(importer) -> None:
     importer.full_text_search()
 
 
-class BaseTestImporter(SerializeMixin, TestCase, ABC):
-    lockfile = __file__
-
+class BaseTestImporter(TestCase, ABC):
     @override
     @classmethod
     def setUpTestData(cls):

tests/importer/test_delete_backstop.py

@@ -23,7 +23,6 @@
 
 import os
 import shutil
-import threading
 from pathlib import Path
 from threading import Event, Lock
 from typing import Final, override
@@ -68,10 +67,14 @@ class _DeleteTestBase(BaseTestImporter):
     @override
     def setUp(self) -> None:
         super().setUp()
-        # Every delete-bearing batch waits between its two probes. Patch
-        # the wait rather than shortening it, so the tests assert *that*
-        # it waited without any of them paying for it.
-        wait_patcher = patch.object(threading.Event, "wait")
+        # Every delete-bearing batch waits between its two probes, on the
+        # importer's abort event. Patch that event's wait rather than
+        # shortening it, so the tests assert *that* it waited without any
+        # of them paying for it. Not ``Event.wait`` for every event:
+        # ``Thread.start()`` waits on one too, and the first queue ``put``
+        # in a process starts the queue's feeder thread.
+        self.abort_event = Event()
+        wait_patcher = patch.object(self.abort_event, "wait")
         self.wait_mock = wait_patcher.start()
         self.addCleanup(wait_patcher.stop)
         self.library = Library.objects.get(pk=self.task.library_id)
@@ -116,17 +119,21 @@ def _make_subdir_folder(self) -> Folder:
             library=self.library, path=str(_SUBDIR), name=_SUBDIR.name
         )
 
+    def _new_importer(self, task: ImportTask) -> ComicImporter:
+        """Build an importer that waits on the test's patched abort event."""
+        return ComicImporter(task, logger, LIBRARIAN_QUEUE, Lock(), self.abort_event)
+
     def _delete(self, **task_kwargs) -> ComicImporter:
         """Run the delete phase with a fresh importer, as a scan would."""
         task = ImportTask(library_id=self.library.pk, **task_kwargs)
-        importer = ComicImporter(task, logger, LIBRARIAN_QUEUE, Lock(), Event())
+        importer = self._new_importer(task)
         importer.delete()
         return importer
 
     def _revive(self, **task_kwargs) -> ComicImporter:
         """Run the revival pre-phase with a fresh importer."""
         task = ImportTask(library_id=self.library.pk, **task_kwargs)
-        importer = ComicImporter(task, logger, LIBRARIAN_QUEUE, Lock(), Event())
+        importer = self._new_importer(task)
         importer.unstamp_revived()
         return importer
 
@@ -198,7 +205,7 @@ class TestMassDeleteWarning(_DeleteTestBase):
     def _importer(self) -> tuple[ComicImporter, MagicMock]:
         """Build an importer whose log is captured for assertions."""
         task = ImportTask(library_id=self.library.pk)
-        importer = ComicImporter(task, logger, LIBRARIAN_QUEUE, Lock(), Event())
+        importer = self._new_importer(task)
         mock_log = MagicMock()
         importer.log = mock_log
         return importer, mock_log
@@ -303,7 +310,7 @@ def test_unreadable_paths_are_reported(self) -> None:
         task = ImportTask(
             library_id=self.library.pk, files_deleted=frozenset({_EXTANT})
         )
-        importer = ComicImporter(task, logger, LIBRARIAN_QUEUE, Lock(), Event())
+        importer = self._new_importer(task)
         mock_log = MagicMock()
         importer.log = mock_log
 
@@ -392,7 +399,7 @@ def test_cascade_is_announced(self) -> None:
         task = ImportTask(
             library_id=self.library.pk, dirs_deleted=frozenset({str(_SUBDIR)})
         )
-        importer = ComicImporter(task, logger, LIBRARIAN_QUEUE, Lock(), Event())
+        importer = self._new_importer(task)
         mock_log = MagicMock()
         importer.log = mock_log
 
@@ -418,7 +425,7 @@ def _poll_and_delete(self) -> None:
         # Nothing under the unreadable directory may even be proposed.
         assert not [path for path in task.files_deleted if str(_SUBDIR) in path]
         assert not [path for path in task.dirs_deleted if str(_SUBDIR) in path]
-        importer = ComicImporter(task, logger, LIBRARIAN_QUEUE, Lock(), Event())
+        importer = self._new_importer(task)
         importer.delete()
 
     def test_bookmarks_survive_an_unreadable_publisher_directory(self) -> None:
@@ -536,7 +543,7 @@ def test_revival_is_reported(self) -> None:
         task = ImportTask(
             library_id=self.library.pk, files_deleted=frozenset({_EXTANT})
         )
-        importer = ComicImporter(task, logger, LIBRARIAN_QUEUE, Lock(), Event())
+        importer = self._new_importer(task)
         mock_log = MagicMock()
         importer.log = mock_log
 

tests/pytest_config_dir.py

@@ -0,0 +1,25 @@
+"""Give every pytest process its own CODEX_CONFIG_DIR before Django loads."""
+
+import os
+
+from tests.tmp_dirs import tmp_dir
+
+# Registered with ``-p`` in ``addopts`` because pytest-django imports the
+# settings in ``pytest_load_initial_conftests``, before ``conftest.py`` loads;
+# ``-p`` modules import earlier, during option pre-parsing.
+#
+# Without this every run shares ``./config``: the developer's real config,
+# and above all its file-based Django caches. Under xdist, cachalot then
+# serves one worker's cached queries to the others, each of which has its
+# own in-memory test database, and test-DB setup fails with ``UNIQUE
+# constraint failed: django_content_type...``. ``cache.clear()`` in one
+# worker would also wipe the others' entries.
+#
+# xdist workers inherit the controller's environment, so a worker always
+# replaces the inherited value. A serial run keeps an explicit
+# CODEX_CONFIG_DIR.
+if "PYTEST_XDIST_WORKER" in os.environ or "CODEX_CONFIG_DIR" not in os.environ:
+    _config_dir = tmp_dir("config")
+    # Settings write the secret key files straight into it at import.
+    _config_dir.mkdir(exist_ok=True)
+    os.environ["CODEX_CONFIG_DIR"] = str(_config_dir)

tests/test_janitor_cleanup_custom_covers.py

@@ -108,8 +108,11 @@ def setUp(self) -> None:
             path.write_bytes(b"not really a png")
             self.covers.append(CustomCover.objects.create(path=str(path)))
         # Patch the second look's wait so the tests assert *that* it
-        # waited without any of them paying for it.
-        wait_patcher = patch.object(Event, "wait")
+        # waited without any of them paying for it. Only the janitor's own
+        # abort event: ``Thread.start()`` waits on an ``Event`` too, and
+        # would satisfy ``assert_called()`` on its own.
+        self.abort_event = Event()  # pyright: ignore[reportUninitializedInstanceVariable]
+        wait_patcher = patch.object(self.abort_event, "wait")
         self.wait_mock = wait_patcher.start()  # pyright: ignore[reportUninitializedInstanceVariable]
         self.addCleanup(wait_patcher.stop)
 
@@ -118,9 +121,10 @@ def tearDown(self) -> None:
         """Remove the temp cover tree."""
         shutil.rmtree(TMP_DIR, ignore_errors=True)
 
-    @staticmethod
-    def _cleanup() -> None:
-        Janitor(logger, LIBRARIAN_QUEUE, Lock(), Event()).cleanup_custom_covers()
+    def _cleanup(self) -> None:
+        Janitor(
+            logger, LIBRARIAN_QUEUE, Lock(), self.abort_event
+        ).cleanup_custom_covers()
 
     def _surviving_pks(self) -> set[int]:
         return set(CustomCover.objects.values_list("pk", flat=True))