diff --git a/src/paperscout/models.py b/src/paperscout/models.py index 3138787..9e28e2b 100644 --- a/src/paperscout/models.py +++ b/src/paperscout/models.py @@ -166,7 +166,19 @@ class MatchReason(str, Enum): PAPER = "paper" -@dataclass(slots=True) +@dataclass(frozen=True, slots=True) +class ProbeTarget: + """One scheduled isocpp.org draft URL to probe: url, tier, and paper-id parts.""" + + url: str + tier: Tier + prefix: str + number: int + revision: int + extension: str + + +@dataclass(frozen=True, slots=True) class ProbeHit: """Successful HEAD to an unpublished draft URL plus optional excerpt text.""" diff --git a/src/paperscout/monitor.py b/src/paperscout/monitor.py index 4c8c703..a63631c 100644 --- a/src/paperscout/monitor.py +++ b/src/paperscout/monitor.py @@ -9,7 +9,7 @@ import threading import time from collections.abc import Mapping, Sequence -from dataclasses import dataclass +from dataclasses import dataclass, field from datetime import datetime, timezone from types import MappingProxyType from typing import Any, cast @@ -104,20 +104,27 @@ class SeedResult: had_prior_state: bool +@dataclass(slots=True) class PollResult: """Outcome of one poll: index diff, probe hits, D→P transitions, per-user matches.""" - def __init__( - self, - diff: DiffResult, - probe_hits: list[ProbeHit], - dp_transitions: list[DPTransition] | None = None, - per_user_matches: dict[str, PerUserMatches] | None = None, - ): - self.diff = diff - self.probe_hits = probe_hits - self.dp_transitions = dp_transitions or [] - self.per_user_matches = per_user_matches or {} + diff: DiffResult + probe_hits: list[ProbeHit] + dp_transitions: list[DPTransition] = field(default_factory=list) + per_user_matches: dict[str, PerUserMatches] = field(default_factory=dict) + + def __post_init__(self) -> None: + if not isinstance(self.diff, DiffResult): + raise TypeError(f"PollResult.diff must be a DiffResult, got {type(self.diff).__name__}") + self.probe_hits = list(self.probe_hits) + if not all(isinstance(h, ProbeHit) for h in self.probe_hits): + raise TypeError("PollResult.probe_hits must contain only ProbeHit instances") + self.dp_transitions = list(self.dp_transitions) + if not all(isinstance(t, DPTransition) for t in self.dp_transitions): + raise TypeError("PollResult.dp_transitions must contain only DPTransition instances") + self.per_user_matches = dict(self.per_user_matches) + if not all(isinstance(v, PerUserMatches) for v in self.per_user_matches.values()): + raise TypeError("PollResult.per_user_matches values must be PerUserMatches instances") # ── Health snapshot (issue 04) ─────────────────────────────────────────────── @@ -265,7 +272,7 @@ async def _poll_sources(self, *, baseline: bool = False) -> tuple[DiffResult, li self._snapshots[source.source_id] = current if source.source_id == SOURCE_WG21_INDEX: - diff = result + diff = cast(DiffResult, result) log.info("INDEX-LOAD papers=%d", len(current)) self._log_index_diff(diff) elif source.source_id == SOURCE_ISO_PROBE: diff --git a/src/paperscout/protocols.py b/src/paperscout/protocols.py index 5c4a30c..5d69ee0 100644 --- a/src/paperscout/protocols.py +++ b/src/paperscout/protocols.py @@ -9,10 +9,14 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable +from typing import TYPE_CHECKING, Any, Protocol, TypeAlias, runtime_checkable if TYPE_CHECKING: - from .monitor import PollResult + from .models import ProbeHit + from .monitor import DiffResult, PollResult + from .sources import OpenStdEntry + + SourceDiff: TypeAlias = DiffResult | list[ProbeHit] | list[OpenStdEntry] # Well-known source identifiers (stable across releases). SOURCE_WG21_INDEX = "wg21_index" @@ -31,7 +35,7 @@ async def fetch(self) -> Any: """Fetch the latest snapshot from this source.""" ... - def diff(self, previous: Any, current: Any) -> Any: + def diff(self, previous: Any, current: Any) -> SourceDiff: """Compare *previous* and *current* snapshots; return source-specific diff.""" ... diff --git a/src/paperscout/sources.py b/src/paperscout/sources.py index 501dd26..398af92 100644 --- a/src/paperscout/sources.py +++ b/src/paperscout/sources.py @@ -13,12 +13,24 @@ from datetime import date, datetime, timedelta, timezone from email.utils import parsedate_to_datetime from types import MappingProxyType +from typing import TYPE_CHECKING import httpx from .config import Settings, settings from .errors import ConfigurationError, FailureCategory, IndexRefreshError -from .models import CycleResult, CycleStatus, IndexRefreshResult, Paper, ProbeHit, Tier +from .models import ( + CycleResult, + CycleStatus, + IndexRefreshResult, + Paper, + ProbeHit, + ProbeTarget, + Tier, +) + +if TYPE_CHECKING: + from .monitor import DiffResult from .protocols import SOURCE_ISO_PROBE, SOURCE_OPEN_STD, SOURCE_WG21_INDEX from .storage import PaperCache, ProbeState, UserWatchlist @@ -240,7 +252,7 @@ def diff( self, previous: dict[str, Paper] | None, current: dict[str, Paper], - ): + ) -> DiffResult: """DataSource: compare two index snapshots.""" from .monitor import _diff_paper_maps @@ -314,11 +326,6 @@ async def _fetch_front_text( return await _fetch_pdf_text(client, pdf_url) -# ── Probe-list entry type ──────────────────────────────────────────────────── -# (url, tier, prefix, number, revision, extension) -_Entry = tuple[str, Tier, str, int, int, str] - - class ISOProber: """Async HEAD probe of isocpp draft URLs: hot every cycle, cold in rotating slices. @@ -399,15 +406,17 @@ async def run_cycle(self) -> CycleResult: self._cycle += 1 self._reset_stats() t0 = time.monotonic() - urls: list[_Entry] = [] + urls: list[ProbeTarget] = [] hot_count = 0 cold_count = 0 try: urls = self._build_probe_list() known_ids = self.index.get_known_paper_ids() - hot_count = sum(1 for u in urls if u[1] in (Tier.WATCHLIST, Tier.FRONTIER, Tier.RECENT)) - cold_count = sum(1 for u in urls if u[1] == Tier.COLD) + hot_count = sum( + 1 for u in urls if u.tier in (Tier.WATCHLIST, Tier.FRONTIER, Tier.RECENT) + ) + cold_count = sum(1 for u in urls if u.tier == Tier.COLD) slice_idx = (self._cycle - 1) % self.cfg.cold_cycle_divisor log.info( "PROBE-START cycle=%d total=%d hot=%d cold=%d slice=%d/%d", @@ -428,8 +437,18 @@ async def run_cycle(self) -> CycleResult: follow_redirects=True, ) as client: tasks = [ - self._probe_one(client, sem, url, prefix, num, rev, ext, tier, known_ids) - for url, tier, prefix, num, rev, ext in urls + self._probe_one( + client, + sem, + t.url, + t.prefix, + t.number, + t.revision, + t.extension, + t.tier, + known_ids, + ) + for t in urls ] results = await asyncio.gather(*tasks, return_exceptions=True) for r in results: @@ -513,7 +532,7 @@ async def run_cycle(self) -> CycleResult: # ── Probe-list builders ────────────────────────────────────────────────── - def _build_probe_list(self) -> list[_Entry]: + def _build_probe_list(self) -> list[ProbeTarget]: frontier = self.index.effective_frontier( self.cfg.frontier_gap_threshold, extra_p_numbers=self.state.paper_nums_from_discovered_iso_urls(), @@ -564,8 +583,8 @@ def _build_hot_list( frontier: int, hot_known: set[int], hot_unknown: set[int], - ) -> list[_Entry]: - results: list[_Entry] = [] + ) -> list[ProbeTarget]: + results: list[ProbeTarget] = [] watchlist_set = self.user_watchlist.get_all_watched_paper_nums() lo = max(1, frontier - self.cfg.frontier_window_below + 1) hi = frontier + self.cfg.frontier_window_above @@ -582,7 +601,16 @@ def _build_hot_list( for rev in range(latest + 1, latest + self.cfg.hot_revision_depth + 1): for ext in self.cfg.probe_extensions: url = f"{ISO_BASE}D{num:04d}R{rev}{ext}" - results.append((url, tier, "D", num, rev, ext)) + results.append( + ProbeTarget( + url=url, + tier=tier, + prefix="D", + number=num, + revision=rev, + extension=ext, + ) + ) # Unknown hot (frontier gaps): probe D+P, R0 .. gap_max_rev for num in sorted(hot_unknown): @@ -590,7 +618,16 @@ def _build_hot_list( for rev in range(0, self.cfg.gap_max_rev + 1): for ext in self.cfg.probe_extensions: url = f"{ISO_BASE}{prefix}{num:04d}R{rev}{ext}" - results.append((url, Tier.FRONTIER, prefix, num, rev, ext)) + results.append( + ProbeTarget( + url=url, + tier=Tier.FRONTIER, + prefix=prefix, + number=num, + revision=rev, + extension=ext, + ) + ) return results @@ -600,10 +637,10 @@ def _build_cold_slice( frontier: int, hot_known: set[int], hot_unknown: set[int], - ) -> list[_Entry]: + ) -> list[ProbeTarget]: """Return the 1/cold_cycle_divisor slice of cold numbers for this cycle.""" slice_idx = (cycle - 1) % self.cfg.cold_cycle_divisor - results: list[_Entry] = [] + results: list[ProbeTarget] = [] known_p_nums = self.index.known_p_numbers() cold_known = known_p_nums - hot_known @@ -620,7 +657,16 @@ def _build_cold_slice( for rev in range(latest + 1, latest + self.cfg.cold_revision_depth + 1): for ext in self.cfg.probe_extensions: url = f"{ISO_BASE}D{num:04d}R{rev}{ext}" - results.append((url, Tier.COLD, "D", num, rev, ext)) + results.append( + ProbeTarget( + url=url, + tier=Tier.COLD, + prefix="D", + number=num, + revision=rev, + extension=ext, + ) + ) # Cold unknown gap numbers: D+P, R0 .. gap_max_rev for num in sorted(cold_unknown): @@ -630,7 +676,16 @@ def _build_cold_slice( for rev in range(0, self.cfg.gap_max_rev + 1): for ext in self.cfg.probe_extensions: url = f"{ISO_BASE}{prefix}{num:04d}R{rev}{ext}" - results.append((url, Tier.COLD, prefix, num, rev, ext)) + results.append( + ProbeTarget( + url=url, + tier=Tier.COLD, + prefix=prefix, + number=num, + revision=rev, + extension=ext, + ) + ) return results diff --git a/tests/test_monitor.py b/tests/test_monitor.py index fe68369..b7c57ad 100644 --- a/tests/test_monitor.py +++ b/tests/test_monitor.py @@ -202,6 +202,31 @@ def test_explicit_per_user_matches(self): result = PollResult(diff=diff, probe_hits=[], per_user_matches={"U1": pum}) assert "U1" in result.per_user_matches + def test_rejects_invalid_diff_type(self): + with pytest.raises(TypeError, match=r"PollResult\.diff must be a DiffResult"): + PollResult(diff={}, probe_hits=[]) # type: ignore[arg-type] + + def test_rejects_invalid_probe_hit_element(self): + diff = DiffResult(new_papers=[], updated_papers=[]) + with pytest.raises(TypeError, match=r"PollResult\.probe_hits must contain only ProbeHit"): + PollResult(diff=diff, probe_hits=["not-a-hit"]) # type: ignore[list-item] + + def test_rejects_invalid_dp_transition_element(self): + diff = DiffResult(new_papers=[], updated_papers=[]) + with pytest.raises( + TypeError, + match=r"PollResult\.dp_transitions must contain only DPTransition", + ): + PollResult(diff=diff, probe_hits=[], dp_transitions=["not-a-transition"]) # type: ignore[list-item] + + def test_rejects_invalid_per_user_matches_value(self): + diff = DiffResult(new_papers=[], updated_papers=[]) + with pytest.raises( + TypeError, + match=r"PollResult\.per_user_matches values must be PerUserMatches", + ): + PollResult(diff=diff, probe_hits=[], per_user_matches={"U1": "bad"}) # type: ignore[dict-item] + # ── Scheduler ───────────────────────────────────────────────────────────────── diff --git a/tests/test_sources.py b/tests/test_sources.py index 43756ba..4522ef1 100644 --- a/tests/test_sources.py +++ b/tests/test_sources.py @@ -598,7 +598,7 @@ def test_build_probe_list_includes_discovered_draft_in_frontier_band(self, fake_ index._parse_and_index(data) state.mark_discovered("https://isocpp.org/files/papers/D4165R0.pdf") urls = prober._build_probe_list() - assert any(r[0].endswith("D4165R0.pdf") for r in urls) + assert any(r.url.endswith("D4165R0.pdf") for r in urls) def _set_frontier(self, index: WG21Index, frontier: int) -> None: index._max_p = frontier @@ -620,8 +620,8 @@ def test_hot_watchlist_paper_probed_every_cycle(self, fake_pool): hot_known, hot_unknown = prober._hot_numbers(frontier) assert 2300 in hot_known urls = prober._build_hot_list(frontier, hot_known, hot_unknown) - numbers = {r[3] for r in urls} - tiers = {r[1] for r in urls if r[3] == 2300} + numbers = {r.number for r in urls} + tiers = {r.tier for r in urls if r.number == 2300} assert 2300 in numbers assert "watchlist" in tiers @@ -637,7 +637,7 @@ def test_hot_frontier_generates_urls(self, fake_pool): frontier = index.effective_frontier() hot_known, hot_unknown = prober._hot_numbers(frontier) urls = prober._build_hot_list(frontier, hot_known, hot_unknown) - numbers = {r[3] for r in urls} + numbers = {r.number for r in urls} assert 100 in numbers assert 101 in numbers assert 102 in numbers @@ -658,7 +658,7 @@ def test_hot_frontier_unknown_numbers_get_d_and_p(self, fake_pool): hot_known, hot_unknown = prober._hot_numbers(frontier) urls = prober._build_hot_list(frontier, hot_known, hot_unknown) # 101, 102 are unknown frontier numbers - prefixes_for_101 = {r[2] for r in urls if r[3] == 101} + prefixes_for_101 = {r.prefix for r in urls if r.number == 101} assert "D" in prefixes_for_101 assert "P" in prefixes_for_101 @@ -739,7 +739,7 @@ def test_cold_slice_covers_all_numbers_over_divisor_cycles(self, fake_pool): for cycle in range(1, 5): # one full divisor window urls = prober._build_cold_slice(cycle, frontier, hot_known, hot_unknown) # Only track known papers (not gap numbers 1..9) - probed_known.update(r[3] for r in urls if r[3] in cold_known) + probed_known.update(r.number for r in urls if r.number in cold_known) # Every cold-known paper must appear in exactly one slice per window assert cold_known == probed_known @@ -758,7 +758,7 @@ def test_cold_slice_index_is_deterministic(self, fake_pool): hot_known, hot_unknown = prober._hot_numbers(frontier) slice_a = prober._build_cold_slice(1, frontier, hot_known, hot_unknown) slice_b = prober._build_cold_slice(5, frontier, hot_known, hot_unknown) - assert {r[3] for r in slice_a} == {r[3] for r in slice_b} + assert {r.number for r in slice_a} == {r.number for r in slice_b} def test_cold_gap_numbers_probed_with_d_and_p(self, fake_pool): prober, index, _ = self._make_prober( @@ -777,7 +777,9 @@ def test_cold_gap_numbers_probed_with_d_and_p(self, fake_pool): frontier = 10 hot_known, hot_unknown = prober._hot_numbers(frontier) urls = prober._build_cold_slice(1, frontier, hot_known, hot_unknown) - gap_entries = [(r[2], r[3]) for r in urls if r[1] == "cold" and r[3] not in (9, 10)] + gap_entries = [ + (r.prefix, r.number) for r in urls if r.tier == "cold" and r.number not in (9, 10) + ] prefixes_found = {p for p, _ in gap_entries} assert "D" in prefixes_found assert "P" in prefixes_found @@ -845,7 +847,7 @@ def test_build_hot_list_explicit_ranges_update_frontier_range(self, fake_pool): frontier = 100 hot_known, hot_unknown = prober._hot_numbers(frontier) urls = prober._build_hot_list(frontier, hot_known, hot_unknown) - assert any(r[3] == 200 and r[1] == "frontier" for r in urls) + assert any(r.number == 200 and r.tier == "frontier" for r in urls) def test_build_hot_list_latest_none_uses_minus_one(self, fake_pool): """Known hot numbers with get_max_revision None should start from R0.""" @@ -865,7 +867,7 @@ def test_build_hot_list_latest_none_uses_minus_one(self, fake_pool): hot_known, hot_unknown = prober._hot_numbers(frontier) assert 9999 in hot_known urls = prober._build_hot_list(frontier, hot_known, hot_unknown) - revisions = [r[4] for r in urls if r[3] == 9999] + revisions = [r.revision for r in urls if r.number == 9999] assert 0 in revisions # latest=-1 → start_rev=0 def test_cold_known_skips_when_latest_none(self, fake_pool): @@ -885,7 +887,7 @@ def test_cold_known_skips_when_latest_none(self, fake_pool): frontier = 5 hot_known, hot_unknown = prober._hot_numbers(frontier) urls = prober._build_cold_slice(1, frontier, hot_known, hot_unknown) - cold_nums = {r[3] for r in urls if r[1] == "cold"} + cold_nums = {r.number for r in urls if r.tier == "cold"} assert 4 not in cold_nums # skipped because get_max_revision is None assert 5 in cold_nums # normally probed @@ -943,7 +945,7 @@ def test_cold_excludes_hot_numbers(self, fake_pool): frontier = index.effective_frontier() hot_known, hot_unknown = prober._hot_numbers(frontier) urls = prober._build_cold_slice(1, frontier, hot_known, hot_unknown) - cold_numbers = {r[3] for r in urls} + cold_numbers = {r.number for r in urls} assert 5000 not in cold_numbers # in watchlist → hot