feat(ci): automated Maven Central release watcher and baseline freshness check (#32)

.github/workflows/ci.yml

@@ -95,6 +95,11 @@ jobs:
           activate-environment: benchmarkenv
           auto-activate-base: false
 
+      - name: Run Python unit tests
+        shell: bash -l {0}
+        run: |
+          python -m unittest discover -s python-utils -p "test_*.py" -v
+
       - name: Run smoke test (dbpedia small only)
         shell: bash -l {0}
         run: |
@@ -258,6 +263,13 @@ jobs:
           cp "$JAR" "$GITHUB_WORKSPACE/libs/corese-core-local.jar"
           echo "corese.next JAR built from $CORESE_COMMIT → libs/corese-core-local.jar"
 
+      - name: Check for new library releases on Maven Central
+        if: github.event_name == 'schedule'
+        run: |
+          source "$CONDA_SH"
+          conda activate benchmarkenv
+          python python-utils/check_new_releases.py --verbose || true
+
       - name: Run benchmark
         env:
           ENGINES: ${{ steps.params.outputs.engines }}

python-utils/check_new_releases.py

@@ -0,0 +1,184 @@
+#!/usr/bin/env python3
+"""Check Maven Central for new stable releases of Jena and RDF4J."""
+import argparse
+import http.client
+import json
+import re
+import sys
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+from pathlib import Path
+
+SEARCH_URL = "https://search.maven.org/solrsearch/select"
+DEFAULT_BASELINE = str(
+    Path(__file__).resolve().parent.parent / "site" / "data" / "baseline.json"
+)
+ARTIFACTS = {
+    "jena": ("org.apache.jena", "jena-arq"),
+    "rdf4j": ("org.eclipse.rdf4j", "rdf4j-storage"),
+}
+
+_UNSTABLE = re.compile(
+    r"(?:^|[.\-_])(snapshot|alpha|beta|rc|m|cr|milestone|dev|preview|pre|ea)\d*(?:$|[.\-_])",
+    re.IGNORECASE,
+)
+_VERSION_RE = re.compile(r"\b\d+\.\d+(?:\.\d+)*\b")
+
+
+def log(verbose, message):
+    if verbose:
+        print(message, file=sys.stderr)
+
+
+def is_stable(version):
+    return bool(re.match(r"\d", version)) and not _UNSTABLE.search(version)
+
+
+def version_key(version):
+    nums = [int(n) for n in re.findall(r"\d+", version)]
+    while len(nums) > 1 and nums[-1] == 0:
+        nums.pop()
+    return tuple(nums)
+
+
+def fetch_versions(group, artifact, timeout=15, max_retries=3):
+    query = urllib.parse.urlencode(
+        {
+            "q": f'g:"{group}" AND a:"{artifact}"',
+            "core": "gav",
+            "rows": "200",
+            "sort": "timestamp desc",
+            "wt": "json",
+        }
+    )
+    request = urllib.request.Request(
+        f"{SEARCH_URL}?{query}", headers={"User-Agent": "check-new-releases/1.0"}
+    )
+    last_err = None
+    for attempt in range(max_retries):
+        try:
+            with urllib.request.urlopen(request, timeout=timeout) as response:
+                payload = json.loads(response.read().decode("utf-8"))
+            return [
+                doc["v"]
+                for doc in payload.get("response", {}).get("docs", [])
+                if "v" in doc
+            ]
+        except (urllib.error.URLError, http.client.HTTPException) as e:
+            last_err = e
+            if attempt < max_retries - 1:
+                time.sleep(0.5 * (attempt + 1))
+    if last_err:
+        raise last_err
+    return []
+
+
+def latest_stable(versions):
+    stable = [v for v in versions if is_stable(v)]
+    return max(stable, key=version_key) if stable else None
+
+
+def _engine_key(name):
+    lowered = str(name).lower()
+    for key in ARTIFACTS:
+        if key in lowered:
+            return key
+    return None
+
+
+def _extract_versions(value):
+    if isinstance(value, str):
+        return set(_VERSION_RE.findall(value))
+    if isinstance(value, dict):
+        return _extract_versions(value.get("version", ""))
+    if isinstance(value, list):
+        found = set()
+        for item in value:
+            found |= _extract_versions(item)
+        return found
+    return set()
+
+
+def baseline_versions(engines):
+    """Return {engine_key: set(versions)} from metadata.engines (dict or list)."""
+    result = {key: set() for key in ARTIFACTS}
+    if isinstance(engines, dict):
+        items = engines.items()
+    elif isinstance(engines, list):
+        items = []
+        for e in engines:
+            if isinstance(e, dict):
+                items.append((e.get("name") or e.get("id") or e.get("engine") or "", e))
+            elif isinstance(e, str):
+                items.append((e, e))
+    else:
+        items = []
+    for name, value in items:
+        key = _engine_key(name)
+        if key:
+            result[key] |= _extract_versions(value)
+    return result
+
+
+def load_baseline(path):
+    with open(path, encoding="utf-8") as fh:
+        data = json.load(fh)
+    return baseline_versions(data.get("metadata", {}).get("engines", {}))
+
+
+def compare(baseline, fetcher=fetch_versions, verbose=False):
+    report = {}
+    for key, (group, artifact) in ARTIFACTS.items():
+        versions = fetcher(group, artifact)
+        latest = latest_stable(versions)
+        known = baseline.get(key, set())
+        current = max(known, key=version_key) if known else None
+        log(verbose, f"{key}: {len(versions)} versions fetched, latest stable={latest}")
+        missing = latest is not None and (
+            current is None or version_key(latest) > version_key(current)
+        )
+        report[key] = {
+            "artifact": f"{group}:{artifact}",
+            "latest": latest,
+            "baseline": current,
+            "missing": missing,
+        }
+    return report
+
+
+def parse_args(argv=None):
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--baseline", default=DEFAULT_BASELINE, help="path to baseline.json")
+    parser.add_argument("--check", action="store_true", help="exit 1 if a new release is missing")
+    parser.add_argument("--json", action="store_true", help="JSON output")
+    parser.add_argument("--verbose", action="store_true", help="log details to stderr")
+    return parser.parse_args(argv)
+
+
+def main(argv=None):
+    args = parse_args(argv)
+    try:
+        log(args.verbose, f"Loading baseline {args.baseline}")
+        baseline = load_baseline(args.baseline)
+        report = compare(baseline, verbose=args.verbose)
+    except (OSError, ValueError, http.client.HTTPException) as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 2
+
+    missing = [k for k, v in report.items() if v["missing"]]
+    if args.json:
+        print(json.dumps({"engines": report, "missing": missing}, indent=2))
+    else:
+        for key, info in report.items():
+            status = "NEW" if info["missing"] else "up-to-date"
+            print(
+                f"{key} ({info['artifact']}): baseline={info['baseline']} "
+                f"latest={info['latest']} [{status}]"
+            )
+    return 1 if args.check and missing else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

python-utils/test_check_new_releases.py

@@ -0,0 +1,163 @@
+import http.client
+import io
+import json
+import os
+import tempfile
+import unittest
+import urllib.error
+from contextlib import redirect_stderr, redirect_stdout
+from unittest import mock
+
+import check_new_releases as cnr
+
+
+def fake_response(versions):
+    body = json.dumps({"response": {"docs": [{"v": v} for v in versions]}}).encode()
+    resp = mock.MagicMock()
+    resp.read.return_value = body
+    resp.__enter__.return_value = resp
+    return resp
+
+
+def fake_urlopen(jena, rdf4j):
+    def _open(request, timeout=None):
+        url = request.full_url if hasattr(request, "full_url") else request
+        return fake_response(jena if "jena-arq" in url else rdf4j)
+
+    return _open
+
+
+class StabilityTests(unittest.TestCase):
+    def test_rejects_unstable(self):
+        for v in [
+            "5.0.0-SNAPSHOT",
+            "5.0.0-rc1",
+            "5.0.0-RC2",
+            "5.0.0-alpha1",
+            "5.0.0-beta2",
+            "5.0.0-M1",
+            "5.0.0.M2",
+            "4.0.0-milestone1",
+        ]:
+            self.assertFalse(cnr.is_stable(v), v)
+
+    def test_accepts_stable(self):
+        for v in ["5.0.0", "4.3.1", "5.1.0.Final", "3.7.7"]:
+            self.assertTrue(cnr.is_stable(v), v)
+
+    def test_latest_stable_numeric_order(self):
+        self.assertEqual(cnr.latest_stable(["5.9.0", "5.10.0", "6.0.0-rc1"]), "5.10.0")
+        self.assertIsNone(cnr.latest_stable(["1.0-SNAPSHOT"]))
+
+    def test_version_key_normalization(self):
+        self.assertEqual(cnr.version_key("5.0"), cnr.version_key("5.0.0"))
+        self.assertGreater(cnr.version_key("5.0.1"), cnr.version_key("5.0"))
+
+
+class FetchTests(unittest.TestCase):
+    def test_fetch_versions(self):
+        with mock.patch("urllib.request.urlopen", return_value=fake_response(["1.0", "2.0"])) as m:
+            self.assertEqual(cnr.fetch_versions("g", "a"), ["1.0", "2.0"])
+        self.assertIn("sort=timestamp+desc", m.call_args[0][0].full_url)
+
+    def test_fetch_versions_retry_on_failure(self):
+        err = urllib.error.URLError("temporary failure")
+        good = fake_response(["2.0.0"])
+        with mock.patch("urllib.request.urlopen", side_effect=[err, good]), \
+             mock.patch("time.sleep"):
+            versions = cnr.fetch_versions("g", "a", max_retries=2)
+            self.assertEqual(versions, ["2.0.0"])
+
+
+class BaselineTests(unittest.TestCase):
+    def test_dict_formats(self):
+        got = cnr.baseline_versions({"Jena": "5.0.0", "rdf4j": {"version": "4.3.1"}})
+        self.assertEqual(got["jena"], {"5.0.0"})
+        self.assertEqual(got["rdf4j"], {"4.3.1"})
+
+    def test_list_of_strings_format(self):
+        got = cnr.baseline_versions(["corese.4.6.5", "jena.6.2.0", "rdf4j.5.3.2"])
+        self.assertEqual(got["jena"], {"6.2.0"})
+        self.assertEqual(got["rdf4j"], {"5.3.2"})
+
+    def test_list_format(self):
+        got = cnr.baseline_versions(
+            [{"name": "Apache Jena", "version": "5.0.0"}, {"name": "RDF4J", "version": "4.3.1"}]
+        )
+        self.assertEqual(got["jena"], {"5.0.0"})
+        self.assertEqual(got["rdf4j"], {"4.3.1"})
+
+    def test_empty(self):
+        self.assertEqual(cnr.baseline_versions(None), {"jena": set(), "rdf4j": set()})
+
+
+class MainTests(unittest.TestCase):
+    def setUp(self):
+        self.tmp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.tmp.cleanup)
+
+    def baseline(self, engines):
+        path = os.path.join(self.tmp.name, "baseline.json")
+        with open(path, "w", encoding="utf-8") as fh:
+            json.dump({"metadata": {"engines": engines}}, fh)
+        return path
+
+    def run_main(self, args, jena, rdf4j):
+        out, err = io.StringIO(), io.StringIO()
+        with mock.patch("urllib.request.urlopen", side_effect=fake_urlopen(jena, rdf4j)), \
+                redirect_stdout(out), redirect_stderr(err):
+            code = cnr.main(args)
+        return code, out.getvalue(), err.getvalue()
+
+    def test_up_to_date(self):
+        path = self.baseline({"jena": "5.0.0", "rdf4j": "4.3.1"})
+        code, out, _ = self.run_main(
+            ["--baseline", path, "--check"], ["5.0.0", "5.1.0-SNAPSHOT"], ["4.3.1", "5.0.0-M1"]
+        )
+        self.assertEqual(code, 0)
+        self.assertIn("up-to-date", out)
+
+    def test_missing_with_check(self):
+        path = self.baseline({"jena": "5.0.0", "rdf4j": "4.3.1"})
+        code, out, _ = self.run_main(["--baseline", path, "--check"], ["5.0.0", "5.1.0"], ["4.3.1"])
+        self.assertEqual(code, 1)
+        self.assertIn("NEW", out)
+
+    def test_missing_without_check_exits_zero(self):
+        path = self.baseline({"jena": "5.0.0", "rdf4j": "4.3.1"})
+        code, _, _ = self.run_main(["--baseline", path], ["5.1.0"], ["4.3.1"])
+        self.assertEqual(code, 0)
+
+    def test_json_output(self):
+        path = self.baseline({"jena": "5.0.0", "rdf4j": "4.3.1"})
+        code, out, _ = self.run_main(["--baseline", path, "--json"], ["5.1.0"], ["4.3.1"])
+        data = json.loads(out)
+        self.assertEqual(code, 0)
+        self.assertEqual(data["missing"], ["jena"])
+        self.assertEqual(data["engines"]["jena"]["latest"], "5.1.0")
+        self.assertEqual(data["engines"]["jena"]["artifact"], "org.apache.jena:jena-arq")
+
+    def test_verbose_logs_to_stderr(self):
+        path = self.baseline({"jena": "5.0.0", "rdf4j": "4.3.1"})
+        _, _, err = self.run_main(["--baseline", path, "--verbose"], ["5.0.0"], ["4.3.1"])
+        self.assertIn("jena: 1 versions fetched", err)
+
+    def test_engine_absent_from_baseline(self):
+        path = self.baseline({"jena": "5.0.0"})
+        code, _, _ = self.run_main(["--baseline", path, "--check"], ["5.0.0"], ["4.3.1"])
+        self.assertEqual(code, 1)
+
+    def test_network_error(self):
+        path = self.baseline({"jena": "5.0.0"})
+        with mock.patch("urllib.request.urlopen", side_effect=urllib.error.URLError("down")), \
+                mock.patch("time.sleep"), \
+                redirect_stderr(io.StringIO()):
+            self.assertEqual(cnr.main(["--baseline", path]), 2)
+
+    def test_missing_baseline_file(self):
+        with redirect_stderr(io.StringIO()):
+            self.assertEqual(cnr.main(["--baseline", "/nonexistent.json"]), 2)
+
+
+if __name__ == "__main__":
+    unittest.main()