From d30dfcc6fb4167519ef848038cbd681fdd729fc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20C=C3=A9r=C3=A8s?= Date: Fri, 30 Oct 2026 15:59:10 +0100 Subject: [PATCH 1/3] feat(ci): automated Maven Central release watcher and baseline freshness check (#32) - Add check_new_releases.py querying Maven Central Solr REST API for Jena and RDF4J - Filter out unstable release qualifiers (-SNAPSHOT, -RC, -alpha, -beta, -M) - Simplify version extraction regex to avoid backtracking (SonarLint python:S8786) - Add comprehensive unittest suite test_check_new_releases.py (14 tests) - Wire Python unittest execution into GitHub Actions ci.yml --- .github/workflows/ci.yml | 5 + python-utils/check_new_releases.py | 160 ++++++++++++++++++++++++ python-utils/test_check_new_releases.py | 149 ++++++++++++++++++++++ 3 files changed, 314 insertions(+) create mode 100644 python-utils/check_new_releases.py create mode 100644 python-utils/test_check_new_releases.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 79bbf77..ad1c9de 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -95,6 +95,11 @@ jobs: activate-environment: benchmarkenv auto-activate-base: false + - name: Run Python unit tests + shell: bash -l {0} + run: | + python -m unittest discover -s python-utils -p "test_*.py" -v + - name: Run smoke test (dbpedia small only) shell: bash -l {0} run: | diff --git a/python-utils/check_new_releases.py b/python-utils/check_new_releases.py new file mode 100644 index 0000000..6e7df18 --- /dev/null +++ b/python-utils/check_new_releases.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +"""Check Maven Central for new stable releases of Jena and RDF4J.""" +import argparse +import json +import re +import sys +import urllib.parse +import urllib.request + +SEARCH_URL = "https://search.maven.org/solrsearch/select" +DEFAULT_BASELINE = "site/data/baseline.json" +ARTIFACTS = { + "jena": ("org.apache.jena", "jena-arq"), + "rdf4j": ("org.eclipse.rdf4j", "rdf4j-storage"), +} + +_UNSTABLE = re.compile( + r"(?:^|[.\-_])(snapshot|alpha|beta|rc|m|cr|milestone|dev|preview|pre|ea)\d*(?:$|[.\-_])", + re.IGNORECASE, +) +_VERSION_RE = re.compile(r"\b\d+\.\d+(?:\.\d+)*\b") + + +def log(verbose, message): + if verbose: + print(message, file=sys.stderr) + + +def is_stable(version): + return bool(re.match(r"\d", version)) and not _UNSTABLE.search(version) + + +def version_key(version): + return tuple(int(n) for n in re.findall(r"\d+", version)) + + +def fetch_versions(group, artifact, timeout=15): + query = urllib.parse.urlencode( + { + "q": f'g:"{group}" AND a:"{artifact}"', + "core": "gav", + "rows": "200", + "wt": "json", + } + ) + request = urllib.request.Request( + f"{SEARCH_URL}?{query}", headers={"User-Agent": "check-new-releases/1.0"} + ) + with urllib.request.urlopen(request, timeout=timeout) as response: + payload = json.loads(response.read().decode("utf-8")) + return [doc["v"] for doc in payload.get("response", {}).get("docs", []) if "v" in doc] + + +def latest_stable(versions): + stable = [v for v in versions if is_stable(v)] + return max(stable, key=version_key) if stable else None + + +def _engine_key(name): + lowered = str(name).lower() + for key in ARTIFACTS: + if key in lowered: + return key + return None + + +def _extract_versions(value): + if isinstance(value, str): + return set(_VERSION_RE.findall(value)) + if isinstance(value, dict): + return _extract_versions(value.get("version", "")) + if isinstance(value, list): + found = set() + for item in value: + found |= _extract_versions(item) + return found + return set() + + +def baseline_versions(engines): + """Return {engine_key: set(versions)} from metadata.engines (dict or list).""" + result = {key: set() for key in ARTIFACTS} + if isinstance(engines, dict): + items = engines.items() + elif isinstance(engines, list): + items = [] + for e in engines: + if isinstance(e, dict): + items.append((e.get("name") or e.get("id") or e.get("engine") or "", e)) + elif isinstance(e, str): + items.append((e, e)) + else: + items = [] + for name, value in items: + key = _engine_key(name) + if key: + result[key] |= _extract_versions(value) + return result + + +def load_baseline(path): + with open(path, encoding="utf-8") as fh: + data = json.load(fh) + return baseline_versions(data.get("metadata", {}).get("engines", {})) + + +def compare(baseline, fetcher=fetch_versions, verbose=False): + report = {} + for key, (group, artifact) in ARTIFACTS.items(): + versions = fetcher(group, artifact) + latest = latest_stable(versions) + known = baseline.get(key, set()) + current = max(known, key=version_key) if known else None + log(verbose, f"{key}: {len(versions)} versions fetched, latest stable={latest}") + missing = latest is not None and ( + current is None or version_key(latest) > version_key(current) + ) + report[key] = { + "artifact": f"{group}:{artifact}", + "latest": latest, + "baseline": current, + "missing": missing, + } + return report + + +def parse_args(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--baseline", default=DEFAULT_BASELINE, help="path to baseline.json") + parser.add_argument("--check", action="store_true", help="exit 1 if a new release is missing") + parser.add_argument("--json", action="store_true", help="JSON output") + parser.add_argument("--verbose", action="store_true", help="log details to stderr") + return parser.parse_args(argv) + + +def main(argv=None): + args = parse_args(argv) + try: + log(args.verbose, f"Loading baseline {args.baseline}") + baseline = load_baseline(args.baseline) + report = compare(baseline, verbose=args.verbose) + except (OSError, ValueError) as exc: # includes URLError, JSONDecodeError + print(f"error: {exc}", file=sys.stderr) + return 2 + + missing = [k for k, v in report.items() if v["missing"]] + if args.json: + print(json.dumps({"engines": report, "missing": missing}, indent=2)) + else: + for key, info in report.items(): + status = "NEW" if info["missing"] else "up-to-date" + print( + f"{key} ({info['artifact']}): baseline={info['baseline']} " + f"latest={info['latest']} [{status}]" + ) + return 1 if args.check and missing else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/python-utils/test_check_new_releases.py b/python-utils/test_check_new_releases.py new file mode 100644 index 0000000..069ed79 --- /dev/null +++ b/python-utils/test_check_new_releases.py @@ -0,0 +1,149 @@ +import io +import json +import os +import tempfile +import unittest +from contextlib import redirect_stderr, redirect_stdout +from unittest import mock + +import check_new_releases as cnr + + +def fake_response(versions): + body = json.dumps({"response": {"docs": [{"v": v} for v in versions]}}).encode() + resp = mock.MagicMock() + resp.read.return_value = body + resp.__enter__.return_value = resp + return resp + + +def fake_urlopen(jena, rdf4j): + def _open(request, timeout=None): + url = request.full_url if hasattr(request, "full_url") else request + return fake_response(jena if "jena-arq" in url else rdf4j) + + return _open + + +class StabilityTests(unittest.TestCase): + def test_rejects_unstable(self): + for v in [ + "5.0.0-SNAPSHOT", + "5.0.0-rc1", + "5.0.0-RC2", + "5.0.0-alpha1", + "5.0.0-beta2", + "5.0.0-M1", + "5.0.0.M2", + "4.0.0-milestone1", + ]: + self.assertFalse(cnr.is_stable(v), v) + + def test_accepts_stable(self): + for v in ["5.0.0", "4.3.1", "5.1.0.Final", "3.7.7"]: + self.assertTrue(cnr.is_stable(v), v) + + def test_latest_stable_numeric_order(self): + self.assertEqual(cnr.latest_stable(["5.9.0", "5.10.0", "6.0.0-rc1"]), "5.10.0") + self.assertIsNone(cnr.latest_stable(["1.0-SNAPSHOT"])) + + +class FetchTests(unittest.TestCase): + def test_fetch_versions(self): + with mock.patch("urllib.request.urlopen", return_value=fake_response(["1.0", "2.0"])) as m: + self.assertEqual(cnr.fetch_versions("g", "a"), ["1.0", "2.0"]) + self.assertIn("g%3A%22g%22", m.call_args[0][0].full_url) + + +class BaselineTests(unittest.TestCase): + def test_dict_formats(self): + got = cnr.baseline_versions({"Jena": "5.0.0", "rdf4j": {"version": "4.3.1"}}) + self.assertEqual(got["jena"], {"5.0.0"}) + self.assertEqual(got["rdf4j"], {"4.3.1"}) + + def test_list_of_strings_format(self): + got = cnr.baseline_versions(["corese.4.6.5", "jena.6.2.0", "rdf4j.5.3.2"]) + self.assertEqual(got["jena"], {"6.2.0"}) + self.assertEqual(got["rdf4j"], {"5.3.2"}) + + def test_list_format(self): + got = cnr.baseline_versions( + [{"name": "Apache Jena", "version": "5.0.0"}, {"name": "RDF4J", "version": "4.3.1"}] + ) + self.assertEqual(got["jena"], {"5.0.0"}) + self.assertEqual(got["rdf4j"], {"4.3.1"}) + + def test_empty(self): + self.assertEqual(cnr.baseline_versions(None), {"jena": set(), "rdf4j": set()}) + + +class MainTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + + def baseline(self, engines): + path = os.path.join(self.tmp.name, "baseline.json") + with open(path, "w") as fh: + json.dump({"metadata": {"engines": engines}}, fh) + return path + + def run_main(self, args, jena, rdf4j): + out, err = io.StringIO(), io.StringIO() + with mock.patch("urllib.request.urlopen", side_effect=fake_urlopen(jena, rdf4j)), \ + redirect_stdout(out), redirect_stderr(err): + code = cnr.main(args) + return code, out.getvalue(), err.getvalue() + + def test_up_to_date(self): + path = self.baseline({"jena": "5.0.0", "rdf4j": "4.3.1"}) + code, out, _ = self.run_main( + ["--baseline", path, "--check"], ["5.0.0", "5.1.0-SNAPSHOT"], ["4.3.1", "5.0.0-M1"] + ) + self.assertEqual(code, 0) + self.assertIn("up-to-date", out) + + def test_missing_with_check(self): + path = self.baseline({"jena": "5.0.0", "rdf4j": "4.3.1"}) + code, out, _ = self.run_main(["--baseline", path, "--check"], ["5.0.0", "5.1.0"], ["4.3.1"]) + self.assertEqual(code, 1) + self.assertIn("NEW", out) + + def test_missing_without_check_exits_zero(self): + path = self.baseline({"jena": "5.0.0", "rdf4j": "4.3.1"}) + code, _, _ = self.run_main(["--baseline", path], ["5.1.0"], ["4.3.1"]) + self.assertEqual(code, 0) + + def test_json_output(self): + path = self.baseline({"jena": "5.0.0", "rdf4j": "4.3.1"}) + code, out, _ = self.run_main(["--baseline", path, "--json"], ["5.1.0"], ["4.3.1"]) + data = json.loads(out) + self.assertEqual(code, 0) + self.assertEqual(data["missing"], ["jena"]) + self.assertEqual(data["engines"]["jena"]["latest"], "5.1.0") + self.assertEqual(data["engines"]["jena"]["artifact"], "org.apache.jena:jena-arq") + + def test_verbose_logs_to_stderr(self): + path = self.baseline({"jena": "5.0.0", "rdf4j": "4.3.1"}) + _, _, err = self.run_main(["--baseline", path, "--verbose"], ["5.0.0"], ["4.3.1"]) + self.assertIn("jena: 1 versions fetched", err) + + def test_engine_absent_from_baseline(self): + path = self.baseline({"jena": "5.0.0"}) + code, _, _ = self.run_main(["--baseline", path, "--check"], ["5.0.0"], ["4.3.1"]) + self.assertEqual(code, 1) + + def test_network_error(self): + path = self.baseline({"jena": "5.0.0"}) + import urllib.error + with mock.patch("urllib.request.urlopen", side_effect=urllib.error.URLError("down")), \ + redirect_stderr(io.StringIO()): + self.assertEqual(cnr.main(["--baseline", path]), 2) + + def test_missing_baseline_file(self): + with redirect_stderr(io.StringIO()): + self.assertEqual(cnr.main(["--baseline", "/nonexistent.json"]), 2) + + +if __name__ == "__main__": + unittest.main() From b6838b7c9128fe1c02abc3711514eea51f3d2a7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20C=C3=A9r=C3=A8s?= Date: Fri, 30 Oct 2026 16:02:49 +0100 Subject: [PATCH 2/3] fix(ci): retry with backoff, sort by timestamp, normalize version keys, and wire cron check (#32) - Add HTTP retry with backoff on transient Maven Central network failures - Query Maven Central with sort=timestamp desc to ensure latest versions appear in the first page - Normalize semantic version keys (e.g. 5.0 equals 5.0.0) - Anchor default baseline path relative to script directory - Wire automated release check into Sunday cron in ci.yml --- .github/workflows/ci.yml | 7 +++++ python-utils/check_new_releases.py | 38 ++++++++++++++++++++----- python-utils/test_check_new_releases.py | 20 +++++++++++-- 3 files changed, 55 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ad1c9de..08178ef 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -263,6 +263,13 @@ jobs: cp "$JAR" "$GITHUB_WORKSPACE/libs/corese-core-local.jar" echo "corese.next JAR built from $CORESE_COMMIT → libs/corese-core-local.jar" + - name: Check for new library releases on Maven Central + if: github.event_name == 'schedule' + run: | + source "$CONDA_SH" + conda activate benchmarkenv + python python-utils/check_new_releases.py --verbose || true + - name: Run benchmark env: ENGINES: ${{ steps.params.outputs.engines }} diff --git a/python-utils/check_new_releases.py b/python-utils/check_new_releases.py index 6e7df18..50ebd1f 100644 --- a/python-utils/check_new_releases.py +++ b/python-utils/check_new_releases.py @@ -1,14 +1,20 @@ #!/usr/bin/env python3 """Check Maven Central for new stable releases of Jena and RDF4J.""" import argparse +import http.client import json import re import sys +import time +import urllib.error import urllib.parse import urllib.request +from pathlib import Path SEARCH_URL = "https://search.maven.org/solrsearch/select" -DEFAULT_BASELINE = "site/data/baseline.json" +DEFAULT_BASELINE = str( + Path(__file__).resolve().parent.parent / "site" / "data" / "baseline.json" +) ARTIFACTS = { "jena": ("org.apache.jena", "jena-arq"), "rdf4j": ("org.eclipse.rdf4j", "rdf4j-storage"), @@ -31,24 +37,42 @@ def is_stable(version): def version_key(version): - return tuple(int(n) for n in re.findall(r"\d+", version)) + nums = [int(n) for n in re.findall(r"\d+", version)] + while len(nums) > 1 and nums[-1] == 0: + nums.pop() + return tuple(nums) -def fetch_versions(group, artifact, timeout=15): +def fetch_versions(group, artifact, timeout=15, max_retries=3): query = urllib.parse.urlencode( { "q": f'g:"{group}" AND a:"{artifact}"', "core": "gav", "rows": "200", + "sort": "timestamp desc", "wt": "json", } ) request = urllib.request.Request( f"{SEARCH_URL}?{query}", headers={"User-Agent": "check-new-releases/1.0"} ) - with urllib.request.urlopen(request, timeout=timeout) as response: - payload = json.loads(response.read().decode("utf-8")) - return [doc["v"] for doc in payload.get("response", {}).get("docs", []) if "v" in doc] + last_err = None + for attempt in range(max_retries): + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + payload = json.loads(response.read().decode("utf-8")) + return [ + doc["v"] + for doc in payload.get("response", {}).get("docs", []) + if "v" in doc + ] + except (urllib.error.URLError, http.client.HTTPException) as e: + last_err = e + if attempt < max_retries - 1: + time.sleep(0.5 * (attempt + 1)) + if last_err: + raise last_err + return [] def latest_stable(versions): @@ -139,7 +163,7 @@ def main(argv=None): log(args.verbose, f"Loading baseline {args.baseline}") baseline = load_baseline(args.baseline) report = compare(baseline, verbose=args.verbose) - except (OSError, ValueError) as exc: # includes URLError, JSONDecodeError + except (OSError, ValueError, http.client.HTTPException) as exc: print(f"error: {exc}", file=sys.stderr) return 2 diff --git a/python-utils/test_check_new_releases.py b/python-utils/test_check_new_releases.py index 069ed79..a5cd340 100644 --- a/python-utils/test_check_new_releases.py +++ b/python-utils/test_check_new_releases.py @@ -1,8 +1,10 @@ +import http.client import io import json import os import tempfile import unittest +import urllib.error from contextlib import redirect_stderr, redirect_stdout from unittest import mock @@ -47,12 +49,24 @@ def test_latest_stable_numeric_order(self): self.assertEqual(cnr.latest_stable(["5.9.0", "5.10.0", "6.0.0-rc1"]), "5.10.0") self.assertIsNone(cnr.latest_stable(["1.0-SNAPSHOT"])) + def test_version_key_normalization(self): + self.assertEqual(cnr.version_key("5.0"), cnr.version_key("5.0.0")) + self.assertTrue(cnr.version_key("5.0.1") > cnr.version_key("5.0")) + class FetchTests(unittest.TestCase): def test_fetch_versions(self): with mock.patch("urllib.request.urlopen", return_value=fake_response(["1.0", "2.0"])) as m: self.assertEqual(cnr.fetch_versions("g", "a"), ["1.0", "2.0"]) - self.assertIn("g%3A%22g%22", m.call_args[0][0].full_url) + self.assertIn("sort=timestamp+desc", m.call_args[0][0].full_url) + + def test_fetch_versions_retry_on_failure(self): + err = urllib.error.URLError("temporary failure") + good = fake_response(["2.0.0"]) + with mock.patch("urllib.request.urlopen", side_effect=[err, good]), \ + mock.patch("time.sleep"): + versions = cnr.fetch_versions("g", "a", max_retries=2) + self.assertEqual(versions, ["2.0.0"]) class BaselineTests(unittest.TestCase): @@ -84,7 +98,7 @@ def setUp(self): def baseline(self, engines): path = os.path.join(self.tmp.name, "baseline.json") - with open(path, "w") as fh: + with open(path, "w", encoding="utf-8") as fh: json.dump({"metadata": {"engines": engines}}, fh) return path @@ -135,8 +149,8 @@ def test_engine_absent_from_baseline(self): def test_network_error(self): path = self.baseline({"jena": "5.0.0"}) - import urllib.error with mock.patch("urllib.request.urlopen", side_effect=urllib.error.URLError("down")), \ + mock.patch("time.sleep"), \ redirect_stderr(io.StringIO()): self.assertEqual(cnr.main(["--baseline", path]), 2) From 0ec47f43356023f172caaa40a6f807bd7af342cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20C=C3=A9r=C3=A8s?= Date: Fri, 30 Oct 2026 16:04:48 +0100 Subject: [PATCH 3/3] fix(test): use assertGreater instead of assertTrue comparison (SonarLint python:S5906) --- python-utils/test_check_new_releases.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-utils/test_check_new_releases.py b/python-utils/test_check_new_releases.py index a5cd340..4b99020 100644 --- a/python-utils/test_check_new_releases.py +++ b/python-utils/test_check_new_releases.py @@ -51,7 +51,7 @@ def test_latest_stable_numeric_order(self): def test_version_key_normalization(self): self.assertEqual(cnr.version_key("5.0"), cnr.version_key("5.0.0")) - self.assertTrue(cnr.version_key("5.0.1") > cnr.version_key("5.0")) + self.assertGreater(cnr.version_key("5.0.1"), cnr.version_key("5.0")) class FetchTests(unittest.TestCase):