From 7ceea679fc9fe87d86081923ffbb02551a80dece Mon Sep 17 00:00:00 2001 From: "warre.veys" Date: Tue, 2 Jun 2026 16:31:29 +0200 Subject: [PATCH] =?UTF-8?q?bump:=20v0.5.1=20=E2=86=92=20v0.6.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 37 ++++++++----------------------------- pyproject.toml | 2 +- 2 files changed, 9 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 12abffc..e2eb7c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,35 +1,14 @@ -## Unreleased +## v0.6.0 (2026-06-02) ### Feat -- ``HouseGradedSkillExtractRanking``: graded-relevance skill-extraction ranking - task on the HOUSE subset of CAREER, re-annotated against the full ESCO v1.1.0 - taxonomy with a 0-4 score scale (HF: ``TechWolf/Skill-extraction-House-graded``, - BEIR layout, validation split only). -- ``TechGradedSkillExtractRanking``: graded-relevance skill-extraction ranking - task on the TECH subset of CAREER, re-annotated against the full ESCO v1.1.0 - taxonomy with a 0-4 score scale (HF: ``TechWolf/Skill-extraction-Tech-graded``, - BEIR layout, validation split only). -- ``SkillSkapeGradedSkillExtractRanking``: graded-relevance skill-extraction - ranking task on SkillSkape, re-annotated against the full ESCO v1.1.0 taxonomy - with a 0-4 score scale (HF: ``TechWolf/Skill-extraction-SkillSkape-graded``, - BEIR layout, validation split only). -- ``ESCOGradedSkillNormRanking``: graded-relevance skill-normalization ranking - task that maps surface skill terms (ESCO alt-labels) to canonical ESCO skills, - annotated against the full ESCO v1.1.0 taxonomy with a 0-4 score scale - (HF: ``TechWolf/Skill-normalisation-ESCO-graded``, BEIR layout, validation - split only). -- graded relevance support for ranking metrics: ``RankingDataset`` accepts an - optional ``target_relevance`` field aligned 1-to-1 with ``target_indices``. - ``ndcg@k`` uses a (2^rel - 1) gain when graded labels are provided; binary - metrics (``map``, ``mrr``, ``recall@k``, ``hit@k``, ``rp@k``) ignore the - field. Binary nDCG behavior is preserved when ``target_relevance`` is - ``None``. See ``examples/custom_task_graded_relevance_example.py``. -- ``RankingTask.binary_relevance_threshold`` (default ``1e-9``) lets a graded - task choose which grades count as positives for binary metrics. Items with - relevance below the threshold are dropped from the binary positive set but - still contribute to graded metrics like ``ndcg@k``. Has no effect when - ``target_relevance`` is ``None``. +- enabled graded (non-binary) relevancy for ranking tasks +- added graded skill extraction and normalisation tasks +- store rankings for quick recomputation of new metrics + +### Fix + +- avoid KeyError when evaluating with a metric set that omits the default key metric ## v0.5.1 (2026-03-13) diff --git a/pyproject.toml b/pyproject.toml index 38f0428..cac26b2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ packages = ["src/workrb"] [project] name = "workrb" -version = "0.5.1" +version = "0.6.0" description = "WorkRB: Work Research Benchmark. Easy benchmarking of AI progress in the work domain." readme = "README.md" requires-python = ">=3.10,<4.0"