From c4d9885a1ed10b800b2ecdb1f2a7e0a6d6a1bbc4 Mon Sep 17 00:00:00 2001 From: "warre.veys" Date: Tue, 26 May 2026 17:00:38 +0200 Subject: [PATCH 1/6] linting --- src/workrb/rankings.py | 4 +--- src/workrb/tasks/abstract/ranking_base.py | 1 - tests/test_save_rankings_artifacts.py | 4 +--- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/src/workrb/rankings.py b/src/workrb/rankings.py index a61c165..ac3f501 100644 --- a/src/workrb/rankings.py +++ b/src/workrb/rankings.py @@ -140,9 +140,7 @@ def materialize_prediction_matrix( matrix = np.zeros((num_queries, num_targets), dtype=np.float32) for q_idx, row in scores.items(): if not 0 <= q_idx < num_queries: - raise IndexError( - f"query_index {q_idx} out of bounds for num_queries={num_queries}" - ) + raise IndexError(f"query_index {q_idx} out of bounds for num_queries={num_queries}") for t_idx, score in row.items(): if not 0 <= t_idx < num_targets: raise IndexError( diff --git a/src/workrb/tasks/abstract/ranking_base.py b/src/workrb/tasks/abstract/ranking_base.py index 00a5842..581cce7 100644 --- a/src/workrb/tasks/abstract/ranking_base.py +++ b/src/workrb/tasks/abstract/ranking_base.py @@ -476,4 +476,3 @@ def compute_metrics_from_prediction_matrix( pos_label_relevance=dataset.target_relevance, binary_relevance_threshold=self.binary_relevance_threshold, ) - diff --git a/tests/test_save_rankings_artifacts.py b/tests/test_save_rankings_artifacts.py index b17bdc2..e6d6037 100644 --- a/tests/test_save_rankings_artifacts.py +++ b/tests/test_save_rankings_artifacts.py @@ -338,9 +338,7 @@ def test_save_rankings_rejects_non_finite_scores(): from workrb.config import BenchmarkConfig - dataset = TinyRankingTask( - split=DatasetSplit.TEST, languages=[Language.EN] - ).datasets["en"] + dataset = TinyRankingTask(split=DatasetSplit.TEST, languages=[Language.EN]).datasets["en"] config = BenchmarkConfig( model_name="tiny", output_folder=str(_fresh_dir("rankings_artifact_non_finite_write")), From c6f44db41ae2c10fa5afcec31cd1cc549fe88bf8 Mon Sep 17 00:00:00 2001 From: "warre.veys" Date: Tue, 26 May 2026 17:43:18 +0200 Subject: [PATCH 2/6] first relevancy task (house) integration --- CHANGELOG.md | 4 + README.md | 1 + src/workrb/tasks/__init__.py | 2 + src/workrb/tasks/ranking/__init__.py | 2 + src/workrb/tasks/ranking/skill_extraction.py | 150 +++++++++++++++++++ tests/test_import.py | 2 + tests/test_task_loading.py | 7 + 7 files changed, 168 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d3cf48..b7049c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ### Feat +- ``HouseGradedSkillExtractRanking``: graded-relevance skill-extraction ranking + task on the HOUSE subset of CAREER, re-annotated against the full ESCO v1.1.0 + taxonomy with a 0-4 score scale (HF: ``TechWolf/skill-extraction-house-with-relevancy``, + BEIR layout, validation split only). - graded relevance support for ranking metrics: ``RankingDataset`` accepts an optional ``target_relevance`` field aligned 1-to-1 with ``target_indices``. ``ndcg@k`` uses a (2^rel - 1) gain when graded labels are provided; binary diff --git a/README.md b/README.md index b42b42b..da5780f 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,7 @@ pip install workrb | Job Normalization MELO | `MELORanking` | multi_label | 633 queries x 33813 targets | 21 | | Skill to Job WorkBench | `ESCOSkill2JobRanking` | multi_label | 13492 queries x 3039 targets | 28 | | Skill Extraction House | `HouseSkillExtractRanking` | multi_label | 262 queries x 13891 targets | 28 | +| Skill Extraction House Graded | `HouseGradedSkillExtractRanking` | multi_label | 61 queries x 13891 targets | 1 | | Skill Extraction Tech | `TechSkillExtractRanking` | multi_label | 338 queries x 13891 targets | 28 | | Skill Extraction SkillSkape | `SkillSkapeExtractRanking` | multi_label | 1191 queries x 13891 targets | 28 | | Skill Extraction TechWolf | `TechWolfSkillExtractRanking` | multi_label | 326 queries x 13891 targets | 28 | diff --git a/src/workrb/tasks/__init__.py b/src/workrb/tasks/__init__.py index ae66be8..1d989a3 100644 --- a/src/workrb/tasks/__init__.py +++ b/src/workrb/tasks/__init__.py @@ -19,6 +19,7 @@ from .ranking.mels import MELSRanking from .ranking.skill2job import ESCOSkill2JobRanking from .ranking.skill_extraction import ( + HouseGradedSkillExtractRanking, HouseSkillExtractRanking, SkillSkapeExtractRanking, SkillXLSkillExtractRanking, @@ -46,6 +47,7 @@ "JobTitleSimilarityRanking", "MELORanking", "MELSRanking", + "HouseGradedSkillExtractRanking", "HouseSkillExtractRanking", "TechSkillExtractRanking", "TechWolfSkillExtractRanking", diff --git a/src/workrb/tasks/ranking/__init__.py b/src/workrb/tasks/ranking/__init__.py index 63f87df..2fab9d9 100644 --- a/src/workrb/tasks/ranking/__init__.py +++ b/src/workrb/tasks/ranking/__init__.py @@ -18,6 +18,7 @@ from workrb.tasks.ranking.mels import MELSRanking from workrb.tasks.ranking.skill2job import ESCOSkill2JobRanking from workrb.tasks.ranking.skill_extraction import ( + HouseGradedSkillExtractRanking, HouseSkillExtractRanking, SkillSkapeExtractRanking, SkillXLSkillExtractRanking, @@ -31,6 +32,7 @@ "ESCOJob2SkillRanking", "ESCOSkill2JobRanking", "ESCOSkillNormRanking", + "HouseGradedSkillExtractRanking", "HouseSkillExtractRanking", "JobBERTJobNormRanking", "JobTitleSimilarityRanking", diff --git a/src/workrb/tasks/ranking/skill_extraction.py b/src/workrb/tasks/ranking/skill_extraction.py index e790a17..7bcd660 100644 --- a/src/workrb/tasks/ranking/skill_extraction.py +++ b/src/workrb/tasks/ranking/skill_extraction.py @@ -2,6 +2,7 @@ import pandas as pd from datasets import Dataset, load_dataset +from huggingface_hub import hf_hub_download from workrb.data.esco import ESCO from workrb.registry import register_task @@ -351,6 +352,155 @@ def citation(self) -> str: """ +@register_task() +class HouseGradedSkillExtractRanking(BaseESCOSkillExtractRanking): + """Skill Extraction from HOUSE Dataset with Graded Relevance. + + Re-annotates the sentences from ``TechWolf/skill-extraction-house`` against + the full ESCO v1.1.0 skill taxonomy with a 0-4 relevance scale, following the + BEIR layout (``queries``, ``corpus``, ``qrels``). + + Score scale (per the dataset card): + 0 - totally unrelated + 1 - domain is correct, plausible in broader context but not in the sentence + 2 - could be recommended, granularity makes it not core to the query + 3 - strongly relevant, more implied than explicitly demonstrated + 4 - explicitly demonstrated or requested + + Only the validation split is annotated at this stage. The test split will + be released alongside the workrb.ai challenge results. + """ + + orig_esco_version = "1.1.0" + + def __init__(self, esco_version: str = "1.1.0", **kwargs): + self.esco_version = esco_version + super().__init__(hf_name="TechWolf/skill-extraction-house-with-relevancy", **kwargs) + + @property + def default_metrics(self) -> list[str]: + """Default metrics include nDCG to leverage the graded labels. + + ``ndcg`` without a cutoff scores the full ranking (k = |target_space|). + """ + return ["ndcg", "ndcg@5", "ndcg@10", "map", "rp@10", "mrr"] + + def load_dataset(self, dataset_id: str, split: DatasetSplit) -> RankingDataset: + """Load BEIR-style graded annotations and convert to a RankingDataset. + + The corpus is the full ESCO v1.1.0 skill taxonomy (URIs in ``corpus-id``). + Only the validation split has been released so far. + """ + if split != DatasetSplit.VAL: + raise ValueError( + f"Split '{split}' not supported for {type(self).__name__}: only the " + f"validation split is annotated at this stage." + ) + + language = Language(dataset_id) + + # BEIR layout: fetch the queries and qrels parquet files directly. The + # repo's README config points at "data/queries/..." paths that don't + # exist; hf_hub_download bypasses that and reads the actual files. + queries_path = hf_hub_download( + self.hf_name, filename="queries/validation.parquet", repo_type="dataset" + ) + qrels_path = hf_hub_download( + self.hf_name, filename="qrels/validation.parquet", repo_type="dataset" + ) + queries_df = pd.read_parquet(queries_path) + qrels_df = pd.read_parquet(qrels_path) + + # Drop grade-0 rows: they are explicit "irrelevant" annotations and map + # to implicit grade 0 (items absent from target_indices). + qrels_df = qrels_df[qrels_df["score"] > 0].reset_index(drop=True).copy() + + # Map corpus URIs to ESCO preferred labels, mirroring the binary tasks + # so the target_space is the deterministic ESCO skill vocabulary. + original_esco = ESCO(version=self.orig_esco_version, language=Language.EN) + original_skill_uris = original_esco.get_skills_uris() + original_uris_to_skill = {v: k for k, v in original_skill_uris.items()} + + if self.esco_version != self.orig_esco_version or language != Language.EN: + target_esco = ESCO(version=self.esco_version, language=language) + target_skill_uris = target_esco.get_skills_uris() + target_uris_to_skill = {v: k for k, v in target_skill_uris.items()} + uri_to_label = { + uri: target_uris_to_skill[uri] + for uri in original_uris_to_skill + if uri in target_uris_to_skill + } + else: + uri_to_label = original_uris_to_skill + + qrels_df["label"] = qrels_df["corpus-id"].map(uri_to_label) + qrels_df = qrels_df[qrels_df["label"].notna()].reset_index(drop=True).copy() + + # Load ESCO skill vocabulary for target version/language. + esco = ESCO(version=self.esco_version, language=language) + skill_vocab = esco.get_skills_vocabulary() + skill2label = {skill: i for i, skill in enumerate(skill_vocab)} + + # Attach the sentence text and the target index for every (query, skill, score). + id_to_query = dict(zip(queries_df["_id"], queries_df["text"], strict=True)) + qrels_df["sentence"] = qrels_df["query-id"].map(id_to_query) + qrels_df["target_idx"] = qrels_df["label"].map(skill2label) + # Some labels may not be present in the target language vocabulary. + qrels_df = qrels_df[qrels_df["target_idx"].notna()].reset_index(drop=True).copy() + qrels_df["target_idx"] = qrels_df["target_idx"].astype(int) + + grouped = qrels_df.groupby("sentence") + filtered_queries: list[str] = [] + filtered_indices: list[list[int]] = [] + filtered_relevance: list[list[float]] = [] + for sentence, group in grouped: + filtered_queries.append(str(sentence)) + filtered_indices.append(group["target_idx"].tolist()) + filtered_relevance.append([float(s) for s in group["score"].tolist()]) + + return RankingDataset( + query_texts=filtered_queries, + target_indices=filtered_indices, + target_space=skill_vocab, + dataset_id=dataset_id, + target_relevance=filtered_relevance, + ) + + @property + def name(self) -> str: + """Skill extraction HOUSE graded task name.""" + return "Skill Extraction House Graded" + + @property + def description(self) -> str: + """Skill extraction HOUSE graded task description.""" + return ( + "Extract skills from general text descriptions in the HOUSE subset of CAREER, " + "with graded 0-4 relevance against the full ESCO v1.1.0 taxonomy." + ) + + @property + def citation(self) -> str: + """Skill extraction HOUSE graded task citation.""" + return """@inproceedings{decorte2022design, + articleno = {{4}}, + author = {{Decorte, Jens-Joris and Van Hautte, Jeroen and Deleu, Johannes and Develder, Chris and Demeester, Thomas}}, + booktitle = {{Proceedings of the 2nd Workshop on Recommender Systems for Human Resources (RecSys-in-HR 2022)}}, + editor = {{Kaya, Mesut and Bogers, Toine and Graus, David and Mesbah, Sepideh and Johnson, Chris and Gutiérrez, Francisco}}, + isbn = {{9781450398565}}, + issn = {{1613-0073}}, + language = {{eng}}, + location = {{Seatle, USA}}, + pages = {{7}}, + publisher = {{CEUR}}, + title = {{Design of negative sampling strategies for distantly supervised skill extraction}}, + url = {{https://ceur-ws.org/Vol-3218/RecSysHR2022-paper_4.pdf}}, + volume = {{3218}}, + year = {{2022}}, +} +""" + + @register_task() class SkillSkapeExtractRanking(BaseESCOSkillExtractRanking): """Skill Extraction from SkillSkape Ranking Task.""" diff --git a/tests/test_import.py b/tests/test_import.py index e2dccf3..446b5ff 100644 --- a/tests/test_import.py +++ b/tests/test_import.py @@ -10,6 +10,7 @@ ESCOJob2SkillRanking, ESCOSkill2JobRanking, ESCOSkillNormRanking, + HouseGradedSkillExtractRanking, HouseSkillExtractRanking, JobBERTJobNormRanking, RankingDataset, @@ -49,6 +50,7 @@ def test_task_ranking_imports(): assert isinstance(ESCOSkill2JobRanking.__name__, str) assert isinstance(ESCOSkillNormRanking.__name__, str) assert isinstance(HouseSkillExtractRanking.__name__, str) + assert isinstance(HouseGradedSkillExtractRanking.__name__, str) assert isinstance(JobBERTJobNormRanking.__name__, str) assert isinstance(SkillMatch1kSkillSimilarityRanking.__name__, str) assert isinstance(TechSkillExtractRanking.__name__, str) diff --git a/tests/test_task_loading.py b/tests/test_task_loading.py index 8e1489c..ad0438f 100644 --- a/tests/test_task_loading.py +++ b/tests/test_task_loading.py @@ -10,6 +10,7 @@ ESCOJob2SkillRanking, ESCOSkill2JobRanking, ESCOSkillNormRanking, + HouseGradedSkillExtractRanking, HouseSkillExtractRanking, JobBERTJobNormRanking, JobTitleSimilarityRanking, @@ -50,6 +51,7 @@ def test_ranking_tasks_init_en_splits(): ("MELORanking", MELORanking), ("MELSRanking", MELSRanking), ("SkillExtractHouseRanking", HouseSkillExtractRanking), + ("SkillExtractHouseGradedRanking", HouseGradedSkillExtractRanking), ("SkillExtractTechRanking", TechSkillExtractRanking), ("SkillExtractSkillSkapeRanking", SkillSkapeExtractRanking), ("SkillSimilarityRanking", SkillMatch1kSkillSimilarityRanking), @@ -60,6 +62,9 @@ def test_ranking_tasks_init_en_splits(): "MELORanking", "MELSRanking", ] + tasks_with_only_val_set = [ + "SkillExtractHouseGradedRanking", + ] results = {"success": [], "failures": []} languages = [Language.EN] @@ -70,6 +75,8 @@ def test_ranking_tasks_init_en_splits(): for task_name, task_class in ranking_tasks: if split != DatasetSplit.TEST and task_name in tasks_with_only_test_set: continue + if split != DatasetSplit.VAL and task_name in tasks_with_only_val_set: + continue nb_total += 1 try: # Try to instantiate with minimal parameters From 1315607ffc03348a6145631cefa46353c3902eb9 Mon Sep 17 00:00:00 2001 From: "warre.veys" Date: Wed, 27 May 2026 11:04:09 +0200 Subject: [PATCH 3/6] Tech graded relevancy --- CHANGELOG.md | 4 + README.md | 1 + src/workrb/tasks/__init__.py | 2 + src/workrb/tasks/ranking/__init__.py | 2 + src/workrb/tasks/ranking/skill_extraction.py | 170 +++++++++++-------- tests/test_import.py | 2 + tests/test_task_loading.py | 3 + 7 files changed, 109 insertions(+), 75 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b7049c1..cc41ac8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ task on the HOUSE subset of CAREER, re-annotated against the full ESCO v1.1.0 taxonomy with a 0-4 score scale (HF: ``TechWolf/skill-extraction-house-with-relevancy``, BEIR layout, validation split only). +- ``TechGradedSkillExtractRanking``: graded-relevance skill-extraction ranking + task on the TECH subset of CAREER, re-annotated against the full ESCO v1.1.0 + taxonomy with a 0-4 score scale (HF: ``TechWolf/skill-extraction-tech-with-relevancy``, + BEIR layout, validation split only). - graded relevance support for ranking metrics: ``RankingDataset`` accepts an optional ``target_relevance`` field aligned 1-to-1 with ``target_indices``. ``ndcg@k`` uses a (2^rel - 1) gain when graded labels are provided; binary diff --git a/README.md b/README.md index da5780f..39661ae 100644 --- a/README.md +++ b/README.md @@ -87,6 +87,7 @@ pip install workrb | Skill Extraction House | `HouseSkillExtractRanking` | multi_label | 262 queries x 13891 targets | 28 | | Skill Extraction House Graded | `HouseGradedSkillExtractRanking` | multi_label | 61 queries x 13891 targets | 1 | | Skill Extraction Tech | `TechSkillExtractRanking` | multi_label | 338 queries x 13891 targets | 28 | +| Skill Extraction Tech Graded | `TechGradedSkillExtractRanking` | multi_label | 75 queries x 13891 targets | 1 | | Skill Extraction SkillSkape | `SkillSkapeExtractRanking` | multi_label | 1191 queries x 13891 targets | 28 | | Skill Extraction TechWolf | `TechWolfSkillExtractRanking` | multi_label | 326 queries x 13891 targets | 28 | | Skill Extraction SkillXL | `SkillXLSkillExtractRanking` | multi_label | 944 queries x 13891 targets | 28 | diff --git a/src/workrb/tasks/__init__.py b/src/workrb/tasks/__init__.py index 1d989a3..79c38c7 100644 --- a/src/workrb/tasks/__init__.py +++ b/src/workrb/tasks/__init__.py @@ -23,6 +23,7 @@ HouseSkillExtractRanking, SkillSkapeExtractRanking, SkillXLSkillExtractRanking, + TechGradedSkillExtractRanking, TechSkillExtractRanking, TechWolfSkillExtractRanking, ) @@ -49,6 +50,7 @@ "MELSRanking", "HouseGradedSkillExtractRanking", "HouseSkillExtractRanking", + "TechGradedSkillExtractRanking", "TechSkillExtractRanking", "TechWolfSkillExtractRanking", "SkillSkapeExtractRanking", diff --git a/src/workrb/tasks/ranking/__init__.py b/src/workrb/tasks/ranking/__init__.py index 2fab9d9..64b443b 100644 --- a/src/workrb/tasks/ranking/__init__.py +++ b/src/workrb/tasks/ranking/__init__.py @@ -22,6 +22,7 @@ HouseSkillExtractRanking, SkillSkapeExtractRanking, SkillXLSkillExtractRanking, + TechGradedSkillExtractRanking, TechSkillExtractRanking, TechWolfSkillExtractRanking, ) @@ -43,6 +44,7 @@ "SkillMatch1kSkillSimilarityRanking", "SkillSkapeExtractRanking", "SkillXLSkillExtractRanking", + "TechGradedSkillExtractRanking", "TechSkillExtractRanking", "TechWolfSkillExtractRanking", ] diff --git a/src/workrb/tasks/ranking/skill_extraction.py b/src/workrb/tasks/ranking/skill_extraction.py index 7bcd660..5fbace1 100644 --- a/src/workrb/tasks/ranking/skill_extraction.py +++ b/src/workrb/tasks/ranking/skill_extraction.py @@ -2,7 +2,6 @@ import pandas as pd from datasets import Dataset, load_dataset -from huggingface_hub import hf_hub_download from workrb.data.esco import ESCO from workrb.registry import register_task @@ -352,31 +351,19 @@ def citation(self) -> str: """ -@register_task() -class HouseGradedSkillExtractRanking(BaseESCOSkillExtractRanking): - """Skill Extraction from HOUSE Dataset with Graded Relevance. - - Re-annotates the sentences from ``TechWolf/skill-extraction-house`` against - the full ESCO v1.1.0 skill taxonomy with a 0-4 relevance scale, following the - BEIR layout (``queries``, ``corpus``, ``qrels``). +class BaseGradedSkillExtractRanking(BaseESCOSkillExtractRanking): + """Base class for BEIR-layout graded skill-extraction tasks. - Score scale (per the dataset card): - 0 - totally unrelated - 1 - domain is correct, plausible in broader context but not in the sentence - 2 - could be recommended, granularity makes it not core to the query - 3 - strongly relevant, more implied than explicitly demonstrated - 4 - explicitly demonstrated or requested + Reads the ``queries``, ``corpus`` and ``qrels`` configs published on the + Hugging Face Hub via ``load_dataset``. The target_space is the corpus's + ``title`` column (ESCO preferred labels, in corpus order); qrels are + expected to contain only non-zero judgments (absent items are implicit + grade 0). Only the validation split is released at this stage; the test + split will be released alongside the workrb.ai challenge results. - Only the validation split is annotated at this stage. The test split will - be released alongside the workrb.ai challenge results. + Subclasses set ``hf_name`` via ``__init__``. """ - orig_esco_version = "1.1.0" - - def __init__(self, esco_version: str = "1.1.0", **kwargs): - self.esco_version = esco_version - super().__init__(hf_name="TechWolf/skill-extraction-house-with-relevancy", **kwargs) - @property def default_metrics(self) -> list[str]: """Default metrics include nDCG to leverage the graded labels. @@ -386,68 +373,37 @@ def default_metrics(self) -> list[str]: return ["ndcg", "ndcg@5", "ndcg@10", "map", "rp@10", "mrr"] def load_dataset(self, dataset_id: str, split: DatasetSplit) -> RankingDataset: - """Load BEIR-style graded annotations and convert to a RankingDataset. - - The corpus is the full ESCO v1.1.0 skill taxonomy (URIs in ``corpus-id``). - Only the validation split has been released so far. - """ + """Load BEIR-style graded annotations and convert to a RankingDataset.""" if split != DatasetSplit.VAL: raise ValueError( f"Split '{split}' not supported for {type(self).__name__}: only the " f"validation split is annotated at this stage." ) - language = Language(dataset_id) - - # BEIR layout: fetch the queries and qrels parquet files directly. The - # repo's README config points at "data/queries/..." paths that don't - # exist; hf_hub_download bypasses that and reads the actual files. - queries_path = hf_hub_download( - self.hf_name, filename="queries/validation.parquet", repo_type="dataset" + queries_ds = load_dataset(self.hf_name, "queries", split="validation") + corpus_ds = load_dataset(self.hf_name, "corpus", split="corpus") + qrels_ds = load_dataset(self.hf_name, "qrels", split="validation") + assert isinstance(queries_ds, Dataset) + assert isinstance(corpus_ds, Dataset) + assert isinstance(qrels_ds, Dataset) + queries_df = queries_ds.to_pandas() + corpus_df = corpus_ds.to_pandas() + qrels_df = qrels_ds.to_pandas() + + # target_space is the corpus titles in corpus order; URIs map by row index. + target_space = corpus_df["title"].tolist() + uri_to_idx = {uri: i for i, uri in enumerate(corpus_df["_id"])} + + qrels_df["target_idx"] = qrels_df["corpus-id"].map(uri_to_idx) + # Every qrel should resolve; if any don't, surface the issue rather than silently dropping. + unresolved = qrels_df["target_idx"].isna().sum() + assert unresolved == 0, ( + f"{unresolved} qrel rows reference corpus-ids not present in the corpus config" ) - qrels_path = hf_hub_download( - self.hf_name, filename="qrels/validation.parquet", repo_type="dataset" - ) - queries_df = pd.read_parquet(queries_path) - qrels_df = pd.read_parquet(qrels_path) - - # Drop grade-0 rows: they are explicit "irrelevant" annotations and map - # to implicit grade 0 (items absent from target_indices). - qrels_df = qrels_df[qrels_df["score"] > 0].reset_index(drop=True).copy() - - # Map corpus URIs to ESCO preferred labels, mirroring the binary tasks - # so the target_space is the deterministic ESCO skill vocabulary. - original_esco = ESCO(version=self.orig_esco_version, language=Language.EN) - original_skill_uris = original_esco.get_skills_uris() - original_uris_to_skill = {v: k for k, v in original_skill_uris.items()} - - if self.esco_version != self.orig_esco_version or language != Language.EN: - target_esco = ESCO(version=self.esco_version, language=language) - target_skill_uris = target_esco.get_skills_uris() - target_uris_to_skill = {v: k for k, v in target_skill_uris.items()} - uri_to_label = { - uri: target_uris_to_skill[uri] - for uri in original_uris_to_skill - if uri in target_uris_to_skill - } - else: - uri_to_label = original_uris_to_skill - - qrels_df["label"] = qrels_df["corpus-id"].map(uri_to_label) - qrels_df = qrels_df[qrels_df["label"].notna()].reset_index(drop=True).copy() - - # Load ESCO skill vocabulary for target version/language. - esco = ESCO(version=self.esco_version, language=language) - skill_vocab = esco.get_skills_vocabulary() - skill2label = {skill: i for i, skill in enumerate(skill_vocab)} + qrels_df["target_idx"] = qrels_df["target_idx"].astype(int) - # Attach the sentence text and the target index for every (query, skill, score). id_to_query = dict(zip(queries_df["_id"], queries_df["text"], strict=True)) qrels_df["sentence"] = qrels_df["query-id"].map(id_to_query) - qrels_df["target_idx"] = qrels_df["label"].map(skill2label) - # Some labels may not be present in the target language vocabulary. - qrels_df = qrels_df[qrels_df["target_idx"].notna()].reset_index(drop=True).copy() - qrels_df["target_idx"] = qrels_df["target_idx"].astype(int) grouped = qrels_df.groupby("sentence") filtered_queries: list[str] = [] @@ -461,11 +417,26 @@ def load_dataset(self, dataset_id: str, split: DatasetSplit) -> RankingDataset: return RankingDataset( query_texts=filtered_queries, target_indices=filtered_indices, - target_space=skill_vocab, + target_space=target_space, dataset_id=dataset_id, target_relevance=filtered_relevance, ) + +@register_task() +class HouseGradedSkillExtractRanking(BaseGradedSkillExtractRanking): + """Skill Extraction from HOUSE Dataset with Graded Relevance. + + Re-annotates the sentences from ``TechWolf/skill-extraction-house`` against + the full ESCO v1.1.0 skill taxonomy with a 0-4 relevance scale, following the + BEIR layout (``queries``, ``corpus``, ``qrels``). Score scale: 0 unrelated, + 1 plausible in domain, 2 recommendable but off-granularity, 3 strongly implied, + 4 explicitly demonstrated. + """ + + def __init__(self, **kwargs): + super().__init__(hf_name="TechWolf/skill-extraction-house-with-relevancy", **kwargs) + @property def name(self) -> str: """Skill extraction HOUSE graded task name.""" @@ -501,6 +472,55 @@ def citation(self) -> str: """ +@register_task() +class TechGradedSkillExtractRanking(BaseGradedSkillExtractRanking): + """Skill Extraction from TECH Dataset with Graded Relevance. + + Re-annotates the sentences from ``TechWolf/skill-extraction-tech`` against + the full ESCO v1.1.0 skill taxonomy with a 0-4 relevance scale, following the + BEIR layout (``queries``, ``corpus``, ``qrels``). Score scale: 0 unrelated, + 1 plausible in domain, 2 recommendable but off-granularity, 3 strongly implied, + 4 explicitly demonstrated. + """ + + def __init__(self, **kwargs): + super().__init__(hf_name="TechWolf/skill-extraction-tech-with-relevancy", **kwargs) + + @property + def name(self) -> str: + """Skill extraction TECH graded task name.""" + return "Skill Extraction Tech Graded" + + @property + def description(self) -> str: + """Skill extraction TECH graded task description.""" + return ( + "Extract skills from technical text descriptions in the TECH subset of CAREER, " + "with graded 0-4 relevance against the full ESCO v1.1.0 taxonomy." + ) + + @property + def citation(self) -> str: + """Skill extraction TECH graded task citation.""" + return """@inproceedings{decorte2022design, + articleno = {{4}}, + author = {{Decorte, Jens-Joris and Van Hautte, Jeroen and Deleu, Johannes and Develder, Chris and Demeester, Thomas}}, + booktitle = {{Proceedings of the 2nd Workshop on Recommender Systems for Human Resources (RecSys-in-HR 2022)}}, + editor = {{Kaya, Mesut and Bogers, Toine and Graus, David and Mesbah, Sepideh and Johnson, Chris and Gutiérrez, Francisco}}, + isbn = {{9781450398565}}, + issn = {{1613-0073}}, + language = {{eng}}, + location = {{Seatle, USA}}, + pages = {{7}}, + publisher = {{CEUR}}, + title = {{Design of negative sampling strategies for distantly supervised skill extraction}}, + url = {{https://ceur-ws.org/Vol-3218/RecSysHR2022-paper_4.pdf}}, + volume = {{3218}}, + year = {{2022}}, +} +""" + + @register_task() class SkillSkapeExtractRanking(BaseESCOSkillExtractRanking): """Skill Extraction from SkillSkape Ranking Task.""" diff --git a/tests/test_import.py b/tests/test_import.py index 446b5ff..529f994 100644 --- a/tests/test_import.py +++ b/tests/test_import.py @@ -17,6 +17,7 @@ RankingTask, SkillMatch1kSkillSimilarityRanking, Task, + TechGradedSkillExtractRanking, TechSkillExtractRanking, ranking, ) @@ -54,6 +55,7 @@ def test_task_ranking_imports(): assert isinstance(JobBERTJobNormRanking.__name__, str) assert isinstance(SkillMatch1kSkillSimilarityRanking.__name__, str) assert isinstance(TechSkillExtractRanking.__name__, str) + assert isinstance(TechGradedSkillExtractRanking.__name__, str) print("✓ Successfully imported ranking task classes") diff --git a/tests/test_task_loading.py b/tests/test_task_loading.py index ad0438f..6c3a7c4 100644 --- a/tests/test_task_loading.py +++ b/tests/test_task_loading.py @@ -18,6 +18,7 @@ MELSRanking, SkillMatch1kSkillSimilarityRanking, SkillSkapeExtractRanking, + TechGradedSkillExtractRanking, TechSkillExtractRanking, ) from workrb.tasks.abstract.base import DatasetSplit, Language, TaskType @@ -53,6 +54,7 @@ def test_ranking_tasks_init_en_splits(): ("SkillExtractHouseRanking", HouseSkillExtractRanking), ("SkillExtractHouseGradedRanking", HouseGradedSkillExtractRanking), ("SkillExtractTechRanking", TechSkillExtractRanking), + ("SkillExtractTechGradedRanking", TechGradedSkillExtractRanking), ("SkillExtractSkillSkapeRanking", SkillSkapeExtractRanking), ("SkillSimilarityRanking", SkillMatch1kSkillSimilarityRanking), ] @@ -64,6 +66,7 @@ def test_ranking_tasks_init_en_splits(): ] tasks_with_only_val_set = [ "SkillExtractHouseGradedRanking", + "SkillExtractTechGradedRanking", ] results = {"success": [], "failures": []} From 979565e225c0146534a52a9076e22b6eaa9448f4 Mon Sep 17 00:00:00 2001 From: "warre.veys" Date: Tue, 2 Jun 2026 12:00:25 +0200 Subject: [PATCH 4/6] Last challenge files --- CHANGELOG.md | 13 +- README.md | 2 + src/workrb/tasks/__init__.py | 5 +- src/workrb/tasks/ranking/__init__.py | 5 +- src/workrb/tasks/ranking/graded_beir.py | 146 +++++++++++++++++++ src/workrb/tasks/ranking/skill_extraction.py | 125 ++++++++-------- src/workrb/tasks/ranking/skillnorm.py | 44 ++++++ tests/test_import.py | 4 + tests/test_task_loading.py | 6 + 9 files changed, 280 insertions(+), 70 deletions(-) create mode 100644 src/workrb/tasks/ranking/graded_beir.py diff --git a/CHANGELOG.md b/CHANGELOG.md index cc41ac8..12abffc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,12 +4,21 @@ - ``HouseGradedSkillExtractRanking``: graded-relevance skill-extraction ranking task on the HOUSE subset of CAREER, re-annotated against the full ESCO v1.1.0 - taxonomy with a 0-4 score scale (HF: ``TechWolf/skill-extraction-house-with-relevancy``, + taxonomy with a 0-4 score scale (HF: ``TechWolf/Skill-extraction-House-graded``, BEIR layout, validation split only). - ``TechGradedSkillExtractRanking``: graded-relevance skill-extraction ranking task on the TECH subset of CAREER, re-annotated against the full ESCO v1.1.0 - taxonomy with a 0-4 score scale (HF: ``TechWolf/skill-extraction-tech-with-relevancy``, + taxonomy with a 0-4 score scale (HF: ``TechWolf/Skill-extraction-Tech-graded``, BEIR layout, validation split only). +- ``SkillSkapeGradedSkillExtractRanking``: graded-relevance skill-extraction + ranking task on SkillSkape, re-annotated against the full ESCO v1.1.0 taxonomy + with a 0-4 score scale (HF: ``TechWolf/Skill-extraction-SkillSkape-graded``, + BEIR layout, validation split only). +- ``ESCOGradedSkillNormRanking``: graded-relevance skill-normalization ranking + task that maps surface skill terms (ESCO alt-labels) to canonical ESCO skills, + annotated against the full ESCO v1.1.0 taxonomy with a 0-4 score scale + (HF: ``TechWolf/Skill-normalisation-ESCO-graded``, BEIR layout, validation + split only). - graded relevance support for ranking metrics: ``RankingDataset`` accepts an optional ``target_relevance`` field aligned 1-to-1 with ``target_indices``. ``ndcg@k`` uses a (2^rel - 1) gain when graded labels are provided; binary diff --git a/README.md b/README.md index 39661ae..b32e8cf 100644 --- a/README.md +++ b/README.md @@ -89,10 +89,12 @@ pip install workrb | Skill Extraction Tech | `TechSkillExtractRanking` | multi_label | 338 queries x 13891 targets | 28 | | Skill Extraction Tech Graded | `TechGradedSkillExtractRanking` | multi_label | 75 queries x 13891 targets | 1 | | Skill Extraction SkillSkape | `SkillSkapeExtractRanking` | multi_label | 1191 queries x 13891 targets | 28 | +| Skill Extraction SkillSkape Graded | `SkillSkapeGradedSkillExtractRanking` | multi_label | 100 queries x 13891 targets | 1 | | Skill Extraction TechWolf | `TechWolfSkillExtractRanking` | multi_label | 326 queries x 13891 targets | 28 | | Skill Extraction SkillXL | `SkillXLSkillExtractRanking` | multi_label | 944 queries x 13891 targets | 28 | | Skill Similarity SkillMatch-1K | `SkillMatch1kSkillSimilarityRanking` | single_label | 900 queries x 2648 targets | 1 | | Skill Normalization ESCO | `ESCOSkillNormRanking` | multi_label | 72008 queries x 13939 targets | 28 | +| Skill Normalization ESCO Graded | `ESCOGradedSkillNormRanking` | multi_label | 50 queries x 13891 targets | 1 | | Skill Normalization MELS | `MELSRanking` | multi_label | 1722 queries x 19466 targets | 5 | | Query-Candidate Matching | `SearchQueryCandidateRanking` | multi_label | 200 queries x 4019 (x-lang) targets | 5 | | Project-Candidate Matching | `ProjectCandidateRanking` | multi_label | 200 queries x 4019 (x-lang) targets | 5 | diff --git a/src/workrb/tasks/__init__.py b/src/workrb/tasks/__init__.py index 79c38c7..ca4ce71 100644 --- a/src/workrb/tasks/__init__.py +++ b/src/workrb/tasks/__init__.py @@ -22,13 +22,14 @@ HouseGradedSkillExtractRanking, HouseSkillExtractRanking, SkillSkapeExtractRanking, + SkillSkapeGradedSkillExtractRanking, SkillXLSkillExtractRanking, TechGradedSkillExtractRanking, TechSkillExtractRanking, TechWolfSkillExtractRanking, ) from .ranking.skill_similarity import SkillMatch1kSkillSimilarityRanking -from .ranking.skillnorm import ESCOSkillNormRanking +from .ranking.skillnorm import ESCOGradedSkillNormRanking, ESCOSkillNormRanking __all__ = [ # Abstract classes @@ -41,6 +42,7 @@ # Classification tasks "ESCOJob2SkillClassification", # Ranking tasks + "ESCOGradedSkillNormRanking", "ESCOJob2SkillRanking", "ESCOSkill2JobRanking", "ESCOSkillNormRanking", @@ -54,6 +56,7 @@ "TechSkillExtractRanking", "TechWolfSkillExtractRanking", "SkillSkapeExtractRanking", + "SkillSkapeGradedSkillExtractRanking", "SkillXLSkillExtractRanking", "SkillMatch1kSkillSimilarityRanking", "ProjectCandidateRanking", diff --git a/src/workrb/tasks/ranking/__init__.py b/src/workrb/tasks/ranking/__init__.py index 64b443b..f233947 100644 --- a/src/workrb/tasks/ranking/__init__.py +++ b/src/workrb/tasks/ranking/__init__.py @@ -21,15 +21,17 @@ HouseGradedSkillExtractRanking, HouseSkillExtractRanking, SkillSkapeExtractRanking, + SkillSkapeGradedSkillExtractRanking, SkillXLSkillExtractRanking, TechGradedSkillExtractRanking, TechSkillExtractRanking, TechWolfSkillExtractRanking, ) from workrb.tasks.ranking.skill_similarity import SkillMatch1kSkillSimilarityRanking -from workrb.tasks.ranking.skillnorm import ESCOSkillNormRanking +from workrb.tasks.ranking.skillnorm import ESCOGradedSkillNormRanking, ESCOSkillNormRanking __all__ = [ + "ESCOGradedSkillNormRanking", "ESCOJob2SkillRanking", "ESCOSkill2JobRanking", "ESCOSkillNormRanking", @@ -43,6 +45,7 @@ "SearchQueryCandidateRanking", "SkillMatch1kSkillSimilarityRanking", "SkillSkapeExtractRanking", + "SkillSkapeGradedSkillExtractRanking", "SkillXLSkillExtractRanking", "TechGradedSkillExtractRanking", "TechSkillExtractRanking", diff --git a/src/workrb/tasks/ranking/graded_beir.py b/src/workrb/tasks/ranking/graded_beir.py new file mode 100644 index 0000000..14272f2 --- /dev/null +++ b/src/workrb/tasks/ranking/graded_beir.py @@ -0,0 +1,146 @@ +"""Shared base for BEIR-layout graded-relevance ranking tasks. + +Several tasks publish graded (0-4) relevance annotations on the Hugging Face +Hub following the BEIR convention (``queries``, ``corpus``, ``qrels`` configs). +They differ only in their task group and query input type (e.g. skill +extraction from sentences vs skill normalization from surface terms), so the +data-loading logic lives here and concrete tasks supply the task-specific +properties. +""" + +import pandas as pd +from datasets import Dataset, load_dataset + +from workrb.tasks.abstract.base import DatasetSplit, LabelType, Language +from workrb.tasks.abstract.ranking_base import RankingDataset, RankingTask +from workrb.types import ModelInputType + + +class GradedBEIRRankingTask(RankingTask): + """Base class for BEIR-layout graded ranking tasks. + + Reads the ``queries``, ``corpus`` and ``qrels`` configs published on the + Hugging Face Hub via ``load_dataset``. The target_space is the corpus's + ``title`` column (ESCO preferred labels, in corpus order); qrels are + expected to contain only non-zero judgments (absent items are implicit + grade 0). + + Which splits a task exposes depends on what the underlying dataset + publishes: some release both a validation and a test split, others only + one. Because several tasks share this loader, the supported splits are + declared per subclass via :attr:`split_to_hf_split` (which maps each + supported :class:`DatasetSplit` to the HF split name its + ``queries``/``qrels`` configs live under) rather than hardcoded with an + inline guard as in the single-split tasks (e.g. ``MELORanking``). The + default exposes only the validation split, the common case for in-progress + benchmark datasets whose test split is withheld. + + Concrete subclasses set ``hf_name`` via ``__init__`` and supply the + task-specific ``task_group``, ``query_input_type``, ``name``, + ``description`` and ``citation``. + """ + + def __init__(self, hf_name: str, **kwargs): + """Initialize the task. + + Args: + hf_name: Name of the Hugging Face dataset (BEIR layout). + **kwargs: Additional arguments for the base class. + """ + self.hf_name = hf_name + super().__init__(**kwargs) + + @property + def split_to_hf_split(self) -> dict[DatasetSplit, str]: + """Map each supported split to the HF split name backing it. + + The corpus config is always loaded from the ``corpus`` split; this + mapping only governs the ``queries`` and ``qrels`` configs. Override to + expose more or fewer splits, e.g. ``{DatasetSplit.VAL: "validation", + DatasetSplit.TEST: "test"}`` for a dataset that releases both. + """ + return {DatasetSplit.VAL: "validation"} + + @property + def supported_query_languages(self) -> list[Language]: + """Annotations are released in English only at this stage.""" + return [Language.EN] + + @property + def supported_target_languages(self) -> list[Language]: + """The corpus titles are released in English only at this stage.""" + return [Language.EN] + + @property + def label_type(self) -> LabelType: + """Label type is multi-label.""" + return LabelType.MULTI_LABEL + + @property + def target_input_type(self) -> ModelInputType: + """Target input type for ESCO skills.""" + return ModelInputType.SKILL_NAME + + @property + def default_metrics(self) -> list[str]: + """Default metrics include nDCG to leverage the graded labels. + + ``ndcg`` without a cutoff scores the full ranking (k = |target_space|). + """ + return ["ndcg", "ndcg@5", "ndcg@10", "map", "rp@10", "mrr"] + + def load_dataset(self, dataset_id: str, split: DatasetSplit) -> RankingDataset: + """Load BEIR-style graded annotations and convert to a RankingDataset.""" + hf_split = self.split_to_hf_split.get(split) + if hf_split is None: + supported = ", ".join(sorted(s.value for s in self.split_to_hf_split)) + raise ValueError( + f"Split '{split.value}' not supported for {type(self).__name__}: " + f"only [{supported}] {'is' if len(self.split_to_hf_split) == 1 else 'are'} " + f"annotated for this dataset." + ) + + queries_ds = load_dataset(self.hf_name, "queries", split=hf_split) + corpus_ds = load_dataset(self.hf_name, "corpus", split="corpus") + qrels_ds = load_dataset(self.hf_name, "qrels", split=hf_split) + assert isinstance(queries_ds, Dataset) + assert isinstance(corpus_ds, Dataset) + assert isinstance(qrels_ds, Dataset) + queries_df = queries_ds.to_pandas() + corpus_df = corpus_ds.to_pandas() + qrels_df = qrels_ds.to_pandas() + assert isinstance(queries_df, pd.DataFrame) + assert isinstance(corpus_df, pd.DataFrame) + assert isinstance(qrels_df, pd.DataFrame) + + # target_space is the corpus titles in corpus order; URIs map by row index. + target_space = corpus_df["title"].tolist() + uri_to_idx = {uri: i for i, uri in enumerate(corpus_df["_id"])} + + qrels_df["target_idx"] = qrels_df["corpus-id"].map(uri_to_idx) + # Every qrel should resolve; if any don't, surface the issue rather than silently dropping. + unresolved = qrels_df["target_idx"].isna().sum() + assert unresolved == 0, ( + f"{unresolved} qrel rows reference corpus-ids not present in the corpus config" + ) + qrels_df["target_idx"] = qrels_df["target_idx"].astype(int) + + id_to_query = dict(zip(queries_df["_id"], queries_df["text"], strict=True)) + qrels_df["sentence"] = qrels_df["query-id"].map(id_to_query) + + grouped = qrels_df.groupby("sentence") + filtered_queries: list[str] = [] + filtered_indices: list[list[int]] = [] + filtered_relevance: list[list[float]] = [] + for sentence, group in grouped: + filtered_queries.append(str(sentence)) + filtered_indices.append(group["target_idx"].tolist()) + filtered_relevance.append([float(s) for s in group["score"].tolist()]) + + return RankingDataset( + query_texts=filtered_queries, + target_indices=filtered_indices, + target_space=target_space, + dataset_id=dataset_id, + target_relevance=filtered_relevance, + ) diff --git a/src/workrb/tasks/ranking/skill_extraction.py b/src/workrb/tasks/ranking/skill_extraction.py index 5fbace1..65ef6a3 100644 --- a/src/workrb/tasks/ranking/skill_extraction.py +++ b/src/workrb/tasks/ranking/skill_extraction.py @@ -11,6 +11,7 @@ RankingTask, RankingTaskGroup, ) +from workrb.tasks.ranking.graded_beir import GradedBEIRRankingTask from workrb.types import ModelInputType @@ -351,76 +352,23 @@ def citation(self) -> str: """ -class BaseGradedSkillExtractRanking(BaseESCOSkillExtractRanking): +class BaseGradedSkillExtractRanking(GradedBEIRRankingTask): """Base class for BEIR-layout graded skill-extraction tasks. - Reads the ``queries``, ``corpus`` and ``qrels`` configs published on the - Hugging Face Hub via ``load_dataset``. The target_space is the corpus's - ``title`` column (ESCO preferred labels, in corpus order); qrels are - expected to contain only non-zero judgments (absent items are implicit - grade 0). Only the validation split is released at this stage; the test - split will be released alongside the workrb.ai challenge results. - - Subclasses set ``hf_name`` via ``__init__``. + Queries are skill-bearing sentences ranked against the ESCO skill taxonomy. + The BEIR loading logic lives in :class:`GradedBEIRRankingTask`; this base + just pins the skill-extraction task group and query input type. """ @property - def default_metrics(self) -> list[str]: - """Default metrics include nDCG to leverage the graded labels. - - ``ndcg`` without a cutoff scores the full ranking (k = |target_space|). - """ - return ["ndcg", "ndcg@5", "ndcg@10", "map", "rp@10", "mrr"] - - def load_dataset(self, dataset_id: str, split: DatasetSplit) -> RankingDataset: - """Load BEIR-style graded annotations and convert to a RankingDataset.""" - if split != DatasetSplit.VAL: - raise ValueError( - f"Split '{split}' not supported for {type(self).__name__}: only the " - f"validation split is annotated at this stage." - ) - - queries_ds = load_dataset(self.hf_name, "queries", split="validation") - corpus_ds = load_dataset(self.hf_name, "corpus", split="corpus") - qrels_ds = load_dataset(self.hf_name, "qrels", split="validation") - assert isinstance(queries_ds, Dataset) - assert isinstance(corpus_ds, Dataset) - assert isinstance(qrels_ds, Dataset) - queries_df = queries_ds.to_pandas() - corpus_df = corpus_ds.to_pandas() - qrels_df = qrels_ds.to_pandas() - - # target_space is the corpus titles in corpus order; URIs map by row index. - target_space = corpus_df["title"].tolist() - uri_to_idx = {uri: i for i, uri in enumerate(corpus_df["_id"])} - - qrels_df["target_idx"] = qrels_df["corpus-id"].map(uri_to_idx) - # Every qrel should resolve; if any don't, surface the issue rather than silently dropping. - unresolved = qrels_df["target_idx"].isna().sum() - assert unresolved == 0, ( - f"{unresolved} qrel rows reference corpus-ids not present in the corpus config" - ) - qrels_df["target_idx"] = qrels_df["target_idx"].astype(int) - - id_to_query = dict(zip(queries_df["_id"], queries_df["text"], strict=True)) - qrels_df["sentence"] = qrels_df["query-id"].map(id_to_query) - - grouped = qrels_df.groupby("sentence") - filtered_queries: list[str] = [] - filtered_indices: list[list[int]] = [] - filtered_relevance: list[list[float]] = [] - for sentence, group in grouped: - filtered_queries.append(str(sentence)) - filtered_indices.append(group["target_idx"].tolist()) - filtered_relevance.append([float(s) for s in group["score"].tolist()]) + def task_group(self) -> RankingTaskGroup: + """Skill extraction task group.""" + return RankingTaskGroup.SKILL_EXTRACTION - return RankingDataset( - query_texts=filtered_queries, - target_indices=filtered_indices, - target_space=target_space, - dataset_id=dataset_id, - target_relevance=filtered_relevance, - ) + @property + def query_input_type(self) -> ModelInputType: + """Query input type for skill extraction sentences.""" + return ModelInputType.SKILL_SENTENCE @register_task() @@ -435,7 +383,7 @@ class HouseGradedSkillExtractRanking(BaseGradedSkillExtractRanking): """ def __init__(self, **kwargs): - super().__init__(hf_name="TechWolf/skill-extraction-house-with-relevancy", **kwargs) + super().__init__(hf_name="TechWolf/Skill-extraction-House-graded", **kwargs) @property def name(self) -> str: @@ -484,7 +432,7 @@ class TechGradedSkillExtractRanking(BaseGradedSkillExtractRanking): """ def __init__(self, **kwargs): - super().__init__(hf_name="TechWolf/skill-extraction-tech-with-relevancy", **kwargs) + super().__init__(hf_name="TechWolf/Skill-extraction-Tech-graded", **kwargs) @property def name(self) -> str: @@ -521,6 +469,51 @@ def citation(self) -> str: """ +@register_task() +class SkillSkapeGradedSkillExtractRanking(BaseGradedSkillExtractRanking): + """Skill Extraction from SkillSkape Dataset with Graded Relevance. + + Re-annotates the sentences from ``jjzha/skillskape`` against the full ESCO + v1.1.0 skill taxonomy with a 0-4 relevance scale, following the BEIR layout + (``queries``, ``corpus``, ``qrels``). Score scale: 0 unrelated, 1 plausible + in domain, 2 recommendable but off-granularity, 3 strongly implied, + 4 explicitly demonstrated. + """ + + def __init__(self, **kwargs): + super().__init__(hf_name="TechWolf/Skill-extraction-SkillSkape-graded", **kwargs) + + @property + def name(self) -> str: + """Skill extraction SkillSkape graded task name.""" + return "Skill Extraction SkillSkape Graded" + + @property + def description(self) -> str: + """Skill extraction SkillSkape graded task description.""" + return ( + "Extract skills from text descriptions in SkillSkape, with graded 0-4 " + "relevance against the full ESCO v1.1.0 taxonomy." + ) + + @property + def citation(self) -> str: + """Skill extraction SkillSkape graded task citation.""" + return """@inproceedings{magron-etal-2024-jobskape, + title = {{JobSkape: A Framework for Generating Synthetic Job Postings to Enhance Skill Matching}}, + author = {{Magron, Antoine and Dai, Anna and Zhang, Mike and Montariol, Syrielle and Bosselut, Antoine}}, + editor = {{Hruschka, Estevam and Lake, Thom and Otani, Naoki and Mitchell, Tom}}, + booktitle = {{Proceedings of the First Workshop on Natural Language Processing for Human Resources (NLP4HR 2024)}}, + month = {{mar}}, + year = {{2024}}, + address = {{St. Julian's, Malta}}, + publisher = {{Association for Computational Linguistics}}, + url = {{https://aclanthology.org/2024.nlp4hr-1.4/}}, + pages = {{43--58}} +} +""" + + @register_task() class SkillSkapeExtractRanking(BaseESCOSkillExtractRanking): """Skill Extraction from SkillSkape Ranking Task.""" diff --git a/src/workrb/tasks/ranking/skillnorm.py b/src/workrb/tasks/ranking/skillnorm.py index b4b1f1d..6ce840c 100644 --- a/src/workrb/tasks/ranking/skillnorm.py +++ b/src/workrb/tasks/ranking/skillnorm.py @@ -8,6 +8,7 @@ from workrb.registry import register_task from workrb.tasks.abstract.base import DatasetSplit, LabelType, Language from workrb.tasks.abstract.ranking_base import RankingDataset, RankingTask, RankingTaskGroup +from workrb.tasks.ranking.graded_beir import GradedBEIRRankingTask from workrb.types import ModelInputType @@ -211,3 +212,46 @@ def citation(self) -> str: journal={arXiv preprint arXiv:2511.07969}, year={2025} }""" + + +@register_task() +class ESCOGradedSkillNormRanking(GradedBEIRRankingTask): + """Skill Normalization on ESCO with Graded Relevance. + + Re-annotates surface skill terms (ESCO alt-labels) against the full ESCO + v1.1.0 skill taxonomy with a 0-4 relevance scale, following the BEIR layout + (``queries``, ``corpus``, ``qrels``). Score scale: 0 unrelated, 1 plausible + in domain, 2 recommendable but off-granularity, 3 strongly implied, + 4 explicitly demonstrated. + """ + + def __init__(self, **kwargs): + super().__init__(hf_name="TechWolf/Skill-normalisation-ESCO-graded", **kwargs) + + @property + def task_group(self) -> RankingTaskGroup: + """Skill normalization task group.""" + return RankingTaskGroup.SKILL_NORMALIZATION + + @property + def query_input_type(self) -> ModelInputType: + """Query input type for surface skill terms.""" + return ModelInputType.SKILL_NAME + + @property + def name(self) -> str: + """Skill normalization ESCO graded task name.""" + return "Skill Normalization ESCO Graded" + + @property + def description(self) -> str: + """Skill normalization ESCO graded task description.""" + return ( + "Normalize surface skill terms to canonical ESCO skills, with graded 0-4 " + "relevance against the full ESCO v1.1.0 taxonomy." + ) + + @property + def citation(self) -> str: + """Skill normalization ESCO graded task citation.""" + return """To be announced.""" diff --git a/tests/test_import.py b/tests/test_import.py index 529f994..ea4dd3c 100644 --- a/tests/test_import.py +++ b/tests/test_import.py @@ -6,6 +6,7 @@ from workrb.config import BenchmarkConfig from workrb.registry import TaskRegistry from workrb.tasks import ( + ESCOGradedSkillNormRanking, ESCOJob2SkillClassification, ESCOJob2SkillRanking, ESCOSkill2JobRanking, @@ -16,6 +17,7 @@ RankingDataset, RankingTask, SkillMatch1kSkillSimilarityRanking, + SkillSkapeGradedSkillExtractRanking, Task, TechGradedSkillExtractRanking, TechSkillExtractRanking, @@ -56,6 +58,8 @@ def test_task_ranking_imports(): assert isinstance(SkillMatch1kSkillSimilarityRanking.__name__, str) assert isinstance(TechSkillExtractRanking.__name__, str) assert isinstance(TechGradedSkillExtractRanking.__name__, str) + assert isinstance(SkillSkapeGradedSkillExtractRanking.__name__, str) + assert isinstance(ESCOGradedSkillNormRanking.__name__, str) print("✓ Successfully imported ranking task classes") diff --git a/tests/test_task_loading.py b/tests/test_task_loading.py index 6c3a7c4..51442fb 100644 --- a/tests/test_task_loading.py +++ b/tests/test_task_loading.py @@ -6,6 +6,7 @@ from tests.test_utils import GeneralRankingTestTask from workrb.tasks import ( + ESCOGradedSkillNormRanking, ESCOJob2SkillClassification, ESCOJob2SkillRanking, ESCOSkill2JobRanking, @@ -18,6 +19,7 @@ MELSRanking, SkillMatch1kSkillSimilarityRanking, SkillSkapeExtractRanking, + SkillSkapeGradedSkillExtractRanking, TechGradedSkillExtractRanking, TechSkillExtractRanking, ) @@ -56,6 +58,8 @@ def test_ranking_tasks_init_en_splits(): ("SkillExtractTechRanking", TechSkillExtractRanking), ("SkillExtractTechGradedRanking", TechGradedSkillExtractRanking), ("SkillExtractSkillSkapeRanking", SkillSkapeExtractRanking), + ("SkillExtractSkillSkapeGradedRanking", SkillSkapeGradedSkillExtractRanking), + ("SkillNormESCOGradedRanking", ESCOGradedSkillNormRanking), ("SkillSimilarityRanking", SkillMatch1kSkillSimilarityRanking), ] @@ -67,6 +71,8 @@ def test_ranking_tasks_init_en_splits(): tasks_with_only_val_set = [ "SkillExtractHouseGradedRanking", "SkillExtractTechGradedRanking", + "SkillExtractSkillSkapeGradedRanking", + "SkillNormESCOGradedRanking", ] results = {"success": [], "failures": []} From cd2540dffde9de94da65d71f6d0b1572b5fb8f1a Mon Sep 17 00:00:00 2001 From: "warre.veys" Date: Tue, 2 Jun 2026 13:08:31 +0200 Subject: [PATCH 5/6] Fix KeyError when requested metrics omit the default key metric _record_dataset_result logged the per-dataset "key metric" via metrics_dict[task.default_metrics[0]]. When a caller passes a custom `metrics` set that excludes the task's first default metric (e.g. ["ndcg@100"] for ranking tasks whose default_metrics[0] is "ndcg"), that lookup raised KeyError and aborted the whole evaluation. Fall back to the first computed metric, and skip the log line if none exist. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/workrb/run.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/workrb/run.py b/src/workrb/run.py index 770cc7e..ebb9880 100644 --- a/src/workrb/run.py +++ b/src/workrb/run.py @@ -783,5 +783,11 @@ def _record_dataset_result( if config: config.save_results_checkpoint(results) + # Prefer the task's first default metric, but fall back to whatever was + # actually computed: a caller-supplied `metrics` set need not include the + # default key metric, and indexing it directly would raise KeyError. key_metric = task.default_metrics[0] - logger.info(f"\t{key_metric}: {metrics_dict[key_metric]:.3f}") + if key_metric not in metrics_dict: + key_metric = next(iter(metrics_dict), None) + if key_metric is not None: + logger.info(f"\t{key_metric}: {metrics_dict[key_metric]:.3f}") From be86b5d0eff2418876e45f57d4ddb865c0685519 Mon Sep 17 00:00:00 2001 From: "warre.veys" Date: Tue, 2 Jun 2026 15:46:00 +0200 Subject: [PATCH 6/6] test tasks --- pyproject.toml | 3 +- src/workrb/tasks/__init__.py | 2 + src/workrb/tasks/ranking/__init__.py | 2 + src/workrb/tasks/ranking/skill_extraction.py | 48 ++++++++++++++++++++ src/workrb/tasks/ranking/skillnorm.py | 9 ++++ 5 files changed, 62 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d9695a3..38f0428 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,8 +13,7 @@ readme = "README.md" requires-python = ">=3.10,<4.0" license = { text = "Apache 2.0" } authors = [ - { name = "TechWolf Research Team" }, - { name = "Matthias De Lange", email = "matthias.delange@techwolf.ai" }, + { name = "TechWolf Research Team", email = "workrb@techwolf.ai" }, ] keywords = [ "nlp", diff --git a/src/workrb/tasks/__init__.py b/src/workrb/tasks/__init__.py index ca4ce71..d008db6 100644 --- a/src/workrb/tasks/__init__.py +++ b/src/workrb/tasks/__init__.py @@ -26,6 +26,7 @@ SkillXLSkillExtractRanking, TechGradedSkillExtractRanking, TechSkillExtractRanking, + TechWolfGradedSkillExtractRanking, TechWolfSkillExtractRanking, ) from .ranking.skill_similarity import SkillMatch1kSkillSimilarityRanking @@ -54,6 +55,7 @@ "HouseSkillExtractRanking", "TechGradedSkillExtractRanking", "TechSkillExtractRanking", + "TechWolfGradedSkillExtractRanking", "TechWolfSkillExtractRanking", "SkillSkapeExtractRanking", "SkillSkapeGradedSkillExtractRanking", diff --git a/src/workrb/tasks/ranking/__init__.py b/src/workrb/tasks/ranking/__init__.py index f233947..876790f 100644 --- a/src/workrb/tasks/ranking/__init__.py +++ b/src/workrb/tasks/ranking/__init__.py @@ -25,6 +25,7 @@ SkillXLSkillExtractRanking, TechGradedSkillExtractRanking, TechSkillExtractRanking, + TechWolfGradedSkillExtractRanking, TechWolfSkillExtractRanking, ) from workrb.tasks.ranking.skill_similarity import SkillMatch1kSkillSimilarityRanking @@ -49,5 +50,6 @@ "SkillXLSkillExtractRanking", "TechGradedSkillExtractRanking", "TechSkillExtractRanking", + "TechWolfGradedSkillExtractRanking", "TechWolfSkillExtractRanking", ] diff --git a/src/workrb/tasks/ranking/skill_extraction.py b/src/workrb/tasks/ranking/skill_extraction.py index 65ef6a3..73762a7 100644 --- a/src/workrb/tasks/ranking/skill_extraction.py +++ b/src/workrb/tasks/ranking/skill_extraction.py @@ -360,6 +360,11 @@ class BaseGradedSkillExtractRanking(GradedBEIRRankingTask): just pins the skill-extraction task group and query input type. """ + @property + def split_to_hf_split(self) -> dict[DatasetSplit, str]: + """Expose both validation and test splits.""" + return {DatasetSplit.VAL: "validation", DatasetSplit.TEST: "test"} + @property def task_group(self) -> RankingTaskGroup: """Skill extraction task group.""" @@ -514,6 +519,49 @@ def citation(self) -> str: """ +@register_task() +class TechWolfGradedSkillExtractRanking(BaseGradedSkillExtractRanking): + """Skill Extraction from TechWolf Dataset with Graded Relevance. + + Re-annotates the sentences from ``TechWolf/skill-extraction-techwolf`` + against the full ESCO v1.1.0 skill taxonomy, following the BEIR layout + (``queries``, ``corpus``, ``qrels``). Only a ``test`` split is published for + this dataset. + """ + + def __init__(self, **kwargs): + super().__init__(hf_name="TechWolf/Skill-extraction-TechWolf-graded", **kwargs) + + @property + def split_to_hf_split(self) -> dict[DatasetSplit, str]: + """Only a test split is published for the TechWolf graded dataset.""" + return {DatasetSplit.TEST: "test"} + + @property + def name(self) -> str: + """Skill extraction TechWolf graded task name.""" + return "Skill Extraction TechWolf Graded" + + @property + def description(self) -> str: + """Skill extraction TechWolf graded task description.""" + return ( + "Extract skills from text descriptions in a generic distribution of job " + "descriptions, against the full ESCO v1.1.0 taxonomy." + ) + + @property + def citation(self) -> str: + """Skill extraction TechWolf graded task citation.""" + return """@article{decorte2023extreme, + title={Extreme multi-label skill extraction training using large language models}, + author={Decorte, Jens-Joris and Verlinden, Severine and Van Hautte, Jeroen and Deleu, Johannes and Develder, Chris and Demeester, Thomas}, + journal={arXiv preprint arXiv:2307.10778}, + year={2023} +} +""" + + @register_task() class SkillSkapeExtractRanking(BaseESCOSkillExtractRanking): """Skill Extraction from SkillSkape Ranking Task.""" diff --git a/src/workrb/tasks/ranking/skillnorm.py b/src/workrb/tasks/ranking/skillnorm.py index 6ce840c..e7069fc 100644 --- a/src/workrb/tasks/ranking/skillnorm.py +++ b/src/workrb/tasks/ranking/skillnorm.py @@ -228,6 +228,15 @@ class ESCOGradedSkillNormRanking(GradedBEIRRankingTask): def __init__(self, **kwargs): super().__init__(hf_name="TechWolf/Skill-normalisation-ESCO-graded", **kwargs) + @property + def split_to_hf_split(self) -> dict[DatasetSplit, str]: + """Expose both validation and test splits. + + The graded repo publishes a ``validation`` split (full 0-4 relevance) + and a ``test`` split, both under the ``queries`` and ``qrels`` configs. + """ + return {DatasetSplit.VAL: "validation", DatasetSplit.TEST: "test"} + @property def task_group(self) -> RankingTaskGroup: """Skill normalization task group."""