diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d3cf48..12abffc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,23 @@ ### Feat +- ``HouseGradedSkillExtractRanking``: graded-relevance skill-extraction ranking + task on the HOUSE subset of CAREER, re-annotated against the full ESCO v1.1.0 + taxonomy with a 0-4 score scale (HF: ``TechWolf/Skill-extraction-House-graded``, + BEIR layout, validation split only). +- ``TechGradedSkillExtractRanking``: graded-relevance skill-extraction ranking + task on the TECH subset of CAREER, re-annotated against the full ESCO v1.1.0 + taxonomy with a 0-4 score scale (HF: ``TechWolf/Skill-extraction-Tech-graded``, + BEIR layout, validation split only). +- ``SkillSkapeGradedSkillExtractRanking``: graded-relevance skill-extraction + ranking task on SkillSkape, re-annotated against the full ESCO v1.1.0 taxonomy + with a 0-4 score scale (HF: ``TechWolf/Skill-extraction-SkillSkape-graded``, + BEIR layout, validation split only). +- ``ESCOGradedSkillNormRanking``: graded-relevance skill-normalization ranking + task that maps surface skill terms (ESCO alt-labels) to canonical ESCO skills, + annotated against the full ESCO v1.1.0 taxonomy with a 0-4 score scale + (HF: ``TechWolf/Skill-normalisation-ESCO-graded``, BEIR layout, validation + split only). - graded relevance support for ranking metrics: ``RankingDataset`` accepts an optional ``target_relevance`` field aligned 1-to-1 with ``target_indices``. ``ndcg@k`` uses a (2^rel - 1) gain when graded labels are provided; binary diff --git a/README.md b/README.md index b42b42b..b32e8cf 100644 --- a/README.md +++ b/README.md @@ -85,12 +85,16 @@ pip install workrb | Job Normalization MELO | `MELORanking` | multi_label | 633 queries x 33813 targets | 21 | | Skill to Job WorkBench | `ESCOSkill2JobRanking` | multi_label | 13492 queries x 3039 targets | 28 | | Skill Extraction House | `HouseSkillExtractRanking` | multi_label | 262 queries x 13891 targets | 28 | +| Skill Extraction House Graded | `HouseGradedSkillExtractRanking` | multi_label | 61 queries x 13891 targets | 1 | | Skill Extraction Tech | `TechSkillExtractRanking` | multi_label | 338 queries x 13891 targets | 28 | +| Skill Extraction Tech Graded | `TechGradedSkillExtractRanking` | multi_label | 75 queries x 13891 targets | 1 | | Skill Extraction SkillSkape | `SkillSkapeExtractRanking` | multi_label | 1191 queries x 13891 targets | 28 | +| Skill Extraction SkillSkape Graded | `SkillSkapeGradedSkillExtractRanking` | multi_label | 100 queries x 13891 targets | 1 | | Skill Extraction TechWolf | `TechWolfSkillExtractRanking` | multi_label | 326 queries x 13891 targets | 28 | | Skill Extraction SkillXL | `SkillXLSkillExtractRanking` | multi_label | 944 queries x 13891 targets | 28 | | Skill Similarity SkillMatch-1K | `SkillMatch1kSkillSimilarityRanking` | single_label | 900 queries x 2648 targets | 1 | | Skill Normalization ESCO | `ESCOSkillNormRanking` | multi_label | 72008 queries x 13939 targets | 28 | +| Skill Normalization ESCO Graded | `ESCOGradedSkillNormRanking` | multi_label | 50 queries x 13891 targets | 1 | | Skill Normalization MELS | `MELSRanking` | multi_label | 1722 queries x 19466 targets | 5 | | Query-Candidate Matching | `SearchQueryCandidateRanking` | multi_label | 200 queries x 4019 (x-lang) targets | 5 | | Project-Candidate Matching | `ProjectCandidateRanking` | multi_label | 200 queries x 4019 (x-lang) targets | 5 | diff --git a/pyproject.toml b/pyproject.toml index d9695a3..38f0428 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,8 +13,7 @@ readme = "README.md" requires-python = ">=3.10,<4.0" license = { text = "Apache 2.0" } authors = [ - { name = "TechWolf Research Team" }, - { name = "Matthias De Lange", email = "matthias.delange@techwolf.ai" }, + { name = "TechWolf Research Team", email = "workrb@techwolf.ai" }, ] keywords = [ "nlp", diff --git a/src/workrb/rankings.py b/src/workrb/rankings.py index a61c165..ac3f501 100644 --- a/src/workrb/rankings.py +++ b/src/workrb/rankings.py @@ -140,9 +140,7 @@ def materialize_prediction_matrix( matrix = np.zeros((num_queries, num_targets), dtype=np.float32) for q_idx, row in scores.items(): if not 0 <= q_idx < num_queries: - raise IndexError( - f"query_index {q_idx} out of bounds for num_queries={num_queries}" - ) + raise IndexError(f"query_index {q_idx} out of bounds for num_queries={num_queries}") for t_idx, score in row.items(): if not 0 <= t_idx < num_targets: raise IndexError( diff --git a/src/workrb/run.py b/src/workrb/run.py index 770cc7e..ebb9880 100644 --- a/src/workrb/run.py +++ b/src/workrb/run.py @@ -783,5 +783,11 @@ def _record_dataset_result( if config: config.save_results_checkpoint(results) + # Prefer the task's first default metric, but fall back to whatever was + # actually computed: a caller-supplied `metrics` set need not include the + # default key metric, and indexing it directly would raise KeyError. key_metric = task.default_metrics[0] - logger.info(f"\t{key_metric}: {metrics_dict[key_metric]:.3f}") + if key_metric not in metrics_dict: + key_metric = next(iter(metrics_dict), None) + if key_metric is not None: + logger.info(f"\t{key_metric}: {metrics_dict[key_metric]:.3f}") diff --git a/src/workrb/tasks/__init__.py b/src/workrb/tasks/__init__.py index ae66be8..d008db6 100644 --- a/src/workrb/tasks/__init__.py +++ b/src/workrb/tasks/__init__.py @@ -19,14 +19,18 @@ from .ranking.mels import MELSRanking from .ranking.skill2job import ESCOSkill2JobRanking from .ranking.skill_extraction import ( + HouseGradedSkillExtractRanking, HouseSkillExtractRanking, SkillSkapeExtractRanking, + SkillSkapeGradedSkillExtractRanking, SkillXLSkillExtractRanking, + TechGradedSkillExtractRanking, TechSkillExtractRanking, + TechWolfGradedSkillExtractRanking, TechWolfSkillExtractRanking, ) from .ranking.skill_similarity import SkillMatch1kSkillSimilarityRanking -from .ranking.skillnorm import ESCOSkillNormRanking +from .ranking.skillnorm import ESCOGradedSkillNormRanking, ESCOSkillNormRanking __all__ = [ # Abstract classes @@ -39,6 +43,7 @@ # Classification tasks "ESCOJob2SkillClassification", # Ranking tasks + "ESCOGradedSkillNormRanking", "ESCOJob2SkillRanking", "ESCOSkill2JobRanking", "ESCOSkillNormRanking", @@ -46,10 +51,14 @@ "JobTitleSimilarityRanking", "MELORanking", "MELSRanking", + "HouseGradedSkillExtractRanking", "HouseSkillExtractRanking", + "TechGradedSkillExtractRanking", "TechSkillExtractRanking", + "TechWolfGradedSkillExtractRanking", "TechWolfSkillExtractRanking", "SkillSkapeExtractRanking", + "SkillSkapeGradedSkillExtractRanking", "SkillXLSkillExtractRanking", "SkillMatch1kSkillSimilarityRanking", "ProjectCandidateRanking", diff --git a/src/workrb/tasks/abstract/ranking_base.py b/src/workrb/tasks/abstract/ranking_base.py index 00a5842..581cce7 100644 --- a/src/workrb/tasks/abstract/ranking_base.py +++ b/src/workrb/tasks/abstract/ranking_base.py @@ -476,4 +476,3 @@ def compute_metrics_from_prediction_matrix( pos_label_relevance=dataset.target_relevance, binary_relevance_threshold=self.binary_relevance_threshold, ) - diff --git a/src/workrb/tasks/ranking/__init__.py b/src/workrb/tasks/ranking/__init__.py index 63f87df..876790f 100644 --- a/src/workrb/tasks/ranking/__init__.py +++ b/src/workrb/tasks/ranking/__init__.py @@ -18,19 +18,25 @@ from workrb.tasks.ranking.mels import MELSRanking from workrb.tasks.ranking.skill2job import ESCOSkill2JobRanking from workrb.tasks.ranking.skill_extraction import ( + HouseGradedSkillExtractRanking, HouseSkillExtractRanking, SkillSkapeExtractRanking, + SkillSkapeGradedSkillExtractRanking, SkillXLSkillExtractRanking, + TechGradedSkillExtractRanking, TechSkillExtractRanking, + TechWolfGradedSkillExtractRanking, TechWolfSkillExtractRanking, ) from workrb.tasks.ranking.skill_similarity import SkillMatch1kSkillSimilarityRanking -from workrb.tasks.ranking.skillnorm import ESCOSkillNormRanking +from workrb.tasks.ranking.skillnorm import ESCOGradedSkillNormRanking, ESCOSkillNormRanking __all__ = [ + "ESCOGradedSkillNormRanking", "ESCOJob2SkillRanking", "ESCOSkill2JobRanking", "ESCOSkillNormRanking", + "HouseGradedSkillExtractRanking", "HouseSkillExtractRanking", "JobBERTJobNormRanking", "JobTitleSimilarityRanking", @@ -40,7 +46,10 @@ "SearchQueryCandidateRanking", "SkillMatch1kSkillSimilarityRanking", "SkillSkapeExtractRanking", + "SkillSkapeGradedSkillExtractRanking", "SkillXLSkillExtractRanking", + "TechGradedSkillExtractRanking", "TechSkillExtractRanking", + "TechWolfGradedSkillExtractRanking", "TechWolfSkillExtractRanking", ] diff --git a/src/workrb/tasks/ranking/graded_beir.py b/src/workrb/tasks/ranking/graded_beir.py new file mode 100644 index 0000000..14272f2 --- /dev/null +++ b/src/workrb/tasks/ranking/graded_beir.py @@ -0,0 +1,146 @@ +"""Shared base for BEIR-layout graded-relevance ranking tasks. + +Several tasks publish graded (0-4) relevance annotations on the Hugging Face +Hub following the BEIR convention (``queries``, ``corpus``, ``qrels`` configs). +They differ only in their task group and query input type (e.g. skill +extraction from sentences vs skill normalization from surface terms), so the +data-loading logic lives here and concrete tasks supply the task-specific +properties. +""" + +import pandas as pd +from datasets import Dataset, load_dataset + +from workrb.tasks.abstract.base import DatasetSplit, LabelType, Language +from workrb.tasks.abstract.ranking_base import RankingDataset, RankingTask +from workrb.types import ModelInputType + + +class GradedBEIRRankingTask(RankingTask): + """Base class for BEIR-layout graded ranking tasks. + + Reads the ``queries``, ``corpus`` and ``qrels`` configs published on the + Hugging Face Hub via ``load_dataset``. The target_space is the corpus's + ``title`` column (ESCO preferred labels, in corpus order); qrels are + expected to contain only non-zero judgments (absent items are implicit + grade 0). + + Which splits a task exposes depends on what the underlying dataset + publishes: some release both a validation and a test split, others only + one. Because several tasks share this loader, the supported splits are + declared per subclass via :attr:`split_to_hf_split` (which maps each + supported :class:`DatasetSplit` to the HF split name its + ``queries``/``qrels`` configs live under) rather than hardcoded with an + inline guard as in the single-split tasks (e.g. ``MELORanking``). The + default exposes only the validation split, the common case for in-progress + benchmark datasets whose test split is withheld. + + Concrete subclasses set ``hf_name`` via ``__init__`` and supply the + task-specific ``task_group``, ``query_input_type``, ``name``, + ``description`` and ``citation``. + """ + + def __init__(self, hf_name: str, **kwargs): + """Initialize the task. + + Args: + hf_name: Name of the Hugging Face dataset (BEIR layout). + **kwargs: Additional arguments for the base class. + """ + self.hf_name = hf_name + super().__init__(**kwargs) + + @property + def split_to_hf_split(self) -> dict[DatasetSplit, str]: + """Map each supported split to the HF split name backing it. + + The corpus config is always loaded from the ``corpus`` split; this + mapping only governs the ``queries`` and ``qrels`` configs. Override to + expose more or fewer splits, e.g. ``{DatasetSplit.VAL: "validation", + DatasetSplit.TEST: "test"}`` for a dataset that releases both. + """ + return {DatasetSplit.VAL: "validation"} + + @property + def supported_query_languages(self) -> list[Language]: + """Annotations are released in English only at this stage.""" + return [Language.EN] + + @property + def supported_target_languages(self) -> list[Language]: + """The corpus titles are released in English only at this stage.""" + return [Language.EN] + + @property + def label_type(self) -> LabelType: + """Label type is multi-label.""" + return LabelType.MULTI_LABEL + + @property + def target_input_type(self) -> ModelInputType: + """Target input type for ESCO skills.""" + return ModelInputType.SKILL_NAME + + @property + def default_metrics(self) -> list[str]: + """Default metrics include nDCG to leverage the graded labels. + + ``ndcg`` without a cutoff scores the full ranking (k = |target_space|). + """ + return ["ndcg", "ndcg@5", "ndcg@10", "map", "rp@10", "mrr"] + + def load_dataset(self, dataset_id: str, split: DatasetSplit) -> RankingDataset: + """Load BEIR-style graded annotations and convert to a RankingDataset.""" + hf_split = self.split_to_hf_split.get(split) + if hf_split is None: + supported = ", ".join(sorted(s.value for s in self.split_to_hf_split)) + raise ValueError( + f"Split '{split.value}' not supported for {type(self).__name__}: " + f"only [{supported}] {'is' if len(self.split_to_hf_split) == 1 else 'are'} " + f"annotated for this dataset." + ) + + queries_ds = load_dataset(self.hf_name, "queries", split=hf_split) + corpus_ds = load_dataset(self.hf_name, "corpus", split="corpus") + qrels_ds = load_dataset(self.hf_name, "qrels", split=hf_split) + assert isinstance(queries_ds, Dataset) + assert isinstance(corpus_ds, Dataset) + assert isinstance(qrels_ds, Dataset) + queries_df = queries_ds.to_pandas() + corpus_df = corpus_ds.to_pandas() + qrels_df = qrels_ds.to_pandas() + assert isinstance(queries_df, pd.DataFrame) + assert isinstance(corpus_df, pd.DataFrame) + assert isinstance(qrels_df, pd.DataFrame) + + # target_space is the corpus titles in corpus order; URIs map by row index. + target_space = corpus_df["title"].tolist() + uri_to_idx = {uri: i for i, uri in enumerate(corpus_df["_id"])} + + qrels_df["target_idx"] = qrels_df["corpus-id"].map(uri_to_idx) + # Every qrel should resolve; if any don't, surface the issue rather than silently dropping. + unresolved = qrels_df["target_idx"].isna().sum() + assert unresolved == 0, ( + f"{unresolved} qrel rows reference corpus-ids not present in the corpus config" + ) + qrels_df["target_idx"] = qrels_df["target_idx"].astype(int) + + id_to_query = dict(zip(queries_df["_id"], queries_df["text"], strict=True)) + qrels_df["sentence"] = qrels_df["query-id"].map(id_to_query) + + grouped = qrels_df.groupby("sentence") + filtered_queries: list[str] = [] + filtered_indices: list[list[int]] = [] + filtered_relevance: list[list[float]] = [] + for sentence, group in grouped: + filtered_queries.append(str(sentence)) + filtered_indices.append(group["target_idx"].tolist()) + filtered_relevance.append([float(s) for s in group["score"].tolist()]) + + return RankingDataset( + query_texts=filtered_queries, + target_indices=filtered_indices, + target_space=target_space, + dataset_id=dataset_id, + target_relevance=filtered_relevance, + ) diff --git a/src/workrb/tasks/ranking/skill_extraction.py b/src/workrb/tasks/ranking/skill_extraction.py index e790a17..73762a7 100644 --- a/src/workrb/tasks/ranking/skill_extraction.py +++ b/src/workrb/tasks/ranking/skill_extraction.py @@ -11,6 +11,7 @@ RankingTask, RankingTaskGroup, ) +from workrb.tasks.ranking.graded_beir import GradedBEIRRankingTask from workrb.types import ModelInputType @@ -351,6 +352,216 @@ def citation(self) -> str: """ +class BaseGradedSkillExtractRanking(GradedBEIRRankingTask): + """Base class for BEIR-layout graded skill-extraction tasks. + + Queries are skill-bearing sentences ranked against the ESCO skill taxonomy. + The BEIR loading logic lives in :class:`GradedBEIRRankingTask`; this base + just pins the skill-extraction task group and query input type. + """ + + @property + def split_to_hf_split(self) -> dict[DatasetSplit, str]: + """Expose both validation and test splits.""" + return {DatasetSplit.VAL: "validation", DatasetSplit.TEST: "test"} + + @property + def task_group(self) -> RankingTaskGroup: + """Skill extraction task group.""" + return RankingTaskGroup.SKILL_EXTRACTION + + @property + def query_input_type(self) -> ModelInputType: + """Query input type for skill extraction sentences.""" + return ModelInputType.SKILL_SENTENCE + + +@register_task() +class HouseGradedSkillExtractRanking(BaseGradedSkillExtractRanking): + """Skill Extraction from HOUSE Dataset with Graded Relevance. + + Re-annotates the sentences from ``TechWolf/skill-extraction-house`` against + the full ESCO v1.1.0 skill taxonomy with a 0-4 relevance scale, following the + BEIR layout (``queries``, ``corpus``, ``qrels``). Score scale: 0 unrelated, + 1 plausible in domain, 2 recommendable but off-granularity, 3 strongly implied, + 4 explicitly demonstrated. + """ + + def __init__(self, **kwargs): + super().__init__(hf_name="TechWolf/Skill-extraction-House-graded", **kwargs) + + @property + def name(self) -> str: + """Skill extraction HOUSE graded task name.""" + return "Skill Extraction House Graded" + + @property + def description(self) -> str: + """Skill extraction HOUSE graded task description.""" + return ( + "Extract skills from general text descriptions in the HOUSE subset of CAREER, " + "with graded 0-4 relevance against the full ESCO v1.1.0 taxonomy." + ) + + @property + def citation(self) -> str: + """Skill extraction HOUSE graded task citation.""" + return """@inproceedings{decorte2022design, + articleno = {{4}}, + author = {{Decorte, Jens-Joris and Van Hautte, Jeroen and Deleu, Johannes and Develder, Chris and Demeester, Thomas}}, + booktitle = {{Proceedings of the 2nd Workshop on Recommender Systems for Human Resources (RecSys-in-HR 2022)}}, + editor = {{Kaya, Mesut and Bogers, Toine and Graus, David and Mesbah, Sepideh and Johnson, Chris and Gutiérrez, Francisco}}, + isbn = {{9781450398565}}, + issn = {{1613-0073}}, + language = {{eng}}, + location = {{Seatle, USA}}, + pages = {{7}}, + publisher = {{CEUR}}, + title = {{Design of negative sampling strategies for distantly supervised skill extraction}}, + url = {{https://ceur-ws.org/Vol-3218/RecSysHR2022-paper_4.pdf}}, + volume = {{3218}}, + year = {{2022}}, +} +""" + + +@register_task() +class TechGradedSkillExtractRanking(BaseGradedSkillExtractRanking): + """Skill Extraction from TECH Dataset with Graded Relevance. + + Re-annotates the sentences from ``TechWolf/skill-extraction-tech`` against + the full ESCO v1.1.0 skill taxonomy with a 0-4 relevance scale, following the + BEIR layout (``queries``, ``corpus``, ``qrels``). Score scale: 0 unrelated, + 1 plausible in domain, 2 recommendable but off-granularity, 3 strongly implied, + 4 explicitly demonstrated. + """ + + def __init__(self, **kwargs): + super().__init__(hf_name="TechWolf/Skill-extraction-Tech-graded", **kwargs) + + @property + def name(self) -> str: + """Skill extraction TECH graded task name.""" + return "Skill Extraction Tech Graded" + + @property + def description(self) -> str: + """Skill extraction TECH graded task description.""" + return ( + "Extract skills from technical text descriptions in the TECH subset of CAREER, " + "with graded 0-4 relevance against the full ESCO v1.1.0 taxonomy." + ) + + @property + def citation(self) -> str: + """Skill extraction TECH graded task citation.""" + return """@inproceedings{decorte2022design, + articleno = {{4}}, + author = {{Decorte, Jens-Joris and Van Hautte, Jeroen and Deleu, Johannes and Develder, Chris and Demeester, Thomas}}, + booktitle = {{Proceedings of the 2nd Workshop on Recommender Systems for Human Resources (RecSys-in-HR 2022)}}, + editor = {{Kaya, Mesut and Bogers, Toine and Graus, David and Mesbah, Sepideh and Johnson, Chris and Gutiérrez, Francisco}}, + isbn = {{9781450398565}}, + issn = {{1613-0073}}, + language = {{eng}}, + location = {{Seatle, USA}}, + pages = {{7}}, + publisher = {{CEUR}}, + title = {{Design of negative sampling strategies for distantly supervised skill extraction}}, + url = {{https://ceur-ws.org/Vol-3218/RecSysHR2022-paper_4.pdf}}, + volume = {{3218}}, + year = {{2022}}, +} +""" + + +@register_task() +class SkillSkapeGradedSkillExtractRanking(BaseGradedSkillExtractRanking): + """Skill Extraction from SkillSkape Dataset with Graded Relevance. + + Re-annotates the sentences from ``jjzha/skillskape`` against the full ESCO + v1.1.0 skill taxonomy with a 0-4 relevance scale, following the BEIR layout + (``queries``, ``corpus``, ``qrels``). Score scale: 0 unrelated, 1 plausible + in domain, 2 recommendable but off-granularity, 3 strongly implied, + 4 explicitly demonstrated. + """ + + def __init__(self, **kwargs): + super().__init__(hf_name="TechWolf/Skill-extraction-SkillSkape-graded", **kwargs) + + @property + def name(self) -> str: + """Skill extraction SkillSkape graded task name.""" + return "Skill Extraction SkillSkape Graded" + + @property + def description(self) -> str: + """Skill extraction SkillSkape graded task description.""" + return ( + "Extract skills from text descriptions in SkillSkape, with graded 0-4 " + "relevance against the full ESCO v1.1.0 taxonomy." + ) + + @property + def citation(self) -> str: + """Skill extraction SkillSkape graded task citation.""" + return """@inproceedings{magron-etal-2024-jobskape, + title = {{JobSkape: A Framework for Generating Synthetic Job Postings to Enhance Skill Matching}}, + author = {{Magron, Antoine and Dai, Anna and Zhang, Mike and Montariol, Syrielle and Bosselut, Antoine}}, + editor = {{Hruschka, Estevam and Lake, Thom and Otani, Naoki and Mitchell, Tom}}, + booktitle = {{Proceedings of the First Workshop on Natural Language Processing for Human Resources (NLP4HR 2024)}}, + month = {{mar}}, + year = {{2024}}, + address = {{St. Julian's, Malta}}, + publisher = {{Association for Computational Linguistics}}, + url = {{https://aclanthology.org/2024.nlp4hr-1.4/}}, + pages = {{43--58}} +} +""" + + +@register_task() +class TechWolfGradedSkillExtractRanking(BaseGradedSkillExtractRanking): + """Skill Extraction from TechWolf Dataset with Graded Relevance. + + Re-annotates the sentences from ``TechWolf/skill-extraction-techwolf`` + against the full ESCO v1.1.0 skill taxonomy, following the BEIR layout + (``queries``, ``corpus``, ``qrels``). Only a ``test`` split is published for + this dataset. + """ + + def __init__(self, **kwargs): + super().__init__(hf_name="TechWolf/Skill-extraction-TechWolf-graded", **kwargs) + + @property + def split_to_hf_split(self) -> dict[DatasetSplit, str]: + """Only a test split is published for the TechWolf graded dataset.""" + return {DatasetSplit.TEST: "test"} + + @property + def name(self) -> str: + """Skill extraction TechWolf graded task name.""" + return "Skill Extraction TechWolf Graded" + + @property + def description(self) -> str: + """Skill extraction TechWolf graded task description.""" + return ( + "Extract skills from text descriptions in a generic distribution of job " + "descriptions, against the full ESCO v1.1.0 taxonomy." + ) + + @property + def citation(self) -> str: + """Skill extraction TechWolf graded task citation.""" + return """@article{decorte2023extreme, + title={Extreme multi-label skill extraction training using large language models}, + author={Decorte, Jens-Joris and Verlinden, Severine and Van Hautte, Jeroen and Deleu, Johannes and Develder, Chris and Demeester, Thomas}, + journal={arXiv preprint arXiv:2307.10778}, + year={2023} +} +""" + + @register_task() class SkillSkapeExtractRanking(BaseESCOSkillExtractRanking): """Skill Extraction from SkillSkape Ranking Task.""" diff --git a/src/workrb/tasks/ranking/skillnorm.py b/src/workrb/tasks/ranking/skillnorm.py index b4b1f1d..e7069fc 100644 --- a/src/workrb/tasks/ranking/skillnorm.py +++ b/src/workrb/tasks/ranking/skillnorm.py @@ -8,6 +8,7 @@ from workrb.registry import register_task from workrb.tasks.abstract.base import DatasetSplit, LabelType, Language from workrb.tasks.abstract.ranking_base import RankingDataset, RankingTask, RankingTaskGroup +from workrb.tasks.ranking.graded_beir import GradedBEIRRankingTask from workrb.types import ModelInputType @@ -211,3 +212,55 @@ def citation(self) -> str: journal={arXiv preprint arXiv:2511.07969}, year={2025} }""" + + +@register_task() +class ESCOGradedSkillNormRanking(GradedBEIRRankingTask): + """Skill Normalization on ESCO with Graded Relevance. + + Re-annotates surface skill terms (ESCO alt-labels) against the full ESCO + v1.1.0 skill taxonomy with a 0-4 relevance scale, following the BEIR layout + (``queries``, ``corpus``, ``qrels``). Score scale: 0 unrelated, 1 plausible + in domain, 2 recommendable but off-granularity, 3 strongly implied, + 4 explicitly demonstrated. + """ + + def __init__(self, **kwargs): + super().__init__(hf_name="TechWolf/Skill-normalisation-ESCO-graded", **kwargs) + + @property + def split_to_hf_split(self) -> dict[DatasetSplit, str]: + """Expose both validation and test splits. + + The graded repo publishes a ``validation`` split (full 0-4 relevance) + and a ``test`` split, both under the ``queries`` and ``qrels`` configs. + """ + return {DatasetSplit.VAL: "validation", DatasetSplit.TEST: "test"} + + @property + def task_group(self) -> RankingTaskGroup: + """Skill normalization task group.""" + return RankingTaskGroup.SKILL_NORMALIZATION + + @property + def query_input_type(self) -> ModelInputType: + """Query input type for surface skill terms.""" + return ModelInputType.SKILL_NAME + + @property + def name(self) -> str: + """Skill normalization ESCO graded task name.""" + return "Skill Normalization ESCO Graded" + + @property + def description(self) -> str: + """Skill normalization ESCO graded task description.""" + return ( + "Normalize surface skill terms to canonical ESCO skills, with graded 0-4 " + "relevance against the full ESCO v1.1.0 taxonomy." + ) + + @property + def citation(self) -> str: + """Skill normalization ESCO graded task citation.""" + return """To be announced.""" diff --git a/tests/test_import.py b/tests/test_import.py index e2dccf3..ea4dd3c 100644 --- a/tests/test_import.py +++ b/tests/test_import.py @@ -6,16 +6,20 @@ from workrb.config import BenchmarkConfig from workrb.registry import TaskRegistry from workrb.tasks import ( + ESCOGradedSkillNormRanking, ESCOJob2SkillClassification, ESCOJob2SkillRanking, ESCOSkill2JobRanking, ESCOSkillNormRanking, + HouseGradedSkillExtractRanking, HouseSkillExtractRanking, JobBERTJobNormRanking, RankingDataset, RankingTask, SkillMatch1kSkillSimilarityRanking, + SkillSkapeGradedSkillExtractRanking, Task, + TechGradedSkillExtractRanking, TechSkillExtractRanking, ranking, ) @@ -49,9 +53,13 @@ def test_task_ranking_imports(): assert isinstance(ESCOSkill2JobRanking.__name__, str) assert isinstance(ESCOSkillNormRanking.__name__, str) assert isinstance(HouseSkillExtractRanking.__name__, str) + assert isinstance(HouseGradedSkillExtractRanking.__name__, str) assert isinstance(JobBERTJobNormRanking.__name__, str) assert isinstance(SkillMatch1kSkillSimilarityRanking.__name__, str) assert isinstance(TechSkillExtractRanking.__name__, str) + assert isinstance(TechGradedSkillExtractRanking.__name__, str) + assert isinstance(SkillSkapeGradedSkillExtractRanking.__name__, str) + assert isinstance(ESCOGradedSkillNormRanking.__name__, str) print("✓ Successfully imported ranking task classes") diff --git a/tests/test_save_rankings_artifacts.py b/tests/test_save_rankings_artifacts.py index b17bdc2..e6d6037 100644 --- a/tests/test_save_rankings_artifacts.py +++ b/tests/test_save_rankings_artifacts.py @@ -338,9 +338,7 @@ def test_save_rankings_rejects_non_finite_scores(): from workrb.config import BenchmarkConfig - dataset = TinyRankingTask( - split=DatasetSplit.TEST, languages=[Language.EN] - ).datasets["en"] + dataset = TinyRankingTask(split=DatasetSplit.TEST, languages=[Language.EN]).datasets["en"] config = BenchmarkConfig( model_name="tiny", output_folder=str(_fresh_dir("rankings_artifact_non_finite_write")), diff --git a/tests/test_task_loading.py b/tests/test_task_loading.py index 8e1489c..51442fb 100644 --- a/tests/test_task_loading.py +++ b/tests/test_task_loading.py @@ -6,10 +6,12 @@ from tests.test_utils import GeneralRankingTestTask from workrb.tasks import ( + ESCOGradedSkillNormRanking, ESCOJob2SkillClassification, ESCOJob2SkillRanking, ESCOSkill2JobRanking, ESCOSkillNormRanking, + HouseGradedSkillExtractRanking, HouseSkillExtractRanking, JobBERTJobNormRanking, JobTitleSimilarityRanking, @@ -17,6 +19,8 @@ MELSRanking, SkillMatch1kSkillSimilarityRanking, SkillSkapeExtractRanking, + SkillSkapeGradedSkillExtractRanking, + TechGradedSkillExtractRanking, TechSkillExtractRanking, ) from workrb.tasks.abstract.base import DatasetSplit, Language, TaskType @@ -50,8 +54,12 @@ def test_ranking_tasks_init_en_splits(): ("MELORanking", MELORanking), ("MELSRanking", MELSRanking), ("SkillExtractHouseRanking", HouseSkillExtractRanking), + ("SkillExtractHouseGradedRanking", HouseGradedSkillExtractRanking), ("SkillExtractTechRanking", TechSkillExtractRanking), + ("SkillExtractTechGradedRanking", TechGradedSkillExtractRanking), ("SkillExtractSkillSkapeRanking", SkillSkapeExtractRanking), + ("SkillExtractSkillSkapeGradedRanking", SkillSkapeGradedSkillExtractRanking), + ("SkillNormESCOGradedRanking", ESCOGradedSkillNormRanking), ("SkillSimilarityRanking", SkillMatch1kSkillSimilarityRanking), ] @@ -60,6 +68,12 @@ def test_ranking_tasks_init_en_splits(): "MELORanking", "MELSRanking", ] + tasks_with_only_val_set = [ + "SkillExtractHouseGradedRanking", + "SkillExtractTechGradedRanking", + "SkillExtractSkillSkapeGradedRanking", + "SkillNormESCOGradedRanking", + ] results = {"success": [], "failures": []} languages = [Language.EN] @@ -70,6 +84,8 @@ def test_ranking_tasks_init_en_splits(): for task_name, task_class in ranking_tasks: if split != DatasetSplit.TEST and task_name in tasks_with_only_test_set: continue + if split != DatasetSplit.VAL and task_name in tasks_with_only_val_set: + continue nb_total += 1 try: # Try to instantiate with minimal parameters