From 261862e9c0d3dd72258336a8af63924977c01a20 Mon Sep 17 00:00:00 2001 From: federetyk Date: Wed, 15 Jul 2026 13:00:00 +0200 Subject: [PATCH] fix: warn when a task resolves to zero datasets Task.__init__ silently produced an empty task in two cases: when the requested languages matched no dataset (for cross-lingual tasks the request must cover both query and target languages), and when every selected dataset was skipped with DatasetConfigNotSupported during loading. Both cases now log a warning that names the task and explains why it is empty. Reported by a RecSys 2026 reviewer: SearchQueryCandidateRanking with languages=["de"] returned empty results with no indication why. --- src/workrb/tasks/abstract/base.py | 17 ++++++++ tests/test_task_loading.py | 71 ++++++++++++++++++++++++++++++- 2 files changed, 87 insertions(+), 1 deletion(-) diff --git a/src/workrb/tasks/abstract/base.py b/src/workrb/tasks/abstract/base.py index b2d522c..4a8298e 100644 --- a/src/workrb/tasks/abstract/base.py +++ b/src/workrb/tasks/abstract/base.py @@ -70,11 +70,28 @@ def __init__( # Select dataset identifiers that match the requested languages self.dataset_ids = self.languages_to_dataset_ids(self.languages) + if not self.dataset_ids: + logger.warning( + "Task '%s' has no datasets for the requested languages %s. " + "The task will produce no results. For cross-lingual tasks, the " + "requested languages must cover both the query and the target " + "language of a dataset (e.g. languages=['de', 'en'] for a " + "German-to-English dataset).", + self.name, + [lang.value for lang in self.languages], + ) # Load datasets for the selected dataset identifiers self.datasets = self._load_datasets(dataset_ids=self.dataset_ids, split=self.split) # Skipped datasets (DatasetConfigNotSupported) are excluded from self.dataset_ids + if self.dataset_ids and not self.datasets: + logger.warning( + "Task '%s': all %d selected datasets were skipped during loading. " + "The task will produce no results.", + self.name, + len(self.dataset_ids), + ) self.dataset_ids = list(self.datasets.keys()) def _parse_languages( diff --git a/tests/test_task_loading.py b/tests/test_task_loading.py index 51442fb..2853c9e 100644 --- a/tests/test_task_loading.py +++ b/tests/test_task_loading.py @@ -1,5 +1,6 @@ """Test loading of all concrete task implementations.""" +import logging from typing import Literal import pytest @@ -17,13 +18,19 @@ JobTitleSimilarityRanking, MELORanking, MELSRanking, + SearchQueryCandidateRanking, SkillMatch1kSkillSimilarityRanking, SkillSkapeExtractRanking, SkillSkapeGradedSkillExtractRanking, TechGradedSkillExtractRanking, TechSkillExtractRanking, ) -from workrb.tasks.abstract.base import DatasetSplit, Language, TaskType +from workrb.tasks.abstract.base import ( + DatasetConfigNotSupported, + DatasetSplit, + Language, + TaskType, +) from workrb.types import DatasetLanguages @@ -233,6 +240,68 @@ def test_init_task_with_skip_on_unsupported_languages( ) +class TestEmptyDatasetWarnings: + """Tasks that resolve to zero datasets must warn instead of staying silent.""" + + @pytest.fixture(autouse=True) + def propagate_workrb_logs(self): + """Enable propagation on the 'workrb' logger for the duration of each test. + + The package logger disables propagation, which hides its records + from caplog's root handler. + """ + workrb_logger = logging.getLogger("workrb") + original = workrb_logger.propagate + workrb_logger.propagate = True + yield + workrb_logger.propagate = original + + def test_warns_when_no_dataset_matches_requested_languages(self, caplog): + """Warn when the language selection matches no dataset. + + A cross-lingual task requested with a single language selects no + datasets: each dataset needs both its query and target languages in + the requested set. + """ + with caplog.at_level(logging.WARNING, logger="workrb.tasks.abstract.base"): + task = SearchQueryCandidateRanking(split="test", languages=[Language.DE.value]) + + assert task.dataset_ids == [] + assert any( + "no datasets for the requested languages" in record.message for record in caplog.records + ) + + def test_warns_when_all_selected_datasets_are_skipped(self, caplog): + """Warn when every selected dataset raises DatasetConfigNotSupported. + + The task ends up with no datasets after loading and must say so. + """ + + class AllSkippedTask(StubSupportedLanguagesRankingTask): + def load_dataset(self, dataset_id, split): + raise DatasetConfigNotSupported(f"unsupported: {dataset_id}") + + with caplog.at_level(logging.WARNING, logger="workrb.tasks.abstract.base"): + task = AllSkippedTask(languages=[Language.EN.value], split="test") + + assert task.dataset_ids == [] + assert any( + "selected datasets were skipped during loading" in record.message + for record in caplog.records + ) + + def test_no_warning_when_datasets_load(self, caplog): + """A task with at least one loaded dataset must not emit either warning.""" + with caplog.at_level(logging.WARNING, logger="workrb.tasks.abstract.base"): + StubSupportedLanguagesRankingTask(languages=[Language.EN.value], split="test") + + assert not any( + "no datasets for the requested languages" in record.message + or "selected datasets were skipped during loading" in record.message + for record in caplog.records + ) + + class TestMELORankingDatasetIds: """Test MELORanking.languages_to_dataset_ids filtering logic."""