diff --git a/benchmark/timeseries/fev_bench/evaluate.py b/benchmark/timeseries/fev_bench/evaluate.py index 70ed2e6cb6..2edd45e6ab 100644 --- a/benchmark/timeseries/fev_bench/evaluate.py +++ b/benchmark/timeseries/fev_bench/evaluate.py @@ -16,45 +16,27 @@ METRICS = ("SQL", "MASE") -def main() -> None: - benchmark_dir = Path(__file__).parent.parent - result_root = benchmark_dir / "fev_bench_out" - output_root = benchmark_dir / "evals" / "fev_bench" - - summaries = [] - for model_config in MODEL_CONFIGS.values(): - path = result_root / model_config.name / "results.csv" - if path.exists(): - summaries.append(pd.read_csv(path)) - - if not summaries: - raise FileNotFoundError( - f"No fev-bench results found under {result_root}" - ) - - # Compare with published results on tasks run by all models: - tasks = set.intersection(*(set(df["task_name"]) for df in summaries)) - for name in REFERENCES: - path = f"{FEV_BENCH_URL}/benchmarks/fev_bench/results/{name}.csv" - summaries.append(pd.read_csv(path)) - summaries = pd.concat(summaries) - summaries = summaries[summaries["task_name"].isin(tasks)] - - output_root.mkdir(parents=True, exist_ok=True) - for metric in METRICS: - leaderboard = fev.leaderboard( - summaries, - metric_column=metric, - baseline_model=BASELINE, - ) - leaderboard.to_csv(output_root / f"leaderboard_{metric}.csv") - print(f"\nfev-bench leaderboard ({metric}, {len(tasks)} tasks):") - print( - leaderboard[["skill_score", "win_rate"]].to_string( - float_format="%.4f" - ) - ) - - -if __name__ == "__main__": - main() +benchmark_dir = Path(__file__).parent.parent +result_root = benchmark_dir / "fev_bench_out" +output_root = benchmark_dir / "evals" / "fev_bench" + +dfs = [] +for config in MODEL_CONFIGS.values(): + path = result_root / config.name / "results.csv" + if path.exists(): + dfs.append(pd.read_csv(path)) +tasks = set.intersection(*(set(df["task_name"]) for df in dfs)) + +for name in REFERENCES: + url = f"{FEV_BENCH_URL}/benchmarks/fev_bench/results/{name}.csv" + dfs.append(pd.read_csv(url)) + +df = pd.concat(dfs) +df = df[df["task_name"].isin(tasks)] + +for i, metric in enumerate(METRICS): + if i > 0: + print() + summary = fev.leaderboard(df, metric, baseline_model=BASELINE) + print(f"fev-bench leaderboard ({metric}, {len(tasks)} tasks):") + print(summary[["skill_score", "win_rate"]].to_string(float_format="%.4f")) diff --git a/benchmark/timeseries/fev_bench/predictor.py b/benchmark/timeseries/fev_bench/predictor.py index bd5a48983f..ce59e8644b 100644 --- a/benchmark/timeseries/fev_bench/predictor.py +++ b/benchmark/timeseries/fev_bench/predictor.py @@ -1,6 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import zlib from collections.abc import Iterator import datasets @@ -146,7 +147,7 @@ def _fit_predict(self, task: fev.Task) -> list[datasets.DatasetDict]: pred = torch.cat(preds, dim=-2)[..., : y.size(-1), :] pred = pred.cpu().numpy() # [B, R, C, Q] - levels = out.numerical_columns[: pred.shape[1]] + levels = out.numerical_columns[: pred.shape[-1]] levels = [float(f"0.{q.split('q')[-1]}") for q in levels] wanted = [0.5, *task.quantile_levels] index = [levels.index(q) for q in wanted] @@ -174,11 +175,28 @@ def _from( columns: list[str], batch_size: int, ) -> Iterator[list[torch.Tensor]]: + string_columns = [ + column + for column in columns + if dataset.features[column].feature.dtype in ("string", "large_string") + ] + for batch in dataset.with_format("numpy").iter(batch_size): if not columns: yield [] continue + for column in string_columns: + batch[column] = [ + [ + zlib.crc32(value.encode()) % 100 + if isinstance(value, str) + else 0 + for value in values + ] + for values in batch[column] + ] + yield [ torch.from_numpy( np.asarray(