[TimesFM3 12/n] Prepare fev-bench data

benchmark/timeseries/fev_bench/evaluate.py

@@ -16,45 +16,27 @@
 METRICS = ("SQL", "MASE")
 
 
-def main() -> None:
-    benchmark_dir = Path(__file__).parent.parent
-    result_root = benchmark_dir / "fev_bench_out"
-    output_root = benchmark_dir / "evals" / "fev_bench"
-
-    summaries = []
-    for model_config in MODEL_CONFIGS.values():
-        path = result_root / model_config.name / "results.csv"
-        if path.exists():
-            summaries.append(pd.read_csv(path))
-
-    if not summaries:
-        raise FileNotFoundError(
-            f"No fev-bench results found under {result_root}"
-        )
-
-    # Compare with published results on tasks run by all models:
-    tasks = set.intersection(*(set(df["task_name"]) for df in summaries))
-    for name in REFERENCES:
-        path = f"{FEV_BENCH_URL}/benchmarks/fev_bench/results/{name}.csv"
-        summaries.append(pd.read_csv(path))
-    summaries = pd.concat(summaries)
-    summaries = summaries[summaries["task_name"].isin(tasks)]
-
-    output_root.mkdir(parents=True, exist_ok=True)
-    for metric in METRICS:
-        leaderboard = fev.leaderboard(
-            summaries,
-            metric_column=metric,
-            baseline_model=BASELINE,
-        )
-        leaderboard.to_csv(output_root / f"leaderboard_{metric}.csv")
-        print(f"\nfev-bench leaderboard ({metric}, {len(tasks)} tasks):")
-        print(
-            leaderboard[["skill_score", "win_rate"]].to_string(
-                float_format="%.4f"
-            )
-        )
-
-
-if __name__ == "__main__":
-    main()
+benchmark_dir = Path(__file__).parent.parent
+result_root = benchmark_dir / "fev_bench_out"
+output_root = benchmark_dir / "evals" / "fev_bench"
+
+dfs = []
+for config in MODEL_CONFIGS.values():
+    path = result_root / config.name / "results.csv"
+    if path.exists():
+        dfs.append(pd.read_csv(path))
+tasks = set.intersection(*(set(df["task_name"]) for df in dfs))
+
+for name in REFERENCES:
+    url = f"{FEV_BENCH_URL}/benchmarks/fev_bench/results/{name}.csv"
+    dfs.append(pd.read_csv(url))
+
+df = pd.concat(dfs)
+df = df[df["task_name"].isin(tasks)]
+
+for i, metric in enumerate(METRICS):
+    if i > 0:
+        print()
+    summary = fev.leaderboard(df, metric, baseline_model=BASELINE)
+    print(f"fev-bench leaderboard ({metric}, {len(tasks)} tasks):")
+    print(summary[["skill_score", "win_rate"]].to_string(float_format="%.4f"))

benchmark/timeseries/fev_bench/predictor.py

@@ -1,6 +1,7 @@
 # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 
+import zlib
 from collections.abc import Iterator
 
 import datasets
@@ -146,7 +147,7 @@ def _fit_predict(self, task: fev.Task) -> list[datasets.DatasetDict]:
 
                     pred = torch.cat(preds, dim=-2)[..., : y.size(-1), :]
                     pred = pred.cpu().numpy()  # [B, R, C, Q]
-                    levels = out.numerical_columns[: pred.shape[1]]
+                    levels = out.numerical_columns[: pred.shape[-1]]
                     levels = [float(f"0.{q.split('q')[-1]}") for q in levels]
                     wanted = [0.5, *task.quantile_levels]
                     index = [levels.index(q) for q in wanted]
@@ -174,11 +175,28 @@ def _from(
     columns: list[str],
     batch_size: int,
 ) -> Iterator[list[torch.Tensor]]:
+    string_columns = [
+        column
+        for column in columns
+        if dataset.features[column].feature.dtype in ("string", "large_string")
+    ]
+
     for batch in dataset.with_format("numpy").iter(batch_size):
         if not columns:
             yield []
             continue
 
+        for column in string_columns:
+            batch[column] = [
+                [
+                    zlib.crc32(value.encode()) % 100
+                    if isinstance(value, str)
+                    else 0
+                    for value in values
+                ]
+                for values in batch[column]
+            ]
+
         yield [
             torch.from_numpy(
                 np.asarray(