Add fev-bench forecasting benchmark

.gitignore

@@ -223,6 +223,7 @@ datasets.json
 
 # Time Series benchmarks
 gift_eval_out/
+fev_bench_out/
 
 # Checkpoint files
 *.pt

benchmark/README.md

@@ -11,3 +11,4 @@ This folder contains benchmark integrations for `structured-data-models`:
   - [**`relarena/`**](relational/relarena): [RelArena](https://github.com/PriorLabs/relarena)
 - [**`timeseries/`**](timeseries/): Time series benchmarks
   - [**`gift_eval/`**](timeseries/gift_eval/): [GIFT-Eval](https://github.com/SalesforceAIResearch/gift-eval)
+  - [**`fev_bench/`**](timeseries/fev_bench/): [fev-bench](https://github.com/autogluon/fev)

benchmark/timeseries/README.md

@@ -1,3 +1,3 @@
 # Time Series Benchmarks
 
-This directory contains time series benchmarks of `structured-data-models` on [GIFT-Eval](gift_eval/).
+This directory contains time series benchmarks of `structured-data-models` on [GIFT-Eval](gift_eval/), and [fev-bench](fev_bench/).

benchmark/timeseries/fev_bench/README.md

@@ -0,0 +1,38 @@
+# Time Series Forecasting Foundation Models on fev-bench
+
+This benchmark evaluates `structured-data-models` on the [fev-bench](https://github.com/autogluon/fev) benchmark.
+
+## Setup
+
+Run the command below from the repository root:
+
+```bash
+pip install . fev==0.10.0
+```
+
+## Run
+
+- **`TimesFM3`**:
+
+  ```bash
+  python -m benchmark.timeseries.fev_bench.main --model timesfm3
+  ```
+
+  `TimesFM3` follows official evaluation with symmetric averaging over two estimators, contexts truncated to 15,360 time steps and up to 32 variates per forward pass.
+
+> [!NOTE]
+> Weights of `TimesFM3` are distributed under the [TimesFM Non-Commercial License v1.0](https://huggingface.co/google/timesfm-3.0-pytorch/blob/main/LICENSE).
+> Review the license before running the `TimesFM3` benchmark, which will download its weights noninteractively.
+
+Results will be written to `benchmark/timeseries/fev_bench_out/<model>/`, and already completed configurations are skipped.
+
+Pass `--task` to run only selected fev-bench tasks, *e.g.*, `--task hospital --task proenfo_gfc12`.
+Pass `--mini` to run the 20-task fev-bench-mini subset:
+
+## Evaluate
+
+Evaluate all available model results with:
+
+```bash
+python -m benchmark.timeseries.fev_bench.evaluate
+```

benchmark/timeseries/fev_bench/__init__.py

@@ -0,0 +1,4 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+FEV_BENCH_URL = "https://raw.githubusercontent.com/autogluon/fev/v0.10.0"

benchmark/timeseries/fev_bench/evaluate.py

@@ -0,0 +1,60 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+r"""Evaluate SDM time series forecasting results on fev-bench."""
+
+from pathlib import Path
+
+import fev
+import pandas as pd
+
+from benchmark.timeseries.fev_bench import FEV_BENCH_URL
+from benchmark.timeseries.model import MODEL_CONFIGS
+
+BASELINE = "Seasonal Naive"
+REFERENCES = ("timesfm-3", "timesfm-2_5", "chronos-2", "seasonal_naive")
+METRICS = ("SQL", "MASE")
+
+
+def main() -> None:
+    benchmark_dir = Path(__file__).parent.parent
+    result_root = benchmark_dir / "fev_bench_out"
+    output_root = benchmark_dir / "evals" / "fev_bench"
+
+    summaries = []
+    for model_config in MODEL_CONFIGS.values():
+        path = result_root / model_config.name / "results.csv"
+        if path.exists():
+            summaries.append(pd.read_csv(path))
+
+    if not summaries:
+        raise FileNotFoundError(
+            f"No fev-bench results found under {result_root}"
+        )
+
+    # Compare with published results on tasks run by all models:
+    tasks = set.intersection(*(set(df["task_name"]) for df in summaries))
+    for name in REFERENCES:
+        path = f"{FEV_BENCH_URL}/benchmarks/fev_bench/results/{name}.csv"
+        summaries.append(pd.read_csv(path))
+    summaries = pd.concat(summaries)
+    summaries = summaries[summaries["task_name"].isin(tasks)]
+
+    output_root.mkdir(parents=True, exist_ok=True)
+    for metric in METRICS:
+        leaderboard = fev.leaderboard(
+            summaries,
+            metric_column=metric,
+            baseline_model=BASELINE,
+        )
+        leaderboard.to_csv(output_root / f"leaderboard_{metric}.csv")
+        print(f"\nfev-bench leaderboard ({metric}, {len(tasks)} tasks):")
+        print(
+            leaderboard[["skill_score", "win_rate"]].to_string(
+                float_format="%.4f"
+            )
+        )
+
+
+if __name__ == "__main__":
+    main()

benchmark/timeseries/fev_bench/main.py

@@ -0,0 +1,70 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+r"""Run an SDM time series forecasting model on fev-bench."""
+
+import argparse
+from pathlib import Path
+
+import fev
+import pandas as pd
+import torch
+
+from benchmark.timeseries.fev_bench import FEV_BENCH_URL
+from benchmark.timeseries.fev_bench.predictor import SDMPredictor
+from benchmark.timeseries.model import MODEL_CONFIGS
+
+parser = argparse.ArgumentParser(description=__doc__)
+parser.add_argument(
+    "--model",
+    choices=tuple(MODEL_CONFIGS),
+    default="timesfm3",
+    help="SDM model to benchmark.",
+)
+parser.add_argument(
+    "--mini",
+    action="store_true",
+    help="Run the 20-task fev-bench-mini subset.",
+)
+parser.add_argument(
+    "--task",
+    action="append",
+    help="Run only the selected fev-bench task; repeat to combine.",
+)
+args = parser.parse_args()
+
+config = MODEL_CONFIGS[args.model]
+path = Path(__file__).parent.parent
+path = path / "fev_bench_out" / config.name / "results.csv"
+path.parent.mkdir(parents=True, exist_ok=True)
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+model = SDMPredictor(
+    model=config.factory(device),
+    config=config,
+)
+
+completed = set(pd.read_csv(path)["task_name"]) if path.exists() else set()
+filename = "tasks_mini.yaml" if args.mini else "tasks.yaml"
+benchmark = fev.Benchmark.from_yaml(
+    f"{FEV_BENCH_URL}/benchmarks/fev_bench/{filename}"
+)
+for task in benchmark.tasks:
+    if args.task is not None and task.task_name not in args.task:
+        continue
+    if task.task_name in completed:
+        continue
+
+    summary = task.evaluation_summary(
+        model.fit_predict(task),
+        model_name=f"[SDM] {config.name}",
+        training_time_s=model.training_time,
+        inference_time_s=model.inference_time,
+    )
+    print(
+        f"{task.task_name}: Error={summary['test_error']:6f}, "
+        f"Time={model.inference_time:.2f}s"
+    )
+    row = pd.DataFrame([summary])
+    row.to_csv(path, mode="a", header=not path.exists(), index=False)
+    completed.add(task.task_name)

benchmark/timeseries/fev_bench/predictor.py

@@ -0,0 +1,190 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+from collections.abc import Iterator
+
+import datasets
+import fev
+import numpy as np
+import torch
+import tqdm
+
+import sdm
+from benchmark.timeseries.model import ModelConfig
+
+
+class SDMPredictor(fev.ForecastingModel):
+    def __init__(
+        self,
+        model: sdm.models.ICLModel,
+        config: ModelConfig,
+        batch_size: int = 64,
+    ) -> None:
+        super().__init__()
+        self.model = model
+        self.config = config
+        self.batch_size = batch_size
+        self.device = next(model.parameters()).device
+
+    def _fit_predict(self, task: fev.Task) -> list[datasets.DatasetDict]:
+        predictions = []
+        for window in tqdm.tqdm(
+            task.iter_windows(),
+            total=task.num_windows,
+            desc=task.task_name,
+        ):
+            past, future = window.get_input_data()
+
+            all_columns = task.known_dynamic_columns.copy()
+            if self.model.supports_context_only_columns:
+                all_columns += task.past_dynamic_columns
+
+            values = []
+            with self._record_inference_time():
+                for ys, xs_context, xs_query in zip(
+                    _from(past, task.target_columns, self.batch_size),
+                    _from(past, all_columns, self.batch_size),
+                    _from(future, task.known_dynamic_columns, self.batch_size),
+                    strict=True,
+                ):
+                    generator = torch.Generator().manual_seed(42)
+
+                    if self.config.max_context_size is not None:
+                        length = self.config.max_context_size
+                        ys = [y[..., -length:] for y in ys]
+                        xs_context = [x[..., -length:] for x in xs_context]
+
+                    length = max(y.size(-1) for y in ys)
+                    y = torch.full(
+                        (len(ys), ys[0].size(0), length),
+                        fill_value=torch.nan,
+                    )
+                    for i, y_i in enumerate(ys):
+                        y[i, :, -y_i.size(-1) :] = y_i
+                    x_context = torch.full(
+                        (
+                            len(ys),
+                            xs_context[0].size(0)
+                            if len(xs_context) > 0
+                            else 0,
+                            length,
+                        ),
+                        fill_value=torch.nan,
+                    )
+                    for i, x_context_i in enumerate(xs_context):
+                        x_context[i, :, -x_context_i.size(-1) :] = x_context_i
+                    x_query = torch.empty(
+                        len(ys),
+                        xs_query[0].size(0) if len(xs_query) > 0 else 0,
+                        task.horizon,
+                    )
+                    for i, x_query_i in enumerate(xs_query):
+                        x_query[i, :, :] = x_query_i
+
+                    if (  # Keep room for at least one target:
+                        self.config.max_variates is not None
+                        and x_query.size(1) >= self.config.max_variates
+                    ):
+                        perm = torch.randperm(
+                            x_query.size(1), generator=generator
+                        )[: self.config.max_variates - 1]
+                        x_context = x_context[:, perm]
+                        x_query = x_query[:, perm]
+                    elif (
+                        self.config.max_variates is not None
+                        and x_context.size(1) >= self.config.max_variates
+                    ):
+                        perm = torch.randperm(
+                            x_context.size(1) - x_query.size(1),
+                            generator=generator,
+                        )[: self.config.max_variates - 1 - x_query.size(1)]
+                        x_context = torch.cat(
+                            [
+                                x_context[:, : x_query.size(1)],
+                                x_context[:, perm + x_query.size(1)],
+                            ],
+                            dim=1,
+                        )
+
+                    y = y.transpose(-1, -2).to(
+                        self.device,
+                        memory_format=torch.contiguous_format,
+                    )
+                    x_context = x_context.transpose(-1, -2).to(
+                        self.device,
+                        memory_format=torch.contiguous_format,
+                    )
+                    x_query = x_query.transpose(-1, -2).to(
+                        self.device,
+                        memory_format=torch.contiguous_format,
+                    )
+
+                    size = y.size(-1)
+                    if self.config.max_variates is not None:
+                        size = min(
+                            size,
+                            self.config.max_variates - x_context.size(-1),
+                        )
+
+                    preds = []
+                    for start in range(0, y.size(-1), size):
+                        y_i = y[..., start : start + size]
+                        if y_i.size(-1) < size:
+                            y_i = torch.cat(
+                                [y_i, y[..., : size - y_i.size(-1)]], dim=-1
+                            )
+
+                        out = self.model(
+                            x_context=sdm.TableTensor.from_tensor(x_context),
+                            y_context=sdm.TableTensor.from_tensor(y_i),
+                            x_query=sdm.TableTensor.from_tensor(x_query),
+                            num_estimators=self.config.num_estimators,
+                        )
+                        pred = out.numerical
+                        pred = pred.view(y.size(0), task.horizon, size, -1)
+                        preds.append(pred)
+
+                    pred = torch.cat(preds, dim=-2)[..., : y.size(-1), :]
+                    pred = pred.cpu().numpy()  # [B, R, C, Q]
+                    levels = out.numerical_columns[: pred.shape[1]]
+                    levels = [float(f"0.{q.split('q')[-1]}") for q in levels]
+                    wanted = [0.5, *task.quantile_levels]
+                    index = [levels.index(q) for q in wanted]
+                    values.append(pred[..., index])
+
+            values = np.concatenate(values)  # [B, R, C, 1 + Q]
+            window_predictions = datasets.DatasetDict()
+            for i, column in enumerate(task.target_columns):
+                window_predictions[column] = datasets.Dataset.from_dict(
+                    {
+                        "predictions": values[:, :, i, 0],
+                        **{
+                            str(level): values[:, :, i, j + 1]
+                            for j, level in enumerate(task.quantile_levels)
+                        },
+                    }
+                )
+            predictions.append(window_predictions)
+
+        return predictions
+
+
+def _from(
+    dataset: datasets.Dataset,
+    columns: list[str],
+    batch_size: int,
+) -> Iterator[list[torch.Tensor]]:
+    for batch in dataset.with_format("numpy").iter(batch_size):
+        if not columns:
+            yield []
+            continue
+
+        yield [
+            torch.from_numpy(
+                np.asarray(
+                    [batch[column][i] for column in columns],
+                    dtype=np.float32,
+                )
+            )
+            for i in range(len(batch[columns[0]]))
+        ]

benchmark/timeseries/gift_eval/README.md

@@ -27,6 +27,8 @@ pip install --no-deps "git+https://github.com/SalesforceAIResearch/gift-eval.git
 
 Results will be written to `benchmark/timeseries/gift_eval_out/<model>/`, and already completed configurations are skipped.
 
+Pass `--dataset` to run only selected GIFT-Eval datasets, *e.g.*, `--dataset hospital --dataset ett1/H`.
+
 ## Evaluate
 
 Evaluate all available model results with: