Add TIME forecasting benchmark

.gitignore

@@ -224,6 +224,7 @@ datasets.json
 # Time Series benchmarks
 gift_eval_out/
 fev_bench_out/
+time_bench_out/
 
 # Checkpoint files
 *.pt

benchmark/README.md

@@ -12,3 +12,4 @@ This folder contains benchmark integrations for `structured-data-models`:
 - [**`timeseries/`**](timeseries/): Time series benchmarks
   - [**`gift_eval/`**](timeseries/gift_eval/): [GIFT-Eval](https://github.com/SalesforceAIResearch/gift-eval)
   - [**`fev_bench/`**](timeseries/fev_bench/): [fev-bench](https://github.com/autogluon/fev)
+  - [**`time_bench/`**](timeseries/time_bench/): [TIME](https://github.com/zqiao11/TIME)

benchmark/timeseries/README.md

@@ -1,3 +1,3 @@
 # Time Series Benchmarks
 
-This directory contains time series benchmarks of `structured-data-models` on [GIFT-Eval](gift_eval/), and [fev-bench](fev_bench/).
+This directory contains time series benchmarks of `structured-data-models` on [GIFT-Eval](gift_eval/), [fev-bench](fev_bench/), and [TIME](time_bench/).

benchmark/timeseries/gift_eval/main.py

@@ -26,8 +26,8 @@
     MED_LONG_DATASETS,
     PRETTY_NAMES,
 )
-from benchmark.timeseries.gift_eval.predictor import SDMPredictor
 from benchmark.timeseries.model import MODEL_CONFIGS
+from benchmark.timeseries.predictor import SDMPredictor
 
 parser = argparse.ArgumentParser(description=__doc__)
 parser.add_argument(

benchmark/timeseries/gift_eval/predictor.py → benchmark/timeseries/predictor.py

@@ -12,8 +12,6 @@
 import sdm
 from benchmark.timeseries.model import ModelConfig
 
-QUANTILE_LEVELS = (0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9)
-
 
 class SDMPredictor(Predictor):
     def __init__(

benchmark/timeseries/time_bench/README.md

@@ -0,0 +1,39 @@
+# Time Series Forecasting Foundation Models on TIME
+
+This benchmark evaluates `structured-data-models` on the [TIME](https://github.com/zqiao11/TIME) benchmark.
+
+## Setup
+
+Run the commands below from the repository root:
+
+```bash
+pip install . gluonts datasets python-dotenv scipy pyyaml
+pip install --no-deps "git+https://github.com/zqiao11/TIME.git@3ca5c41d71c76f3c70632c0121142d325350ee32"
+```
+
+## Run
+
+- **`TimesFM3`**:
+
+  ```bash
+  python -m benchmark.timeseries.time_bench.main --model timesfm3
+  ```
+
+  `TimesFM3` follows official evaluation with symmetric averaging over two estimators, contexts truncated to 15,360 time steps and up to 32 variates per forward pass.
+  Unlike the official `float32` evaluation, we use `bfloat16` mixed precision for improved efficiency with negligible differences in aggregate benchmark accuracy.
+
+> [!NOTE]
+> Weights of `TimesFM3` are distributed under the [TimesFM Non-Commercial License v1.0](https://huggingface.co/google/timesfm-3.0-pytorch/blob/main/LICENSE).
+> Review the license before running the `TimesFM3` benchmark, which will download its weights noninteractively.
+
+Results are written to `benchmark/timeseries/time_bench_out/<model>/`, and already completed configurations are skipped.
+
+Pass `--dataset` to run only selected TIME datasets, *e.g.*, `--dataset Job_Claims/W --dataset CPHL/15T`.
+
+## Evaluate
+
+Evaluate all available model results with:
+
+```bash
+python -m benchmark.timeseries.time_bench.evaluate
+```

benchmark/timeseries/time_bench/__init__.py

@@ -0,0 +1,2 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0

benchmark/timeseries/time_bench/evaluate.py

@@ -0,0 +1,68 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+r"""Evaluate SDM time series forecasting results on TIME."""
+
+import gc
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+from huggingface_hub import snapshot_download
+
+from benchmark.timeseries.model import MODEL_CONFIGS
+
+BASELINE = "seasonal_naive"
+REFERENCES = ("TimesFM-3", "TimesFM-2.5", "Chronos-2", BASELINE)
+METRICS = ("MASE", "CRPS")
+
+runs = {}
+for config in MODEL_CONFIGS.values():
+    path = Path(__file__).parent.parent / "time_bench_out" / config.name
+    if any(path.glob("**/metrics.npz")):
+        runs[f"[SDM] {config.name}"] = path
+
+configs = set.intersection(
+    *(
+        {path.parent.relative_to(run) for path in run.rglob("metrics.npz")}
+        for run in runs.values()
+    )
+)
+
+path = snapshot_download(
+    repo_id="Real-TSF/TIME-Output",
+    repo_type="dataset",
+    allow_patterns=[
+        f"results/{name}/{config.as_posix()}/metrics.npz"
+        for name in REFERENCES
+        for config in sorted(configs)
+    ],
+)
+gc.collect()
+for name in REFERENCES:
+    runs[name] = Path(path) / "results" / name
+
+rows = []
+for model, run in runs.items():
+    for config in sorted(configs):
+        with np.load(run / config / "metrics.npz") as metrics:
+            rows.append(
+                {
+                    "model": model,
+                    "dataset": config.as_posix(),
+                    **{
+                        metric: np.nanmean(metrics[metric])
+                        for metric in METRICS
+                    },
+                }
+            )
+
+df = pd.DataFrame(rows).set_index(["model", "dataset"])
+scores = np.log(df.div(df.loc[BASELINE], level="dataset"))
+terms = scores.index.get_level_values("dataset").str.split("/").str[-1]
+for i, (group, score) in enumerate([("all", scores), *scores.groupby(terms)]):
+    if i > 0:
+        print()
+    summary = np.exp(score.groupby("model").mean()).sort_values("CRPS")
+    print(f"TIME leaderboard ({group}):")
+    print(summary.reset_index().to_string(index=False, float_format="%.4f"))

benchmark/timeseries/time_bench/main.py

@@ -0,0 +1,98 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+r"""Run an SDM time series forecasting model on TIME."""
+
+import argparse
+import os
+import warnings
+from contextlib import redirect_stdout
+from pathlib import Path
+
+import numpy as np
+import torch
+import tqdm
+from gluonts.time_feature import get_seasonality
+from huggingface_hub import snapshot_download
+from timebench.evaluation.data import (
+    Dataset,
+    get_dataset_settings,
+    load_dataset_config,
+)
+from timebench.evaluation.saver import save_window_predictions
+from timebench.evaluation.utils import get_available_terms
+
+from benchmark.timeseries.model import MODEL_CONFIGS
+from benchmark.timeseries.predictor import SDMPredictor
+
+parser = argparse.ArgumentParser(description=__doc__)
+parser.add_argument(
+    "--model",
+    choices=tuple(MODEL_CONFIGS),
+    default="timesfm3",
+    help="SDM model to benchmark.",
+)
+parser.add_argument(
+    "--dataset",
+    action="append",
+    help="Run only the selected TIME dataset; repeat to combine.",
+)
+args = parser.parse_args()
+
+warnings.filterwarnings("ignore", message=r"'.*' is deprecated.*")
+
+config = MODEL_CONFIGS[args.model]
+dataset_config = load_dataset_config()
+path = Path(__file__).parent.parent / "time_bench_out" / config.name
+
+names = args.dataset or dataset_config["datasets"]
+if not os.getenv("TIME_DATASET"):
+    os.environ["TIME_DATASET"] = snapshot_download(
+        repo_id="Real-TSF/TIME",
+        repo_type="dataset",
+        allow_patterns=[f"{name}/*" for name in names],
+    )
+
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+model = config.factory(device)
+
+for name in names:
+    for term in get_available_terms(name, dataset_config):
+        if (path / name / term / "metrics.npz").exists():
+            continue
+
+        settings = get_dataset_settings(name, term, dataset_config)
+        dataset = Dataset(
+            name=name,
+            term=term,
+            prediction_length=settings["prediction_length"],
+            test_length=settings["test_length"],
+        )
+
+        predictor = SDMPredictor(model, config, dataset.prediction_length)
+        outs = list(
+            tqdm.tqdm(
+                predictor.predict(dataset.test_data.input),
+                total=len(dataset.test_data),
+                desc=f"{name}/{term}",
+            )
+        )
+
+        quantiles = np.stack([out.forecast_array for out in outs])
+        if quantiles.ndim == 4:
+            quantiles = quantiles.swapaxes(-1, -2)  # [B, Q, C, R]
+
+        with redirect_stdout(None):
+            save_window_predictions(
+                dataset=dataset,
+                fc_quantiles=quantiles,
+                ds_config=f"{name}/{term}",
+                output_base_dir=str(path),
+                seasonality=get_seasonality(dataset.freq),
+                quantile_levels=[float(q) for q in outs[0].forecast_keys],
+            )
+        with np.load(path / name / term / "metrics.npz") as metrics:
+            mase = np.nanmean(metrics["MASE"])
+            crps = np.nanmean(metrics["CRPS"])
+        print(f"{name}/{term}: MASE={mase:.4f}, CRPS={crps:.4f}")