diff --git a/.gitignore b/.gitignore index f2c550828..b5763ae04 100644 --- a/.gitignore +++ b/.gitignore @@ -224,6 +224,7 @@ datasets.json # Time Series benchmarks gift_eval_out/ fev_bench_out/ +time_bench_out/ # Checkpoint files *.pt diff --git a/benchmark/README.md b/benchmark/README.md index 1af452860..f95a3988f 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -12,3 +12,4 @@ This folder contains benchmark integrations for `structured-data-models`: - [**`timeseries/`**](timeseries/): Time series benchmarks - [**`gift_eval/`**](timeseries/gift_eval/): [GIFT-Eval](https://github.com/SalesforceAIResearch/gift-eval) - [**`fev_bench/`**](timeseries/fev_bench/): [fev-bench](https://github.com/autogluon/fev) + - [**`time_bench/`**](timeseries/time_bench/): [TIME](https://github.com/zqiao11/TIME) diff --git a/benchmark/timeseries/README.md b/benchmark/timeseries/README.md index 1f2955fc8..77073eb84 100644 --- a/benchmark/timeseries/README.md +++ b/benchmark/timeseries/README.md @@ -1,3 +1,3 @@ # Time Series Benchmarks -This directory contains time series benchmarks of `structured-data-models` on [GIFT-Eval](gift_eval/), and [fev-bench](fev_bench/). +This directory contains time series benchmarks of `structured-data-models` on [GIFT-Eval](gift_eval/), [fev-bench](fev_bench/), and [TIME](time_bench/). diff --git a/benchmark/timeseries/gift_eval/main.py b/benchmark/timeseries/gift_eval/main.py index cc028a591..3f2fd8d2f 100644 --- a/benchmark/timeseries/gift_eval/main.py +++ b/benchmark/timeseries/gift_eval/main.py @@ -26,8 +26,8 @@ MED_LONG_DATASETS, PRETTY_NAMES, ) -from benchmark.timeseries.gift_eval.predictor import SDMPredictor from benchmark.timeseries.model import MODEL_CONFIGS +from benchmark.timeseries.predictor import SDMPredictor parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( diff --git a/benchmark/timeseries/gift_eval/predictor.py b/benchmark/timeseries/predictor.py similarity index 98% rename from benchmark/timeseries/gift_eval/predictor.py rename to benchmark/timeseries/predictor.py index 7be3448a4..7c774326b 100644 --- a/benchmark/timeseries/gift_eval/predictor.py +++ b/benchmark/timeseries/predictor.py @@ -12,8 +12,6 @@ import sdm from benchmark.timeseries.model import ModelConfig -QUANTILE_LEVELS = (0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9) - class SDMPredictor(Predictor): def __init__( diff --git a/benchmark/timeseries/time_bench/README.md b/benchmark/timeseries/time_bench/README.md new file mode 100644 index 000000000..52bf2a125 --- /dev/null +++ b/benchmark/timeseries/time_bench/README.md @@ -0,0 +1,39 @@ +# Time Series Forecasting Foundation Models on TIME + +This benchmark evaluates `structured-data-models` on the [TIME](https://github.com/zqiao11/TIME) benchmark. + +## Setup + +Run the commands below from the repository root: + +```bash +pip install . gluonts datasets python-dotenv scipy pyyaml +pip install --no-deps "git+https://github.com/zqiao11/TIME.git@3ca5c41d71c76f3c70632c0121142d325350ee32" +``` + +## Run + +- **`TimesFM3`**: + + ```bash + python -m benchmark.timeseries.time_bench.main --model timesfm3 + ``` + + `TimesFM3` follows official evaluation with symmetric averaging over two estimators, contexts truncated to 15,360 time steps and up to 32 variates per forward pass. + Unlike the official `float32` evaluation, we use `bfloat16` mixed precision for improved efficiency with negligible differences in aggregate benchmark accuracy. + +> [!NOTE] +> Weights of `TimesFM3` are distributed under the [TimesFM Non-Commercial License v1.0](https://huggingface.co/google/timesfm-3.0-pytorch/blob/main/LICENSE). +> Review the license before running the `TimesFM3` benchmark, which will download its weights noninteractively. + +Results are written to `benchmark/timeseries/time_bench_out//`, and already completed configurations are skipped. + +Pass `--dataset` to run only selected TIME datasets, *e.g.*, `--dataset Job_Claims/W --dataset CPHL/15T`. + +## Evaluate + +Evaluate all available model results with: + +```bash +python -m benchmark.timeseries.time_bench.evaluate +``` diff --git a/benchmark/timeseries/time_bench/__init__.py b/benchmark/timeseries/time_bench/__init__.py new file mode 100644 index 000000000..d51c4fe1e --- /dev/null +++ b/benchmark/timeseries/time_bench/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/benchmark/timeseries/time_bench/evaluate.py b/benchmark/timeseries/time_bench/evaluate.py new file mode 100644 index 000000000..e7943cf59 --- /dev/null +++ b/benchmark/timeseries/time_bench/evaluate.py @@ -0,0 +1,68 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +r"""Evaluate SDM time series forecasting results on TIME.""" + +import gc +from pathlib import Path + +import numpy as np +import pandas as pd +from huggingface_hub import snapshot_download + +from benchmark.timeseries.model import MODEL_CONFIGS + +BASELINE = "seasonal_naive" +REFERENCES = ("TimesFM-3", "TimesFM-2.5", "Chronos-2", BASELINE) +METRICS = ("MASE", "CRPS") + +runs = {} +for config in MODEL_CONFIGS.values(): + path = Path(__file__).parent.parent / "time_bench_out" / config.name + if any(path.glob("**/metrics.npz")): + runs[f"[SDM] {config.name}"] = path + +configs = set.intersection( + *( + {path.parent.relative_to(run) for path in run.rglob("metrics.npz")} + for run in runs.values() + ) +) + +path = snapshot_download( + repo_id="Real-TSF/TIME-Output", + repo_type="dataset", + allow_patterns=[ + f"results/{name}/{config.as_posix()}/metrics.npz" + for name in REFERENCES + for config in sorted(configs) + ], +) +gc.collect() +for name in REFERENCES: + runs[name] = Path(path) / "results" / name + +rows = [] +for model, run in runs.items(): + for config in sorted(configs): + with np.load(run / config / "metrics.npz") as metrics: + rows.append( + { + "model": model, + "dataset": config.as_posix(), + **{ + metric: np.nanmean(metrics[metric]) + for metric in METRICS + }, + } + ) + +df = pd.DataFrame(rows).set_index(["model", "dataset"]) +scores = np.log(df.div(df.loc[BASELINE], level="dataset")) +terms = scores.index.get_level_values("dataset").str.split("/").str[-1] +for i, (group, score) in enumerate([("all", scores), *scores.groupby(terms)]): + if i > 0: + print() + summary = np.exp(score.groupby("model").mean()).sort_values("CRPS") + print(f"TIME leaderboard ({group}):") + print(summary.reset_index().to_string(index=False, float_format="%.4f")) diff --git a/benchmark/timeseries/time_bench/main.py b/benchmark/timeseries/time_bench/main.py new file mode 100644 index 000000000..6a80b776d --- /dev/null +++ b/benchmark/timeseries/time_bench/main.py @@ -0,0 +1,98 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +r"""Run an SDM time series forecasting model on TIME.""" + +import argparse +import os +import warnings +from contextlib import redirect_stdout +from pathlib import Path + +import numpy as np +import torch +import tqdm +from gluonts.time_feature import get_seasonality +from huggingface_hub import snapshot_download +from timebench.evaluation.data import ( + Dataset, + get_dataset_settings, + load_dataset_config, +) +from timebench.evaluation.saver import save_window_predictions +from timebench.evaluation.utils import get_available_terms + +from benchmark.timeseries.model import MODEL_CONFIGS +from benchmark.timeseries.predictor import SDMPredictor + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument( + "--model", + choices=tuple(MODEL_CONFIGS), + default="timesfm3", + help="SDM model to benchmark.", +) +parser.add_argument( + "--dataset", + action="append", + help="Run only the selected TIME dataset; repeat to combine.", +) +args = parser.parse_args() + +warnings.filterwarnings("ignore", message=r"'.*' is deprecated.*") + +config = MODEL_CONFIGS[args.model] +dataset_config = load_dataset_config() +path = Path(__file__).parent.parent / "time_bench_out" / config.name + +names = args.dataset or dataset_config["datasets"] +if not os.getenv("TIME_DATASET"): + os.environ["TIME_DATASET"] = snapshot_download( + repo_id="Real-TSF/TIME", + repo_type="dataset", + allow_patterns=[f"{name}/*" for name in names], + ) + + +device = torch.device("cuda" if torch.cuda.is_available() else "cpu") +model = config.factory(device) + +for name in names: + for term in get_available_terms(name, dataset_config): + if (path / name / term / "metrics.npz").exists(): + continue + + settings = get_dataset_settings(name, term, dataset_config) + dataset = Dataset( + name=name, + term=term, + prediction_length=settings["prediction_length"], + test_length=settings["test_length"], + ) + + predictor = SDMPredictor(model, config, dataset.prediction_length) + outs = list( + tqdm.tqdm( + predictor.predict(dataset.test_data.input), + total=len(dataset.test_data), + desc=f"{name}/{term}", + ) + ) + + quantiles = np.stack([out.forecast_array for out in outs]) + if quantiles.ndim == 4: + quantiles = quantiles.swapaxes(-1, -2) # [B, Q, C, R] + + with redirect_stdout(None): + save_window_predictions( + dataset=dataset, + fc_quantiles=quantiles, + ds_config=f"{name}/{term}", + output_base_dir=str(path), + seasonality=get_seasonality(dataset.freq), + quantile_levels=[float(q) for q in outs[0].forecast_keys], + ) + with np.load(path / name / term / "metrics.npz") as metrics: + mase = np.nanmean(metrics["MASE"]) + crps = np.nanmean(metrics["CRPS"]) + print(f"{name}/{term}: MASE={mase:.4f}, CRPS={crps:.4f}")