From 8c25ce03d405f62179966c603c9a5449104eca51 Mon Sep 17 00:00:00 2001 From: Filippo Guerranti Date: Tue, 13 Oct 2026 09:02:43 +0000 Subject: [PATCH 1/7] Add `sdm.evaluation.to_quantiles` --- docs/source/icl.md | 1 + sdm/evaluation/__init__.py | 2 + sdm/evaluation/regression.py | 61 ++++++++++++++++++++++++++++++ test/evaluation/test_regression.py | 26 +++++++++++++ 4 files changed, 90 insertions(+) create mode 100644 sdm/evaluation/regression.py create mode 100644 test/evaluation/test_regression.py diff --git a/docs/source/icl.md b/docs/source/icl.md index c8174aee44..d8e19fd700 100644 --- a/docs/source/icl.md +++ b/docs/source/icl.md @@ -98,6 +98,7 @@ Predictions are returned as a general {py:class}`~sdm.tensor.TableTensor`, where Since predictions are returned as {py:class}`~sdm.tensor.TableTensor`, you can zero-copy them to [`pandas`](https://pandas.pydata.org/docs), [`arrow`](https://arrow.apache.org/docs), or [`cudf`](https://docs.rapids.ai/api/cudf) via {py:meth}`~sdm.tensor.TableTensor.to_pandas`, {py:meth}`~sdm.tensor.TableTensor.to_arrow`, and {py:meth}`~sdm.tensor.TableTensor.to_cudf` for further downstream processing. In order to simplify metric calculation (*e.g.*, via [`torchmetrics`](https://lightning.ai/docs/torchmetrics)), we provide helper functions in the [`sdm.evaluation`](api/evaluation) package to convert target columns to class indices and align prediction columns to them (see {py:func}`~sdm.evaluation.to_class_indices` and {py:func}`~sdm.evaluation.to_binary_class`). +Similarly, {py:func}`~sdm.evaluation.to_quantiles` converts quantile predictions of one or more targets into a single tensor together with their quantile levels. ## Ensembling diff --git a/sdm/evaluation/__init__.py b/sdm/evaluation/__init__.py index 7d3b78251e..08304db084 100644 --- a/sdm/evaluation/__init__.py +++ b/sdm/evaluation/__init__.py @@ -4,8 +4,10 @@ """Evaluation utilities.""" from sdm.evaluation.classification import to_class_indices, to_binary_class +from sdm.evaluation.regression import to_quantiles __all__ = [ "to_class_indices", "to_binary_class", + "to_quantiles", ] diff --git a/sdm/evaluation/regression.py b/sdm/evaluation/regression.py new file mode 100644 index 0000000000..8ff24bc101 --- /dev/null +++ b/sdm/evaluation/regression.py @@ -0,0 +1,61 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from torch import Tensor + +from sdm import Stype, TableTensor + + +def to_quantiles(pred: TableTensor) -> tuple[Tensor, Tensor]: + r"""Convert quantile predictions to a per-target quantile tensor. + + Numerical columns must be named ``"q"`` for a single target or + ``"__q"`` for multiple targets, where the digits encode the + quantile level as a decimal fraction, *e.g.*, ``"q500"`` and ``"q50"`` both + denote the median. + Columns are expected to be grouped by target, with the same quantile levels + in the same order for every target. + + Args: + pred: Prediction table whose numerical columns contain quantiles. + + Returns: + The quantile predictions with shape ``[..., V, Q]`` for ``V`` targets + and ``Q`` quantile levels, and the quantile levels with shape ``[Q]``. + + >>> import torch + >>> import sdm + >>> from sdm.evaluation import to_quantiles + >>> pred = sdm.TableTensor.from_tensor( + ... torch.tensor([[1.0, 2.0, 3.0, 4.0]]), + ... columns=["q100", "q400", "q700", "q900"], + ... ) + >>> values, levels = to_quantiles(pred) + >>> values + tensor([[[1., 2., 3., 4.]]]) + >>> levels + tensor([0.1000, 0.4000, 0.7000, 0.9000]) + >>> pred = sdm.TableTensor.from_tensor( + ... torch.tensor([[1.0, 2.0, 3.0, 4.0]]), + ... columns=["a__q10", "a__q90", "b__q10", "b__q90"], + ... ) + >>> values, levels = to_quantiles(pred) + >>> values + tensor([[[1., 2.], + [3., 4.]]]) + >>> levels + tensor([0.1000, 0.9000]) + """ + columns = pred.columns[Stype.numerical] + prefixes, _, digits = zip(*(column.rpartition("q") for column in columns)) + targets = dict.fromkeys(prefixes) + digits = digits[: len(columns) // len(targets)] + if columns != tuple(f"{t}q{d}" for t in targets for d in digits): + raise ValueError( + "Expected quantile columns grouped by target with the same " + f"quantile levels (got {list(columns)})" + ) + + levels = [int(d) / 10 ** len(d) for d in digits] + values = pred.numerical.unflatten(-1, (len(targets), -1)) # [..., V, Q] + return values, values.new_tensor(levels) diff --git a/test/evaluation/test_regression.py b/test/evaluation/test_regression.py new file mode 100644 index 0000000000..9ae0941a8f --- /dev/null +++ b/test/evaluation/test_regression.py @@ -0,0 +1,26 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import pytest +import torch + +import sdm + + +def test_to_quantiles() -> None: + pred = sdm.TableTensor.from_tensor( + torch.randn(2, 5, 4), + columns=["b__q10", "b__q90", "a__q10", "a__q90"], + ) + values, levels = sdm.evaluation.to_quantiles(pred) + torch.testing.assert_close(values, pred.numerical.view(2, 5, 2, 2)) + torch.testing.assert_close(levels, torch.tensor([0.1, 0.9])) + + +def test_to_quantiles_invalid() -> None: + pred = sdm.TableTensor.from_tensor( + torch.randn(5, 4), + columns=["a__q10", "b__q10", "a__q90", "b__q90"], + ) + with pytest.raises(ValueError, match="grouped by target"): + sdm.evaluation.to_quantiles(pred) From 1f4c5bbcf9c51867214ed78a9c596f4fb213b363 Mon Sep 17 00:00:00 2001 From: Filippo Guerranti Date: Tue, 13 Oct 2026 09:33:10 +0000 Subject: [PATCH 2/7] Add GIFT-Eval forecasting benchmark --- .gitignore | 3 + benchmark/README.md | 2 + benchmark/forecasting/README.md | 51 ++++++ benchmark/forecasting/__init__.py | 2 + benchmark/forecasting/gift_eval/__init__.py | 8 + benchmark/forecasting/gift_eval/evaluate.py | 62 +++++++ benchmark/forecasting/gift_eval/main.py | 172 ++++++++++++++++++++ benchmark/forecasting/gift_eval/models.py | 85 ++++++++++ benchmark/forecasting/model.py | 118 ++++++++++++++ 9 files changed, 503 insertions(+) create mode 100644 benchmark/forecasting/README.md create mode 100644 benchmark/forecasting/__init__.py create mode 100644 benchmark/forecasting/gift_eval/__init__.py create mode 100644 benchmark/forecasting/gift_eval/evaluate.py create mode 100644 benchmark/forecasting/gift_eval/main.py create mode 100644 benchmark/forecasting/gift_eval/models.py create mode 100644 benchmark/forecasting/model.py diff --git a/.gitignore b/.gitignore index 73fa2fdd28..8790ab32a3 100644 --- a/.gitignore +++ b/.gitignore @@ -221,5 +221,8 @@ scoringbench_out/ evals/ datasets.json +# Forecasting benchmarks +gift_eval_out/ + # Checkpoint files *.pt diff --git a/benchmark/README.md b/benchmark/README.md index 964c325f69..36232fd7c7 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -9,3 +9,5 @@ This folder contains benchmark integrations for `structured-data-models`: - [**`talent/`**](tabular/talent/): [TALENT](https://github.com/LAMDA-Tabular/TALENT) - [**`relational/`**](relational/): Relational benchmarks - [**`relarena/`**](relational/relarena): [RelArena](https://github.com/PriorLabs/relarena) +- [**`forecasting/`**](forecasting/): Time-series forecasting benchmarks + - [**`gift_eval/`**](forecasting/): [GIFT-Eval](https://github.com/SalesforceAIResearch/gift-eval) diff --git a/benchmark/forecasting/README.md b/benchmark/forecasting/README.md new file mode 100644 index 0000000000..aee5f50a81 --- /dev/null +++ b/benchmark/forecasting/README.md @@ -0,0 +1,51 @@ +# Time-Series Forecasting Benchmarks + +This directory contains benchmarks of `structured-data-models` on [GIFT-Eval](https://github.com/SalesforceAIResearch/gift-eval). + +All benchmarks share one model adapter (see `benchmark/forecasting/model.py`) that maps each forecasting task onto the `ICLModel` interface: time steps become rows, past targets become `y_context`, all covariates and timestamps of the context become `x_context`, and the forecast horizon becomes `x_query` with one row of known covariates and timestamps per future time step. Columns keep their semantic types, so each model decides which ones it consumes. Series in a batch are left-padded with missing values, and multivariate series are flattened for models without multi-target support. + +______________________________________________________________________ + +> [!NOTE] +> Weights of `TimesFM3` are distributed under the [TimesFM Non-Commercial License v1.0](https://huggingface.co/google/timesfm-3.0-pytorch/blob/main/LICENSE). +> Review the license before running the `TimesFM3` benchmark, which will download its weights noninteractively. + +## GIFT-Eval + +### Setup + +Run the commands below from the repository root: + +```bash +pip install . gluonts datasets python-dotenv +pip install --no-deps "git+https://github.com/SalesforceAIResearch/gift-eval.git@577a65cfa3acb812405c00d989230bf59f7f8a3d" + +hf download Salesforce/GiftEval --repo-type dataset --local-dir /path/to/gift_eval +export GIFT_EVAL=/path/to/gift_eval +``` + +GIFT-Eval is not released on PyPI and is therefore pinned to a commit. It is installed without dependencies since it pins outdated `numpy` and `datasets` versions. + +### Run + +```bash +python -m benchmark.forecasting.gift_eval.main --model timesfm3 +``` + +Pass `--dataset` to run only selected GIFT-Eval datasets, *e.g.*, `--dataset hospital --dataset ett1/H`. +Use `--batch_size` to set the number of series per model call and `--max_context_size` to truncate long contexts. +Results are written to `benchmark/forecasting/gift_eval_out//`, and already completed configurations are skipped. + +### Evaluate + +```bash +python -m benchmark.forecasting.gift_eval.evaluate +``` + +This compares all available SDM results against published GIFT-Eval results on the same configurations, normalized by a seasonal naive baseline and aggregated by geometric mean. + +`TimesFM3` follows the published `TimesFM-3` evaluation with symmetric averaging over two estimators and contexts truncated to 15,360 time steps. Its results can still differ from the published ones: + +- Published results batch series of different context lengths together, which shifts their positional encodings on datasets with varying context lengths. +- Published results clip forecasts of non-negative series at zero. +- Published results forecast at most 32 variates jointly, subsampling covariates of `bizitobs_application` and `bizitobs_service`. diff --git a/benchmark/forecasting/__init__.py b/benchmark/forecasting/__init__.py new file mode 100644 index 0000000000..d51c4fe1e0 --- /dev/null +++ b/benchmark/forecasting/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/benchmark/forecasting/gift_eval/__init__.py b/benchmark/forecasting/gift_eval/__init__.py new file mode 100644 index 0000000000..417150a406 --- /dev/null +++ b/benchmark/forecasting/gift_eval/__init__.py @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Pinned GIFT-Eval revision for dataset properties and reference results: +GIFT_EVAL_URL = ( + "https://raw.githubusercontent.com/SalesforceAIResearch/gift-eval/" + "577a65cfa3acb812405c00d989230bf59f7f8a3d" +) diff --git a/benchmark/forecasting/gift_eval/evaluate.py b/benchmark/forecasting/gift_eval/evaluate.py new file mode 100644 index 0000000000..d6176985d9 --- /dev/null +++ b/benchmark/forecasting/gift_eval/evaluate.py @@ -0,0 +1,62 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +r"""Evaluate SDM forecasting model results on GIFT-Eval.""" + +from pathlib import Path + +import numpy as np +import pandas as pd + +from benchmark.forecasting.gift_eval import GIFT_EVAL_URL +from benchmark.forecasting.model import MODEL_CONFIGS + +BASELINE = "Seasonal_Naive" +REFERENCES = ("TimesFM-3", "TimesFM-2.5", "Chronos-2", BASELINE) +METRICS = { + "eval_metrics/MASE[0.5]": "MASE", + "eval_metrics/mean_weighted_sum_quantile_loss": "CRPS", +} + + +def main() -> None: + benchmark_dir = Path(__file__).parent.parent + result_root = benchmark_dir / "gift_eval_out" + output_root = benchmark_dir / "evals" / "gift_eval" + + results = [] + for model_config in MODEL_CONFIGS.values(): + path = result_root / model_config.name / "all_results.csv" + if path.exists(): + results.append(pd.read_csv(path)) + + if not results: + raise FileNotFoundError( + f"No GIFT-Eval results found under {result_root}" + ) + + # Compare with published results on configurations run by all models: + configs = set.intersection(*(set(result["dataset"]) for result in results)) + for name in REFERENCES: + path = f"{GIFT_EVAL_URL}/results/{name}/all_results.csv" + results.append(pd.read_csv(path)) + + df = pd.concat(results) + df = df[df["dataset"].isin(configs)].set_index(["model", "dataset"]) + df = df[list(METRICS)].rename(columns=METRICS) + + # Normalize by the seasonal naive baseline and aggregate by geometric mean: + log_scores = np.log(df.div(df.loc[BASELINE], level="dataset")) + terms = log_scores.index.get_level_values("dataset").str.split("/").str[-1] + + output_root.mkdir(parents=True, exist_ok=True) + for term, scores in [("overall", log_scores), *log_scores.groupby(terms)]: + leaderboard = np.exp(scores.groupby("model").mean()) + leaderboard = leaderboard.sort_values("CRPS") + leaderboard.to_csv(output_root / f"leaderboard_{term}.csv") + print(f"\nGIFT-Eval leaderboard ({term}):") + print(leaderboard.to_string(float_format="%.4f")) + + +if __name__ == "__main__": + main() diff --git a/benchmark/forecasting/gift_eval/main.py b/benchmark/forecasting/gift_eval/main.py new file mode 100644 index 0000000000..5138153280 --- /dev/null +++ b/benchmark/forecasting/gift_eval/main.py @@ -0,0 +1,172 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +r"""Run an SDM forecasting model on GIFT-Eval.""" + +import argparse +from dataclasses import replace +from pathlib import Path + +import pandas as pd +import torch +from gift_eval.data import Dataset +from gluonts.ev.metrics import ( + MAE, + MAPE, + MASE, + MSE, + MSIS, + ND, + NRMSE, + RMSE, + SMAPE, + MeanWeightedSumQuantileLoss, +) +from gluonts.model import evaluate_model +from gluonts.time_feature import get_seasonality + +from benchmark.forecasting.gift_eval import GIFT_EVAL_URL +from benchmark.forecasting.gift_eval.models import ( + QUANTILE_LEVELS, + SDMPredictor, +) +from benchmark.forecasting.model import MODEL_CONFIGS, add_model_args + +# Datasets and terms from GIFT-Eval's notebooks: +SHORT_DATASETS = ( # noqa: SIM905 + "m4_yearly m4_quarterly m4_monthly m4_weekly m4_daily m4_hourly " + "electricity/15T electricity/H electricity/D electricity/W " + "solar/10T solar/H solar/D solar/W hospital covid_deaths " + "us_births/D us_births/M us_births/W " + "saugeenday/D saugeenday/M saugeenday/W " + "temperature_rain_with_missing " + "kdd_cup_2018_with_missing/H kdd_cup_2018_with_missing/D " + "car_parts_with_missing restaurant " + "hierarchical_sales/D hierarchical_sales/W " + "LOOP_SEATTLE/5T LOOP_SEATTLE/H LOOP_SEATTLE/D SZ_TAXI/15T SZ_TAXI/H " + "M_DENSE/H M_DENSE/D ett1/15T ett1/H ett1/D ett1/W " + "ett2/15T ett2/H ett2/D ett2/W jena_weather/10T jena_weather/H " + "jena_weather/D bitbrains_fast_storage/5T bitbrains_fast_storage/H " + "bitbrains_rnd/5T bitbrains_rnd/H bizitobs_application bizitobs_service " + "bizitobs_l2c/5T bizitobs_l2c/H" +).split() +MED_LONG_DATASETS = ( # noqa: SIM905 + "electricity/15T electricity/H solar/10T solar/H " + "kdd_cup_2018_with_missing/H LOOP_SEATTLE/5T LOOP_SEATTLE/H SZ_TAXI/15T " + "M_DENSE/H ett1/15T ett1/H ett2/15T ett2/H jena_weather/10T " + "jena_weather/H bitbrains_fast_storage/5T bitbrains_rnd/5T " + "bizitobs_application bizitobs_service bizitobs_l2c/5T bizitobs_l2c/H" +).split() +PRETTY_NAMES = { + "saugeenday": "saugeen", + "temperature_rain_with_missing": "temperature_rain", + "kdd_cup_2018_with_missing": "kdd_cup_2018", + "car_parts_with_missing": "car_parts", +} +METRICS = [ + MSE(forecast_type="mean"), + MSE(forecast_type="0.5"), + MAE(), + MASE(), + MAPE(), + SMAPE(), + MSIS(), + RMSE(), + NRMSE(), + ND(), + MeanWeightedSumQuantileLoss(quantile_levels=QUANTILE_LEVELS), +] + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + add_model_args(parser) + parser.add_argument( + "--dataset", + action="append", + help="Run only the selected GIFT-Eval dataset; repeat to combine.", + ) + args = parser.parse_args() + + model_config = MODEL_CONFIGS[args.model] + if args.max_context_size is not None: + model_config = replace( + model_config, max_context_size=args.max_context_size + ) + result_path = ( + Path(__file__).parent.parent + / "gift_eval_out" + / model_config.name + / "all_results.csv" + ) + result_path.parent.mkdir(parents=True, exist_ok=True) + completed = ( + set(pd.read_csv(result_path)["dataset"]) + if result_path.exists() + else set() + ) + properties = pd.read_json( + f"{GIFT_EVAL_URL}/notebooks/dataset_properties.json" + ) + + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + model = model_config.factory(device) + + for name in args.dataset or SHORT_DATASETS: + key, _, freq = name.partition("/") + key = PRETTY_NAMES.get(key.lower(), key.lower()) + freq = freq or properties[key]["frequency"] + for term in ("short", "medium", "long"): + if term != "short" and name not in MED_LONG_DATASETS: + continue + config = f"{key}/{freq}/{term}" + if config in completed: + continue + + # Flatten multivariate series for models without joint multi-target + # forecasting: + dataset = Dataset( + name=name, + term=term, + to_univariate=( + not model.supports_multi_target + and properties[key]["num_variates"] > 1 + ), + ) + predictor = SDMPredictor( + model, + config=model_config, + prediction_length=dataset.prediction_length, + batch_size=args.batch_size, + ) + result = evaluate_model( + predictor, + test_data=dataset.test_data, + metrics=METRICS, + seasonality=get_seasonality(dataset.freq), + ) + row = pd.DataFrame( + [ + { + "dataset": config, + "model": f"[SDM] {model_config.name}", + **{ + f"eval_metrics/{k}": v + for k, v in result.iloc[0].items() + }, + "domain": properties[key]["domain"], + "num_variates": properties[key]["num_variates"], + } + ] + ) + row.to_csv( + result_path, + mode="a", + header=not result_path.exists(), + index=False, + ) + print(row.to_string(index=False)) + + +if __name__ == "__main__": + main() diff --git a/benchmark/forecasting/gift_eval/models.py b/benchmark/forecasting/gift_eval/models.py new file mode 100644 index 0000000000..c14b7f2541 --- /dev/null +++ b/benchmark/forecasting/gift_eval/models.py @@ -0,0 +1,85 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""SDM forecasting adapter for GIFT-Eval.""" + +from collections.abc import Iterator + +import numpy as np +import pandas as pd +from gluonts.dataset import Dataset +from gluonts.itertools import batcher +from gluonts.model.forecast import QuantileForecast +from gluonts.model.predictor import Predictor + +import sdm +from benchmark.forecasting.model import ModelConfig, forecast, select_levels + +QUANTILE_LEVELS = (0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9) + + +class SDMPredictor(Predictor): + """GluonTS predictor forecasting batches of series with an SDM model.""" + + def __init__( + self, + model: sdm.models.ICLModel, + config: ModelConfig, + prediction_length: int, + batch_size: int, + ) -> None: + super().__init__(prediction_length=prediction_length) + self.model = model + self.config = config + self.batch_size = batch_size + + def predict( + self, + dataset: Dataset, + **kwargs: object, + ) -> Iterator[QuantileForecast]: + for batch in batcher(dataset, batch_size=self.batch_size): + # GluonTS stores series as [V, T] (or [T] if univariate): + num_variates = np.atleast_2d(batch[0]["target"]).shape[0] + targets = [f"target_{i}" for i in range(num_variates)] + past, future = [], [] + for entry in batch: + target = np.atleast_2d(entry["target"]) + length = target.shape[-1] + timestamp = pd.period_range( + entry["start"], + periods=length + self.prediction_length, + ).to_timestamp() + columns = { + "item_id": entry["item_id"], + "timestamp": timestamp[:length], + } + columns |= dict(zip(targets, target, strict=True)) + for i, x in enumerate(entry.get("past_feat_dynamic_real", [])): + columns[f"past_feat_dynamic_real_{i}"] = x + past.append(pd.DataFrame(columns)) + future.append( + pd.DataFrame( + { + "item_id": entry["item_id"], + "timestamp": timestamp[length:], + } + ) + ) + values, levels = forecast( + self.model, + config=self.config, + past=past, + future=future, + targets=targets, + ) + values = select_levels(values, levels, QUANTILE_LEVELS) + for entry, value in zip(batch, values, strict=True): + value = value.transpose(2, 0, 1) # [Q, H, V] + if entry["target"].ndim == 1: + value = value[..., 0] + yield QuantileForecast( + forecast_arrays=value, + forecast_keys=[str(q) for q in QUANTILE_LEVELS], + start_date=entry["start"] + entry["target"].shape[-1], + ) diff --git a/benchmark/forecasting/model.py b/benchmark/forecasting/model.py new file mode 100644 index 0000000000..13304709bd --- /dev/null +++ b/benchmark/forecasting/model.py @@ -0,0 +1,118 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""SDM model adapter shared by the time-series forecasting benchmarks.""" + +import argparse +from collections.abc import Callable, Sequence +from dataclasses import dataclass + +import numpy as np +import pandas as pd +import torch + +import sdm +from sdm.models.timesfm3 import TimesFM3 + + +@dataclass(frozen=True) +class ModelConfig: + name: str + factory: Callable[[torch.device], sdm.models.ICLModel] + num_estimators: int = 1 + max_context_size: int | None = None + + +MODEL_CONFIGS = { + "timesfm3": ModelConfig( + name="TimesFM3", + factory=lambda device: TimesFM3(accept_license=True, device=device), + num_estimators=2, + max_context_size=15360, + ), +} + + +def add_model_args(parser: argparse.ArgumentParser) -> None: + """Add the model flags shared by all forecasting benchmarks.""" + parser.add_argument( + "--model", + choices=tuple(MODEL_CONFIGS), + default="timesfm3", + help="SDM model to benchmark.", + ) + parser.add_argument( + "--batch_size", + type=int, + default=64, + help="Number of series forecast per model call.", + ) + parser.add_argument( + "--max_context_size", + type=int, + help="Truncate each context to its most recent time steps " + "(defaults to the model's maximum context size).", + ) + + +def forecast( + model: sdm.models.ICLModel, + config: ModelConfig, + past: Sequence[pd.DataFrame], # B x [T_i, V + C] + future: Sequence[pd.DataFrame], # B x [H, K] + targets: Sequence[str], +) -> tuple[np.ndarray, np.ndarray]: # [B, H, V, Q], [Q] + """Forecast the quantiles of a batch of (multivariate) time series. + + Time steps map to rows: ``past`` holds the targets and all covariates of + the context, while ``future`` holds the covariates known over the + forecast horizon. Columns keep their semantic types, so that the model + decides which ones to consume. Missing values are passed through. + """ + device = next(model.parameters()).device + num_series, horizon = len(past), len(future[0]) + size = max(len(df) for df in past) + if config.max_context_size is not None: + size = min(size, config.max_context_size) + + # Left-pad shorter series with missing values so that all series end at + # the forecast start: + past = [df.iloc[-size:].reset_index(drop=True) for df in past] + past = [df.reindex(range(len(df) - size, len(df))) for df in past] + + # Infer semantic types jointly so that context and query share them: + df = pd.concat([*past, *future], ignore_index=True) + table = sdm.TableTensor.from_pandas( + df=df, + stypes=sdm.infer_stypes(df, id="infer"), + device=device, + ) + # Split rows into the context [B, T, V + C] and the query [B, H, V + C]: + context = table[: num_series * size].view(num_series, size, -1) + query = table[num_series * size :].view(num_series, horizon, -1) + assert isinstance(context, sdm.TableTensor) + assert isinstance(query, sdm.TableTensor) + + out = model( + x_context=context.drop_columns(targets), + y_context=context.select_columns(targets), + x_query=query.select_columns(future[0].columns), + num_estimators=config.num_estimators, + ) + values, levels = sdm.evaluation.to_quantiles(out) + return values.cpu().numpy(), levels.cpu().numpy() + + +def select_levels( + values: np.ndarray, # [..., Q] + levels: np.ndarray, # [Q] + wanted: Sequence[float], +) -> np.ndarray: # [..., len(wanted)] + """Select the quantile levels requested by a benchmark.""" + index = [int(np.abs(levels - level).argmin()) for level in wanted] + if not np.allclose(levels[index], wanted): + raise ValueError( + f"Model does not predict quantile levels {list(wanted)} " + f"(got {levels.tolist()})" + ) + return values[..., index] From 7497cf1d92cc435890cb9017c9075235bcfe73b0 Mon Sep 17 00:00:00 2001 From: Filippo Guerranti Date: Tue, 13 Oct 2026 15:59:29 +0000 Subject: [PATCH 3/7] Add fev-bench forecasting benchmark --- .gitignore | 1 + benchmark/README.md | 1 + benchmark/forecasting/README.md | 35 ++++++++- benchmark/forecasting/fev_bench/__init__.py | 5 ++ benchmark/forecasting/fev_bench/evaluate.py | 60 ++++++++++++++ benchmark/forecasting/fev_bench/main.py | 87 +++++++++++++++++++++ benchmark/forecasting/fev_bench/models.py | 87 +++++++++++++++++++++ benchmark/forecasting/model.py | 4 +- 8 files changed, 278 insertions(+), 2 deletions(-) create mode 100644 benchmark/forecasting/fev_bench/__init__.py create mode 100644 benchmark/forecasting/fev_bench/evaluate.py create mode 100644 benchmark/forecasting/fev_bench/main.py create mode 100644 benchmark/forecasting/fev_bench/models.py diff --git a/.gitignore b/.gitignore index 8790ab32a3..aea92019b3 100644 --- a/.gitignore +++ b/.gitignore @@ -223,6 +223,7 @@ datasets.json # Forecasting benchmarks gift_eval_out/ +fev_bench_out/ # Checkpoint files *.pt diff --git a/benchmark/README.md b/benchmark/README.md index 36232fd7c7..8ce0c1c59d 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -11,3 +11,4 @@ This folder contains benchmark integrations for `structured-data-models`: - [**`relarena/`**](relational/relarena): [RelArena](https://github.com/PriorLabs/relarena) - [**`forecasting/`**](forecasting/): Time-series forecasting benchmarks - [**`gift_eval/`**](forecasting/): [GIFT-Eval](https://github.com/SalesforceAIResearch/gift-eval) + - [**`fev_bench/`**](forecasting/): [fev-bench](https://github.com/autogluon/fev) diff --git a/benchmark/forecasting/README.md b/benchmark/forecasting/README.md index aee5f50a81..e15986cd27 100644 --- a/benchmark/forecasting/README.md +++ b/benchmark/forecasting/README.md @@ -1,6 +1,6 @@ # Time-Series Forecasting Benchmarks -This directory contains benchmarks of `structured-data-models` on [GIFT-Eval](https://github.com/SalesforceAIResearch/gift-eval). +This directory contains benchmarks of `structured-data-models` on [GIFT-Eval](https://github.com/SalesforceAIResearch/gift-eval) and [fev-bench](https://github.com/autogluon/fev). All benchmarks share one model adapter (see `benchmark/forecasting/model.py`) that maps each forecasting task onto the `ICLModel` interface: time steps become rows, past targets become `y_context`, all covariates and timestamps of the context become `x_context`, and the forecast horizon becomes `x_query` with one row of known covariates and timestamps per future time step. Columns keep their semantic types, so each model decides which ones it consumes. Series in a batch are left-padded with missing values, and multivariate series are flattened for models without multi-target support. @@ -49,3 +49,36 @@ This compares all available SDM results against published GIFT-Eval results on t - Published results batch series of different context lengths together, which shifts their positional encodings on datasets with varying context lengths. - Published results clip forecasts of non-negative series at zero. - Published results forecast at most 32 variates jointly, subsampling covariates of `bizitobs_application` and `bizitobs_service`. + +______________________________________________________________________ + +## fev-bench + +### Setup + +Run the command below from the repository root: + +```bash +pip install . fev==0.10.0 +``` + +Tasks are downloaded from Hugging Face on first use. + +### Run + +```bash +python -m benchmark.forecasting.fev_bench.main --model timesfm3 +``` + +Pass `--task` to run only selected fev-bench tasks, *e.g.*, `--task hospital --task proenfo_gfc12`. +Results are written to `benchmark/forecasting/fev_bench_out//`, and already completed tasks are skipped. + +### Evaluate + +```bash +python -m benchmark.forecasting.fev_bench.evaluate +``` + +This compares all available SDM results against published fev-bench results on the same tasks with `fev.leaderboard`, using a seasonal naive baseline. + +In addition to the differences listed for GIFT-Eval, published `TimesFM-3` results impute missing values with the last observed value, convert non-numerical covariates to numbers, and ignore static covariates. diff --git a/benchmark/forecasting/fev_bench/__init__.py b/benchmark/forecasting/fev_bench/__init__.py new file mode 100644 index 0000000000..6fdaa7520c --- /dev/null +++ b/benchmark/forecasting/fev_bench/__init__.py @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Task definitions and reference results of the installed fev release: +FEV_URL = "https://raw.githubusercontent.com/autogluon/fev/v0.10.0" diff --git a/benchmark/forecasting/fev_bench/evaluate.py b/benchmark/forecasting/fev_bench/evaluate.py new file mode 100644 index 0000000000..3b9c278f4f --- /dev/null +++ b/benchmark/forecasting/fev_bench/evaluate.py @@ -0,0 +1,60 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +r"""Evaluate SDM forecasting model results on fev-bench.""" + +from pathlib import Path + +import fev +import pandas as pd + +from benchmark.forecasting.fev_bench import FEV_URL +from benchmark.forecasting.model import MODEL_CONFIGS + +BASELINE = "Seasonal Naive" +REFERENCES = ("timesfm-3", "timesfm-2_5", "chronos-2", "seasonal_naive") +METRICS = ("SQL", "MASE") + + +def main() -> None: + benchmark_dir = Path(__file__).parent.parent + result_root = benchmark_dir / "fev_bench_out" + output_root = benchmark_dir / "evals" / "fev_bench" + + summaries = [] + for model_config in MODEL_CONFIGS.values(): + path = result_root / model_config.name / "results.csv" + if path.exists(): + summaries.append(pd.read_csv(path)) + + if not summaries: + raise FileNotFoundError( + f"No fev-bench results found under {result_root}" + ) + + # Compare with published results on tasks run by all models: + tasks = set.intersection(*(set(df["task_name"]) for df in summaries)) + for name in REFERENCES: + path = f"{FEV_URL}/benchmarks/fev_bench/results/{name}.csv" + summaries.append(pd.read_csv(path)) + summaries = pd.concat(summaries) + summaries = summaries[summaries["task_name"].isin(tasks)] + + output_root.mkdir(parents=True, exist_ok=True) + for metric in METRICS: + leaderboard = fev.leaderboard( + summaries, + metric_column=metric, + baseline_model=BASELINE, + ) + leaderboard.to_csv(output_root / f"leaderboard_{metric}.csv") + print(f"\nfev-bench leaderboard ({metric}, {len(tasks)} tasks):") + print( + leaderboard[["skill_score", "win_rate"]].to_string( + float_format="%.4f" + ) + ) + + +if __name__ == "__main__": + main() diff --git a/benchmark/forecasting/fev_bench/main.py b/benchmark/forecasting/fev_bench/main.py new file mode 100644 index 0000000000..ba0c44fb46 --- /dev/null +++ b/benchmark/forecasting/fev_bench/main.py @@ -0,0 +1,87 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +r"""Run an SDM forecasting model on fev-bench.""" + +import argparse +import json +from dataclasses import replace +from pathlib import Path + +import fev +import pandas as pd +import torch + +from benchmark.forecasting.fev_bench import FEV_URL +from benchmark.forecasting.fev_bench.models import SDMModel +from benchmark.forecasting.model import MODEL_CONFIGS, add_model_args + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + add_model_args(parser) + parser.add_argument( + "--task", + action="append", + help="Run only the selected fev-bench task; repeat to combine.", + ) + args = parser.parse_args() + + model_config = MODEL_CONFIGS[args.model] + if args.max_context_size is not None: + model_config = replace( + model_config, max_context_size=args.max_context_size + ) + result_path = ( + Path(__file__).parent.parent + / "fev_bench_out" + / model_config.name + / "results.csv" + ) + result_path.parent.mkdir(parents=True, exist_ok=True) + completed = ( + set(pd.read_csv(result_path)["task_name"]) + if result_path.exists() + else set() + ) + + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + model = SDMModel( + model_config.factory(device), + config=model_config, + batch_size=args.batch_size, + ) + + # Mirrors fev's official `models/evaluate.py` runner: + benchmark = fev.Benchmark.from_yaml( + f"{FEV_URL}/benchmarks/fev_bench/tasks.yaml" + ) + for task in benchmark.tasks: + if args.task is not None and task.task_name not in args.task: + continue + if task.task_name in completed: + continue + + predictions = model.fit_predict(task) + summary = task.evaluation_summary( + predictions, + model_name=f"[SDM] {model_config.name}", + training_time_s=model.training_time, + inference_time_s=model.inference_time, + trained_on_this_dataset=( + task.dataset_config in model.trained_on_datasets + ), + extra_info={"model_kwargs": json.dumps(vars(args))}, + ) + row = pd.DataFrame([summary]) + row.to_csv( + result_path, + mode="a", + header=not result_path.exists(), + index=False, + ) + print(row[["task_name", "test_error"]].to_string(index=False)) + + +if __name__ == "__main__": + main() diff --git a/benchmark/forecasting/fev_bench/models.py b/benchmark/forecasting/fev_bench/models.py new file mode 100644 index 0000000000..eba1ac6eef --- /dev/null +++ b/benchmark/forecasting/fev_bench/models.py @@ -0,0 +1,87 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""SDM forecasting adapter for fev-bench.""" + +from collections.abc import Iterator + +import datasets +import fev +import numpy as np +import pandas as pd + +import sdm +from benchmark.forecasting.model import ModelConfig, forecast, select_levels + + +class SDMModel(fev.ForecastingModel): + """fev model forecasting batches of series with an SDM model.""" + + def __init__( + self, + model: sdm.models.ICLModel, + config: ModelConfig, + batch_size: int, + ) -> None: + super().__init__() + self.model = model + self.config = config + self.batch_size = batch_size + + def _fit_predict(self, task: fev.Task) -> list[datasets.DatasetDict]: + predictions = [] + for window in task.iter_windows(): + past, future = window.get_input_data() + + values = [] + with self._record_inference_time(): + for past_batch, future_batch in zip( + _to_frames(past, self.batch_size), + _to_frames(future, self.batch_size), + strict=True, + ): + out, levels = forecast( + self.model, + config=self.config, + past=past_batch, + future=future_batch, + targets=task.target_columns, + ) + # Median point forecast followed by the task quantiles: + wanted = [0.5, *task.quantile_levels] + values.append(select_levels(out, levels, wanted)) + values = np.concatenate(values) # [N, H, V, 1 + Q] + + window_predictions = datasets.DatasetDict() + for i, column in enumerate(task.target_columns): + window_predictions[column] = datasets.Dataset.from_dict( + { + "predictions": values[:, :, i, 0], + **{ + str(level): values[:, :, i, j + 1] + for j, level in enumerate(task.quantile_levels) + }, + } + ) + predictions.append(window_predictions) + return predictions + + +def _to_frames( + dataset: datasets.Dataset, + batch_size: int, +) -> Iterator[list[pd.DataFrame]]: + # One frame per series, with static values repeated over time steps and + # the column types declared by the benchmark: + dtypes = { + name: pd.ArrowDtype(getattr(feature, "feature", feature).pa_type) + for name, feature in dataset.features.items() + } + for batch in dataset.with_format("numpy").iter(batch_size): + num_series = len(next(iter(batch.values()))) + yield [ + pd.DataFrame({name: x[i] for name, x in batch.items()}).astype( + dtypes + ) + for i in range(num_series) + ] diff --git a/benchmark/forecasting/model.py b/benchmark/forecasting/model.py index 13304709bd..0d2b4b1fb0 100644 --- a/benchmark/forecasting/model.py +++ b/benchmark/forecasting/model.py @@ -80,8 +80,10 @@ def forecast( past = [df.iloc[-size:].reset_index(drop=True) for df in past] past = [df.reindex(range(len(df) - size, len(df))) for df in past] - # Infer semantic types jointly so that context and query share them: + # Infer semantic types jointly so that context and query share them, and + # order targets first so that forecasts follow ``targets``: df = pd.concat([*past, *future], ignore_index=True) + df = df[[*targets, *df.columns.drop(targets)]] table = sdm.TableTensor.from_pandas( df=df, stypes=sdm.infer_stypes(df, id="infer"), From e980c312d8fb345ce2bd49ad503f9d88b8dcd2b7 Mon Sep 17 00:00:00 2001 From: Filippo Guerranti Date: Tue, 13 Oct 2026 16:15:03 +0000 Subject: [PATCH 4/7] Limit variates per forward pass for TimesFM3 benchmarks --- benchmark/forecasting/README.md | 3 +- benchmark/forecasting/model.py | 79 +++++++++++++++++++++++++++++---- 2 files changed, 72 insertions(+), 10 deletions(-) diff --git a/benchmark/forecasting/README.md b/benchmark/forecasting/README.md index e15986cd27..a3df47e969 100644 --- a/benchmark/forecasting/README.md +++ b/benchmark/forecasting/README.md @@ -44,11 +44,10 @@ python -m benchmark.forecasting.gift_eval.evaluate This compares all available SDM results against published GIFT-Eval results on the same configurations, normalized by a seasonal naive baseline and aggregated by geometric mean. -`TimesFM3` follows the published `TimesFM-3` evaluation with symmetric averaging over two estimators and contexts truncated to 15,360 time steps. Its results can still differ from the published ones: +`TimesFM3` follows the published `TimesFM-3` evaluation with symmetric averaging over two estimators, contexts truncated to 15,360 time steps, and at most 32 variates per forward pass. Its results can still differ from the published ones: - Published results batch series of different context lengths together, which shifts their positional encodings on datasets with varying context lengths. - Published results clip forecasts of non-negative series at zero. -- Published results forecast at most 32 variates jointly, subsampling covariates of `bizitobs_application` and `bizitobs_service`. ______________________________________________________________________ diff --git a/benchmark/forecasting/model.py b/benchmark/forecasting/model.py index 0d2b4b1fb0..e05fc902cb 100644 --- a/benchmark/forecasting/model.py +++ b/benchmark/forecasting/model.py @@ -4,7 +4,7 @@ """SDM model adapter shared by the time-series forecasting benchmarks.""" import argparse -from collections.abc import Callable, Sequence +from collections.abc import Callable, Collection, Sequence from dataclasses import dataclass import numpy as np @@ -21,6 +21,7 @@ class ModelConfig: factory: Callable[[torch.device], sdm.models.ICLModel] num_estimators: int = 1 max_context_size: int | None = None + max_variates: int | None = None MODEL_CONFIGS = { @@ -29,6 +30,7 @@ class ModelConfig: factory=lambda device: TimesFM3(accept_license=True, device=device), num_estimators=2, max_context_size=15360, + max_variates=32, ), } @@ -95,16 +97,77 @@ def forecast( assert isinstance(context, sdm.TableTensor) assert isinstance(query, sdm.TableTensor) - out = model( - x_context=context.drop_columns(targets), - y_context=context.select_columns(targets), - x_query=query.select_columns(future[0].columns), - num_estimators=config.num_estimators, - ) - values, levels = sdm.evaluation.to_quantiles(out) + x_context = context.drop_columns(targets) + x_query = query.select_columns(future[0].columns) + y = context.select_columns(targets).numerical # [B, T, V] + + chunks = [list(range(len(targets)))] + if config.max_variates is not None: + x_context, x_query, chunks = _split_variates( + x_context, + x_query, + num_targets=len(targets), + max_variates=config.max_variates, + stypes=model.supported_feature_stypes, + ) + + values = [] + for index in chunks: + out = model( + x_context=x_context, + y_context=sdm.TableTensor.from_tensor(y[..., index]), + x_query=x_query, + num_estimators=config.num_estimators, + ) + chunk, levels = sdm.evaluation.to_quantiles(out) # [B, H, V', Q] + values.append(chunk) + # Only the last chunk can end with filler targets: + values = torch.cat(values, dim=-2)[..., : len(targets), :] # [B, H, V, Q] return values.cpu().numpy(), levels.cpu().numpy() +def _split_variates( + x_context: sdm.TableTensor, + x_query: sdm.TableTensor, + num_targets: int, + max_variates: int, + stypes: Collection[sdm.Stype], + seed: int = 42, +) -> tuple[sdm.TableTensor, sdm.TableTensor, list[list[int]]]: + """Split targets and covariates into passes of at most ``max_variates``. + + Covariates of the given semantic types count as variates. If there are + too many, covariates are randomly subsampled (future-known ones first) to + leave room for at least one target. Targets are then split into chunks of + equal size, padding the last chunk with leading targets. Returns the + remaining covariates and the target indices of each chunk. + """ + covariates = [ + c for c, stype in x_context.stypes.items() if stype in stypes + ] + known = [c for c in covariates if c in x_query.column_names] + past = [c for c in covariates if c not in x_query.column_names] + + rng = np.random.default_rng(seed) + keep: list[str] = [] + for columns in (known, past): + size = min(len(columns), max_variates - 1 - len(keep)) + if size < len(columns): + index = np.sort(rng.choice(len(columns), size, replace=False)) + columns = [columns[i] for i in index] + keep += columns + drop = set(covariates).difference(keep) + x_context = x_context.drop_columns(drop) + x_query = x_query.drop_columns(drop & x_query.column_names) + + size = min(max_variates - len(keep), num_targets) + chunks = [ + [i % num_targets for i in range(start, start + size)] + for start in range(0, num_targets, size) + ] + return x_context, x_query, chunks + + def select_levels( values: np.ndarray, # [..., Q] levels: np.ndarray, # [Q] From 90c8b86cbfc02b0e89323b9e45af4b39840f8cb8 Mon Sep 17 00:00:00 2001 From: rusty1s Date: Mon, 26 Oct 2026 04:51:27 +0000 Subject: [PATCH 5/7] update --- benchmark/forecasting/README.md | 83 --------- benchmark/forecasting/__init__.py | 2 - benchmark/forecasting/fev_bench/__init__.py | 5 - benchmark/forecasting/fev_bench/evaluate.py | 60 ------- benchmark/forecasting/fev_bench/main.py | 87 ---------- benchmark/forecasting/fev_bench/models.py | 87 ---------- benchmark/forecasting/gift_eval/__init__.py | 8 - benchmark/forecasting/gift_eval/evaluate.py | 62 ------- benchmark/forecasting/gift_eval/main.py | 172 ------------------ benchmark/forecasting/gift_eval/models.py | 85 --------- benchmark/forecasting/model.py | 183 -------------------- benchmark/timeseries/gift_eval/README.md | 16 +- benchmark/timeseries/gift_eval/predictor.py | 51 ++++-- benchmark/timeseries/model.py | 2 + docs/source/icl.md | 1 - sdm/evaluation/__init__.py | 2 - sdm/evaluation/regression.py | 61 ------- test/evaluation/test_regression.py | 26 --- 18 files changed, 53 insertions(+), 940 deletions(-) delete mode 100644 benchmark/forecasting/README.md delete mode 100644 benchmark/forecasting/__init__.py delete mode 100644 benchmark/forecasting/fev_bench/__init__.py delete mode 100644 benchmark/forecasting/fev_bench/evaluate.py delete mode 100644 benchmark/forecasting/fev_bench/main.py delete mode 100644 benchmark/forecasting/fev_bench/models.py delete mode 100644 benchmark/forecasting/gift_eval/__init__.py delete mode 100644 benchmark/forecasting/gift_eval/evaluate.py delete mode 100644 benchmark/forecasting/gift_eval/main.py delete mode 100644 benchmark/forecasting/gift_eval/models.py delete mode 100644 benchmark/forecasting/model.py delete mode 100644 sdm/evaluation/regression.py delete mode 100644 test/evaluation/test_regression.py diff --git a/benchmark/forecasting/README.md b/benchmark/forecasting/README.md deleted file mode 100644 index a3df47e969..0000000000 --- a/benchmark/forecasting/README.md +++ /dev/null @@ -1,83 +0,0 @@ -# Time-Series Forecasting Benchmarks - -This directory contains benchmarks of `structured-data-models` on [GIFT-Eval](https://github.com/SalesforceAIResearch/gift-eval) and [fev-bench](https://github.com/autogluon/fev). - -All benchmarks share one model adapter (see `benchmark/forecasting/model.py`) that maps each forecasting task onto the `ICLModel` interface: time steps become rows, past targets become `y_context`, all covariates and timestamps of the context become `x_context`, and the forecast horizon becomes `x_query` with one row of known covariates and timestamps per future time step. Columns keep their semantic types, so each model decides which ones it consumes. Series in a batch are left-padded with missing values, and multivariate series are flattened for models without multi-target support. - -______________________________________________________________________ - -> [!NOTE] -> Weights of `TimesFM3` are distributed under the [TimesFM Non-Commercial License v1.0](https://huggingface.co/google/timesfm-3.0-pytorch/blob/main/LICENSE). -> Review the license before running the `TimesFM3` benchmark, which will download its weights noninteractively. - -## GIFT-Eval - -### Setup - -Run the commands below from the repository root: - -```bash -pip install . gluonts datasets python-dotenv -pip install --no-deps "git+https://github.com/SalesforceAIResearch/gift-eval.git@577a65cfa3acb812405c00d989230bf59f7f8a3d" - -hf download Salesforce/GiftEval --repo-type dataset --local-dir /path/to/gift_eval -export GIFT_EVAL=/path/to/gift_eval -``` - -GIFT-Eval is not released on PyPI and is therefore pinned to a commit. It is installed without dependencies since it pins outdated `numpy` and `datasets` versions. - -### Run - -```bash -python -m benchmark.forecasting.gift_eval.main --model timesfm3 -``` - -Pass `--dataset` to run only selected GIFT-Eval datasets, *e.g.*, `--dataset hospital --dataset ett1/H`. -Use `--batch_size` to set the number of series per model call and `--max_context_size` to truncate long contexts. -Results are written to `benchmark/forecasting/gift_eval_out//`, and already completed configurations are skipped. - -### Evaluate - -```bash -python -m benchmark.forecasting.gift_eval.evaluate -``` - -This compares all available SDM results against published GIFT-Eval results on the same configurations, normalized by a seasonal naive baseline and aggregated by geometric mean. - -`TimesFM3` follows the published `TimesFM-3` evaluation with symmetric averaging over two estimators, contexts truncated to 15,360 time steps, and at most 32 variates per forward pass. Its results can still differ from the published ones: - -- Published results batch series of different context lengths together, which shifts their positional encodings on datasets with varying context lengths. -- Published results clip forecasts of non-negative series at zero. - -______________________________________________________________________ - -## fev-bench - -### Setup - -Run the command below from the repository root: - -```bash -pip install . fev==0.10.0 -``` - -Tasks are downloaded from Hugging Face on first use. - -### Run - -```bash -python -m benchmark.forecasting.fev_bench.main --model timesfm3 -``` - -Pass `--task` to run only selected fev-bench tasks, *e.g.*, `--task hospital --task proenfo_gfc12`. -Results are written to `benchmark/forecasting/fev_bench_out//`, and already completed tasks are skipped. - -### Evaluate - -```bash -python -m benchmark.forecasting.fev_bench.evaluate -``` - -This compares all available SDM results against published fev-bench results on the same tasks with `fev.leaderboard`, using a seasonal naive baseline. - -In addition to the differences listed for GIFT-Eval, published `TimesFM-3` results impute missing values with the last observed value, convert non-numerical covariates to numbers, and ignore static covariates. diff --git a/benchmark/forecasting/__init__.py b/benchmark/forecasting/__init__.py deleted file mode 100644 index d51c4fe1e0..0000000000 --- a/benchmark/forecasting/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 diff --git a/benchmark/forecasting/fev_bench/__init__.py b/benchmark/forecasting/fev_bench/__init__.py deleted file mode 100644 index 6fdaa7520c..0000000000 --- a/benchmark/forecasting/fev_bench/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# Task definitions and reference results of the installed fev release: -FEV_URL = "https://raw.githubusercontent.com/autogluon/fev/v0.10.0" diff --git a/benchmark/forecasting/fev_bench/evaluate.py b/benchmark/forecasting/fev_bench/evaluate.py deleted file mode 100644 index 3b9c278f4f..0000000000 --- a/benchmark/forecasting/fev_bench/evaluate.py +++ /dev/null @@ -1,60 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -r"""Evaluate SDM forecasting model results on fev-bench.""" - -from pathlib import Path - -import fev -import pandas as pd - -from benchmark.forecasting.fev_bench import FEV_URL -from benchmark.forecasting.model import MODEL_CONFIGS - -BASELINE = "Seasonal Naive" -REFERENCES = ("timesfm-3", "timesfm-2_5", "chronos-2", "seasonal_naive") -METRICS = ("SQL", "MASE") - - -def main() -> None: - benchmark_dir = Path(__file__).parent.parent - result_root = benchmark_dir / "fev_bench_out" - output_root = benchmark_dir / "evals" / "fev_bench" - - summaries = [] - for model_config in MODEL_CONFIGS.values(): - path = result_root / model_config.name / "results.csv" - if path.exists(): - summaries.append(pd.read_csv(path)) - - if not summaries: - raise FileNotFoundError( - f"No fev-bench results found under {result_root}" - ) - - # Compare with published results on tasks run by all models: - tasks = set.intersection(*(set(df["task_name"]) for df in summaries)) - for name in REFERENCES: - path = f"{FEV_URL}/benchmarks/fev_bench/results/{name}.csv" - summaries.append(pd.read_csv(path)) - summaries = pd.concat(summaries) - summaries = summaries[summaries["task_name"].isin(tasks)] - - output_root.mkdir(parents=True, exist_ok=True) - for metric in METRICS: - leaderboard = fev.leaderboard( - summaries, - metric_column=metric, - baseline_model=BASELINE, - ) - leaderboard.to_csv(output_root / f"leaderboard_{metric}.csv") - print(f"\nfev-bench leaderboard ({metric}, {len(tasks)} tasks):") - print( - leaderboard[["skill_score", "win_rate"]].to_string( - float_format="%.4f" - ) - ) - - -if __name__ == "__main__": - main() diff --git a/benchmark/forecasting/fev_bench/main.py b/benchmark/forecasting/fev_bench/main.py deleted file mode 100644 index ba0c44fb46..0000000000 --- a/benchmark/forecasting/fev_bench/main.py +++ /dev/null @@ -1,87 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -r"""Run an SDM forecasting model on fev-bench.""" - -import argparse -import json -from dataclasses import replace -from pathlib import Path - -import fev -import pandas as pd -import torch - -from benchmark.forecasting.fev_bench import FEV_URL -from benchmark.forecasting.fev_bench.models import SDMModel -from benchmark.forecasting.model import MODEL_CONFIGS, add_model_args - - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - add_model_args(parser) - parser.add_argument( - "--task", - action="append", - help="Run only the selected fev-bench task; repeat to combine.", - ) - args = parser.parse_args() - - model_config = MODEL_CONFIGS[args.model] - if args.max_context_size is not None: - model_config = replace( - model_config, max_context_size=args.max_context_size - ) - result_path = ( - Path(__file__).parent.parent - / "fev_bench_out" - / model_config.name - / "results.csv" - ) - result_path.parent.mkdir(parents=True, exist_ok=True) - completed = ( - set(pd.read_csv(result_path)["task_name"]) - if result_path.exists() - else set() - ) - - device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - model = SDMModel( - model_config.factory(device), - config=model_config, - batch_size=args.batch_size, - ) - - # Mirrors fev's official `models/evaluate.py` runner: - benchmark = fev.Benchmark.from_yaml( - f"{FEV_URL}/benchmarks/fev_bench/tasks.yaml" - ) - for task in benchmark.tasks: - if args.task is not None and task.task_name not in args.task: - continue - if task.task_name in completed: - continue - - predictions = model.fit_predict(task) - summary = task.evaluation_summary( - predictions, - model_name=f"[SDM] {model_config.name}", - training_time_s=model.training_time, - inference_time_s=model.inference_time, - trained_on_this_dataset=( - task.dataset_config in model.trained_on_datasets - ), - extra_info={"model_kwargs": json.dumps(vars(args))}, - ) - row = pd.DataFrame([summary]) - row.to_csv( - result_path, - mode="a", - header=not result_path.exists(), - index=False, - ) - print(row[["task_name", "test_error"]].to_string(index=False)) - - -if __name__ == "__main__": - main() diff --git a/benchmark/forecasting/fev_bench/models.py b/benchmark/forecasting/fev_bench/models.py deleted file mode 100644 index eba1ac6eef..0000000000 --- a/benchmark/forecasting/fev_bench/models.py +++ /dev/null @@ -1,87 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""SDM forecasting adapter for fev-bench.""" - -from collections.abc import Iterator - -import datasets -import fev -import numpy as np -import pandas as pd - -import sdm -from benchmark.forecasting.model import ModelConfig, forecast, select_levels - - -class SDMModel(fev.ForecastingModel): - """fev model forecasting batches of series with an SDM model.""" - - def __init__( - self, - model: sdm.models.ICLModel, - config: ModelConfig, - batch_size: int, - ) -> None: - super().__init__() - self.model = model - self.config = config - self.batch_size = batch_size - - def _fit_predict(self, task: fev.Task) -> list[datasets.DatasetDict]: - predictions = [] - for window in task.iter_windows(): - past, future = window.get_input_data() - - values = [] - with self._record_inference_time(): - for past_batch, future_batch in zip( - _to_frames(past, self.batch_size), - _to_frames(future, self.batch_size), - strict=True, - ): - out, levels = forecast( - self.model, - config=self.config, - past=past_batch, - future=future_batch, - targets=task.target_columns, - ) - # Median point forecast followed by the task quantiles: - wanted = [0.5, *task.quantile_levels] - values.append(select_levels(out, levels, wanted)) - values = np.concatenate(values) # [N, H, V, 1 + Q] - - window_predictions = datasets.DatasetDict() - for i, column in enumerate(task.target_columns): - window_predictions[column] = datasets.Dataset.from_dict( - { - "predictions": values[:, :, i, 0], - **{ - str(level): values[:, :, i, j + 1] - for j, level in enumerate(task.quantile_levels) - }, - } - ) - predictions.append(window_predictions) - return predictions - - -def _to_frames( - dataset: datasets.Dataset, - batch_size: int, -) -> Iterator[list[pd.DataFrame]]: - # One frame per series, with static values repeated over time steps and - # the column types declared by the benchmark: - dtypes = { - name: pd.ArrowDtype(getattr(feature, "feature", feature).pa_type) - for name, feature in dataset.features.items() - } - for batch in dataset.with_format("numpy").iter(batch_size): - num_series = len(next(iter(batch.values()))) - yield [ - pd.DataFrame({name: x[i] for name, x in batch.items()}).astype( - dtypes - ) - for i in range(num_series) - ] diff --git a/benchmark/forecasting/gift_eval/__init__.py b/benchmark/forecasting/gift_eval/__init__.py deleted file mode 100644 index 417150a406..0000000000 --- a/benchmark/forecasting/gift_eval/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# Pinned GIFT-Eval revision for dataset properties and reference results: -GIFT_EVAL_URL = ( - "https://raw.githubusercontent.com/SalesforceAIResearch/gift-eval/" - "577a65cfa3acb812405c00d989230bf59f7f8a3d" -) diff --git a/benchmark/forecasting/gift_eval/evaluate.py b/benchmark/forecasting/gift_eval/evaluate.py deleted file mode 100644 index d6176985d9..0000000000 --- a/benchmark/forecasting/gift_eval/evaluate.py +++ /dev/null @@ -1,62 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -r"""Evaluate SDM forecasting model results on GIFT-Eval.""" - -from pathlib import Path - -import numpy as np -import pandas as pd - -from benchmark.forecasting.gift_eval import GIFT_EVAL_URL -from benchmark.forecasting.model import MODEL_CONFIGS - -BASELINE = "Seasonal_Naive" -REFERENCES = ("TimesFM-3", "TimesFM-2.5", "Chronos-2", BASELINE) -METRICS = { - "eval_metrics/MASE[0.5]": "MASE", - "eval_metrics/mean_weighted_sum_quantile_loss": "CRPS", -} - - -def main() -> None: - benchmark_dir = Path(__file__).parent.parent - result_root = benchmark_dir / "gift_eval_out" - output_root = benchmark_dir / "evals" / "gift_eval" - - results = [] - for model_config in MODEL_CONFIGS.values(): - path = result_root / model_config.name / "all_results.csv" - if path.exists(): - results.append(pd.read_csv(path)) - - if not results: - raise FileNotFoundError( - f"No GIFT-Eval results found under {result_root}" - ) - - # Compare with published results on configurations run by all models: - configs = set.intersection(*(set(result["dataset"]) for result in results)) - for name in REFERENCES: - path = f"{GIFT_EVAL_URL}/results/{name}/all_results.csv" - results.append(pd.read_csv(path)) - - df = pd.concat(results) - df = df[df["dataset"].isin(configs)].set_index(["model", "dataset"]) - df = df[list(METRICS)].rename(columns=METRICS) - - # Normalize by the seasonal naive baseline and aggregate by geometric mean: - log_scores = np.log(df.div(df.loc[BASELINE], level="dataset")) - terms = log_scores.index.get_level_values("dataset").str.split("/").str[-1] - - output_root.mkdir(parents=True, exist_ok=True) - for term, scores in [("overall", log_scores), *log_scores.groupby(terms)]: - leaderboard = np.exp(scores.groupby("model").mean()) - leaderboard = leaderboard.sort_values("CRPS") - leaderboard.to_csv(output_root / f"leaderboard_{term}.csv") - print(f"\nGIFT-Eval leaderboard ({term}):") - print(leaderboard.to_string(float_format="%.4f")) - - -if __name__ == "__main__": - main() diff --git a/benchmark/forecasting/gift_eval/main.py b/benchmark/forecasting/gift_eval/main.py deleted file mode 100644 index 5138153280..0000000000 --- a/benchmark/forecasting/gift_eval/main.py +++ /dev/null @@ -1,172 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -r"""Run an SDM forecasting model on GIFT-Eval.""" - -import argparse -from dataclasses import replace -from pathlib import Path - -import pandas as pd -import torch -from gift_eval.data import Dataset -from gluonts.ev.metrics import ( - MAE, - MAPE, - MASE, - MSE, - MSIS, - ND, - NRMSE, - RMSE, - SMAPE, - MeanWeightedSumQuantileLoss, -) -from gluonts.model import evaluate_model -from gluonts.time_feature import get_seasonality - -from benchmark.forecasting.gift_eval import GIFT_EVAL_URL -from benchmark.forecasting.gift_eval.models import ( - QUANTILE_LEVELS, - SDMPredictor, -) -from benchmark.forecasting.model import MODEL_CONFIGS, add_model_args - -# Datasets and terms from GIFT-Eval's notebooks: -SHORT_DATASETS = ( # noqa: SIM905 - "m4_yearly m4_quarterly m4_monthly m4_weekly m4_daily m4_hourly " - "electricity/15T electricity/H electricity/D electricity/W " - "solar/10T solar/H solar/D solar/W hospital covid_deaths " - "us_births/D us_births/M us_births/W " - "saugeenday/D saugeenday/M saugeenday/W " - "temperature_rain_with_missing " - "kdd_cup_2018_with_missing/H kdd_cup_2018_with_missing/D " - "car_parts_with_missing restaurant " - "hierarchical_sales/D hierarchical_sales/W " - "LOOP_SEATTLE/5T LOOP_SEATTLE/H LOOP_SEATTLE/D SZ_TAXI/15T SZ_TAXI/H " - "M_DENSE/H M_DENSE/D ett1/15T ett1/H ett1/D ett1/W " - "ett2/15T ett2/H ett2/D ett2/W jena_weather/10T jena_weather/H " - "jena_weather/D bitbrains_fast_storage/5T bitbrains_fast_storage/H " - "bitbrains_rnd/5T bitbrains_rnd/H bizitobs_application bizitobs_service " - "bizitobs_l2c/5T bizitobs_l2c/H" -).split() -MED_LONG_DATASETS = ( # noqa: SIM905 - "electricity/15T electricity/H solar/10T solar/H " - "kdd_cup_2018_with_missing/H LOOP_SEATTLE/5T LOOP_SEATTLE/H SZ_TAXI/15T " - "M_DENSE/H ett1/15T ett1/H ett2/15T ett2/H jena_weather/10T " - "jena_weather/H bitbrains_fast_storage/5T bitbrains_rnd/5T " - "bizitobs_application bizitobs_service bizitobs_l2c/5T bizitobs_l2c/H" -).split() -PRETTY_NAMES = { - "saugeenday": "saugeen", - "temperature_rain_with_missing": "temperature_rain", - "kdd_cup_2018_with_missing": "kdd_cup_2018", - "car_parts_with_missing": "car_parts", -} -METRICS = [ - MSE(forecast_type="mean"), - MSE(forecast_type="0.5"), - MAE(), - MASE(), - MAPE(), - SMAPE(), - MSIS(), - RMSE(), - NRMSE(), - ND(), - MeanWeightedSumQuantileLoss(quantile_levels=QUANTILE_LEVELS), -] - - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - add_model_args(parser) - parser.add_argument( - "--dataset", - action="append", - help="Run only the selected GIFT-Eval dataset; repeat to combine.", - ) - args = parser.parse_args() - - model_config = MODEL_CONFIGS[args.model] - if args.max_context_size is not None: - model_config = replace( - model_config, max_context_size=args.max_context_size - ) - result_path = ( - Path(__file__).parent.parent - / "gift_eval_out" - / model_config.name - / "all_results.csv" - ) - result_path.parent.mkdir(parents=True, exist_ok=True) - completed = ( - set(pd.read_csv(result_path)["dataset"]) - if result_path.exists() - else set() - ) - properties = pd.read_json( - f"{GIFT_EVAL_URL}/notebooks/dataset_properties.json" - ) - - device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - model = model_config.factory(device) - - for name in args.dataset or SHORT_DATASETS: - key, _, freq = name.partition("/") - key = PRETTY_NAMES.get(key.lower(), key.lower()) - freq = freq or properties[key]["frequency"] - for term in ("short", "medium", "long"): - if term != "short" and name not in MED_LONG_DATASETS: - continue - config = f"{key}/{freq}/{term}" - if config in completed: - continue - - # Flatten multivariate series for models without joint multi-target - # forecasting: - dataset = Dataset( - name=name, - term=term, - to_univariate=( - not model.supports_multi_target - and properties[key]["num_variates"] > 1 - ), - ) - predictor = SDMPredictor( - model, - config=model_config, - prediction_length=dataset.prediction_length, - batch_size=args.batch_size, - ) - result = evaluate_model( - predictor, - test_data=dataset.test_data, - metrics=METRICS, - seasonality=get_seasonality(dataset.freq), - ) - row = pd.DataFrame( - [ - { - "dataset": config, - "model": f"[SDM] {model_config.name}", - **{ - f"eval_metrics/{k}": v - for k, v in result.iloc[0].items() - }, - "domain": properties[key]["domain"], - "num_variates": properties[key]["num_variates"], - } - ] - ) - row.to_csv( - result_path, - mode="a", - header=not result_path.exists(), - index=False, - ) - print(row.to_string(index=False)) - - -if __name__ == "__main__": - main() diff --git a/benchmark/forecasting/gift_eval/models.py b/benchmark/forecasting/gift_eval/models.py deleted file mode 100644 index c14b7f2541..0000000000 --- a/benchmark/forecasting/gift_eval/models.py +++ /dev/null @@ -1,85 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""SDM forecasting adapter for GIFT-Eval.""" - -from collections.abc import Iterator - -import numpy as np -import pandas as pd -from gluonts.dataset import Dataset -from gluonts.itertools import batcher -from gluonts.model.forecast import QuantileForecast -from gluonts.model.predictor import Predictor - -import sdm -from benchmark.forecasting.model import ModelConfig, forecast, select_levels - -QUANTILE_LEVELS = (0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9) - - -class SDMPredictor(Predictor): - """GluonTS predictor forecasting batches of series with an SDM model.""" - - def __init__( - self, - model: sdm.models.ICLModel, - config: ModelConfig, - prediction_length: int, - batch_size: int, - ) -> None: - super().__init__(prediction_length=prediction_length) - self.model = model - self.config = config - self.batch_size = batch_size - - def predict( - self, - dataset: Dataset, - **kwargs: object, - ) -> Iterator[QuantileForecast]: - for batch in batcher(dataset, batch_size=self.batch_size): - # GluonTS stores series as [V, T] (or [T] if univariate): - num_variates = np.atleast_2d(batch[0]["target"]).shape[0] - targets = [f"target_{i}" for i in range(num_variates)] - past, future = [], [] - for entry in batch: - target = np.atleast_2d(entry["target"]) - length = target.shape[-1] - timestamp = pd.period_range( - entry["start"], - periods=length + self.prediction_length, - ).to_timestamp() - columns = { - "item_id": entry["item_id"], - "timestamp": timestamp[:length], - } - columns |= dict(zip(targets, target, strict=True)) - for i, x in enumerate(entry.get("past_feat_dynamic_real", [])): - columns[f"past_feat_dynamic_real_{i}"] = x - past.append(pd.DataFrame(columns)) - future.append( - pd.DataFrame( - { - "item_id": entry["item_id"], - "timestamp": timestamp[length:], - } - ) - ) - values, levels = forecast( - self.model, - config=self.config, - past=past, - future=future, - targets=targets, - ) - values = select_levels(values, levels, QUANTILE_LEVELS) - for entry, value in zip(batch, values, strict=True): - value = value.transpose(2, 0, 1) # [Q, H, V] - if entry["target"].ndim == 1: - value = value[..., 0] - yield QuantileForecast( - forecast_arrays=value, - forecast_keys=[str(q) for q in QUANTILE_LEVELS], - start_date=entry["start"] + entry["target"].shape[-1], - ) diff --git a/benchmark/forecasting/model.py b/benchmark/forecasting/model.py deleted file mode 100644 index e05fc902cb..0000000000 --- a/benchmark/forecasting/model.py +++ /dev/null @@ -1,183 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""SDM model adapter shared by the time-series forecasting benchmarks.""" - -import argparse -from collections.abc import Callable, Collection, Sequence -from dataclasses import dataclass - -import numpy as np -import pandas as pd -import torch - -import sdm -from sdm.models.timesfm3 import TimesFM3 - - -@dataclass(frozen=True) -class ModelConfig: - name: str - factory: Callable[[torch.device], sdm.models.ICLModel] - num_estimators: int = 1 - max_context_size: int | None = None - max_variates: int | None = None - - -MODEL_CONFIGS = { - "timesfm3": ModelConfig( - name="TimesFM3", - factory=lambda device: TimesFM3(accept_license=True, device=device), - num_estimators=2, - max_context_size=15360, - max_variates=32, - ), -} - - -def add_model_args(parser: argparse.ArgumentParser) -> None: - """Add the model flags shared by all forecasting benchmarks.""" - parser.add_argument( - "--model", - choices=tuple(MODEL_CONFIGS), - default="timesfm3", - help="SDM model to benchmark.", - ) - parser.add_argument( - "--batch_size", - type=int, - default=64, - help="Number of series forecast per model call.", - ) - parser.add_argument( - "--max_context_size", - type=int, - help="Truncate each context to its most recent time steps " - "(defaults to the model's maximum context size).", - ) - - -def forecast( - model: sdm.models.ICLModel, - config: ModelConfig, - past: Sequence[pd.DataFrame], # B x [T_i, V + C] - future: Sequence[pd.DataFrame], # B x [H, K] - targets: Sequence[str], -) -> tuple[np.ndarray, np.ndarray]: # [B, H, V, Q], [Q] - """Forecast the quantiles of a batch of (multivariate) time series. - - Time steps map to rows: ``past`` holds the targets and all covariates of - the context, while ``future`` holds the covariates known over the - forecast horizon. Columns keep their semantic types, so that the model - decides which ones to consume. Missing values are passed through. - """ - device = next(model.parameters()).device - num_series, horizon = len(past), len(future[0]) - size = max(len(df) for df in past) - if config.max_context_size is not None: - size = min(size, config.max_context_size) - - # Left-pad shorter series with missing values so that all series end at - # the forecast start: - past = [df.iloc[-size:].reset_index(drop=True) for df in past] - past = [df.reindex(range(len(df) - size, len(df))) for df in past] - - # Infer semantic types jointly so that context and query share them, and - # order targets first so that forecasts follow ``targets``: - df = pd.concat([*past, *future], ignore_index=True) - df = df[[*targets, *df.columns.drop(targets)]] - table = sdm.TableTensor.from_pandas( - df=df, - stypes=sdm.infer_stypes(df, id="infer"), - device=device, - ) - # Split rows into the context [B, T, V + C] and the query [B, H, V + C]: - context = table[: num_series * size].view(num_series, size, -1) - query = table[num_series * size :].view(num_series, horizon, -1) - assert isinstance(context, sdm.TableTensor) - assert isinstance(query, sdm.TableTensor) - - x_context = context.drop_columns(targets) - x_query = query.select_columns(future[0].columns) - y = context.select_columns(targets).numerical # [B, T, V] - - chunks = [list(range(len(targets)))] - if config.max_variates is not None: - x_context, x_query, chunks = _split_variates( - x_context, - x_query, - num_targets=len(targets), - max_variates=config.max_variates, - stypes=model.supported_feature_stypes, - ) - - values = [] - for index in chunks: - out = model( - x_context=x_context, - y_context=sdm.TableTensor.from_tensor(y[..., index]), - x_query=x_query, - num_estimators=config.num_estimators, - ) - chunk, levels = sdm.evaluation.to_quantiles(out) # [B, H, V', Q] - values.append(chunk) - # Only the last chunk can end with filler targets: - values = torch.cat(values, dim=-2)[..., : len(targets), :] # [B, H, V, Q] - return values.cpu().numpy(), levels.cpu().numpy() - - -def _split_variates( - x_context: sdm.TableTensor, - x_query: sdm.TableTensor, - num_targets: int, - max_variates: int, - stypes: Collection[sdm.Stype], - seed: int = 42, -) -> tuple[sdm.TableTensor, sdm.TableTensor, list[list[int]]]: - """Split targets and covariates into passes of at most ``max_variates``. - - Covariates of the given semantic types count as variates. If there are - too many, covariates are randomly subsampled (future-known ones first) to - leave room for at least one target. Targets are then split into chunks of - equal size, padding the last chunk with leading targets. Returns the - remaining covariates and the target indices of each chunk. - """ - covariates = [ - c for c, stype in x_context.stypes.items() if stype in stypes - ] - known = [c for c in covariates if c in x_query.column_names] - past = [c for c in covariates if c not in x_query.column_names] - - rng = np.random.default_rng(seed) - keep: list[str] = [] - for columns in (known, past): - size = min(len(columns), max_variates - 1 - len(keep)) - if size < len(columns): - index = np.sort(rng.choice(len(columns), size, replace=False)) - columns = [columns[i] for i in index] - keep += columns - drop = set(covariates).difference(keep) - x_context = x_context.drop_columns(drop) - x_query = x_query.drop_columns(drop & x_query.column_names) - - size = min(max_variates - len(keep), num_targets) - chunks = [ - [i % num_targets for i in range(start, start + size)] - for start in range(0, num_targets, size) - ] - return x_context, x_query, chunks - - -def select_levels( - values: np.ndarray, # [..., Q] - levels: np.ndarray, # [Q] - wanted: Sequence[float], -) -> np.ndarray: # [..., len(wanted)] - """Select the quantile levels requested by a benchmark.""" - index = [int(np.abs(levels - level).argmin()) for level in wanted] - if not np.allclose(levels[index], wanted): - raise ValueError( - f"Model does not predict quantile levels {list(wanted)} " - f"(got {levels.tolist()})" - ) - return values[..., index] diff --git a/benchmark/timeseries/gift_eval/README.md b/benchmark/timeseries/gift_eval/README.md index d49f394c04..fb1cd669d3 100644 --- a/benchmark/timeseries/gift_eval/README.md +++ b/benchmark/timeseries/gift_eval/README.md @@ -13,13 +13,17 @@ pip install --no-deps "git+https://github.com/SalesforceAIResearch/gift-eval.git ## Run -> [!NOTE] -> Weights of `TimesFM3` are distributed under the [TimesFM Non-Commercial License v1.0](https://huggingface.co/google/timesfm-3.0-pytorch/blob/main/LICENSE). -> Review the license before running the `TimesFM3` benchmark, which will download its weights noninteractively. +- **`TimesFM3`**: -```bash -python -m benchmark.timeseries.gift_eval.main --model timesfm3 -``` + ```bash + python -m benchmark.timeseries.gift_eval.main --model timesfm3 + ``` + + > [!NOTE] + > Weights of `TimesFM3` are distributed under the [TimesFM Non-Commercial License v1.0](https://huggingface.co/google/timesfm-3.0-pytorch/blob/main/LICENSE). + > Review the license before running the `TimesFM3` benchmark, which will download its weights noninteractively. + + `TimesFM3` follows official evaluation with symmetric averaging over two estimators, contexts truncated to 15,360 time steps and up to 32 variates per forward pass. Results will be written to `benchmark/timeseries/gift_eval_out//`, and already completed configurations are skipped. diff --git a/benchmark/timeseries/gift_eval/predictor.py b/benchmark/timeseries/gift_eval/predictor.py index 4266a55174..67a8be9785 100644 --- a/benchmark/timeseries/gift_eval/predictor.py +++ b/benchmark/timeseries/gift_eval/predictor.py @@ -3,6 +3,7 @@ from collections.abc import Iterator +import numpy as np import torch from gluonts.dataset import Dataset from gluonts.itertools import batcher @@ -41,6 +42,7 @@ def predict( xs = [] if ( self.model.supports_context_only_columns + and sdm.Stype.numerical in self.model.supported_feature_stypes and "past_feat_dynamic_real" in batch[0] ): xs = [ @@ -67,6 +69,19 @@ def predict( for i, x_i in enumerate(xs): x[i, :, -x_i.size(-1) :] = x_i + size = y.size(1) + if self.config.max_variates is not None: + # Keep room for at least one target, matching upstream's + # fixed-seed selection of past covariates. + count = min(x.size(1), self.config.max_variates - 1) + if count < x.size(1): + rng = np.random.default_rng(42) + index = np.sort( + rng.choice(x.size(1), count, replace=False) + ) + x = x[:, index] + size = min(size, self.config.max_variates - count) + y = y.transpose(-1, -2).to( self.device, memory_format=torch.contiguous_format, @@ -76,17 +91,33 @@ def predict( memory_format=torch.contiguous_format, ) - out = self.model( - x_context=x, - y_context=y, - x_query=torch.empty( - (x.size(0), self.prediction_length, 0), - device=self.device, - ), - num_estimators=self.config.num_estimators, + x_query = x.new_empty( + (x.size(0), self.prediction_length, 0), + ) + predictions = [] + for start in range(0, y.size(-1), size): + chunk = y[..., start : start + size] + # Upstream fills the last chunk with leading target channels. + if chunk.size(-1) < size: + chunk = torch.cat( + [chunk, y[..., : size - chunk.size(-1)]], dim=-1 + ) + out = self.model( + x_context=x, + y_context=chunk, + x_query=x_query, + num_estimators=self.config.num_estimators, + ) + predictions.append( + out.numerical.view( + y.size(0), self.prediction_length, size, -1 + ) + ) + pred = ( + predictions[0] + if len(predictions) == 1 + else torch.cat(predictions, dim=-2)[..., : y.size(-1), :] ) - pred = out.numerical - pred = pred.view(y.size(0), self.prediction_length, y.size(2), -1) pred = pred.permute(0, 3, 1, 2).cpu().numpy() # [B, Q, R, C] levels = out.numerical_columns[: pred.shape[1]] levels = [str(float(f"0.{q.split('q')[-1]}")) for q in levels] diff --git a/benchmark/timeseries/model.py b/benchmark/timeseries/model.py index feb9b85dba..2689b66aae 100644 --- a/benchmark/timeseries/model.py +++ b/benchmark/timeseries/model.py @@ -15,6 +15,7 @@ class ModelConfig: factory: Callable[[torch.device], sdm.models.ICLModel] num_estimators: int = 1 max_context_size: int | None = None + max_variates: int | None = None MODEL_CONFIGS = { @@ -26,5 +27,6 @@ class ModelConfig: ), num_estimators=2, max_context_size=15_360, + max_variates=32, ), } diff --git a/docs/source/icl.md b/docs/source/icl.md index 8a34ba05d0..b4a1e1e588 100644 --- a/docs/source/icl.md +++ b/docs/source/icl.md @@ -98,7 +98,6 @@ Predictions are returned as a general {py:class}`~sdm.tensor.TableTensor`, where Since predictions are returned as a {py:class}`~sdm.tensor.TableTensor`, you can zero-copy them to [`pandas`](https://pandas.pydata.org/docs), [`arrow`](https://arrow.apache.org/docs), or [`cudf`](https://docs.rapids.ai/api/cudf) via {py:meth}`~sdm.tensor.TableTensor.to_pandas`, {py:meth}`~sdm.tensor.TableTensor.to_arrow`, and {py:meth}`~sdm.tensor.TableTensor.to_cudf` for further downstream processing. In order to simplify metric calculation (*e.g.*, via [`torchmetrics`](https://lightning.ai/docs/torchmetrics)), we provide helper functions in the [`sdm.evaluation`](api/evaluation) package to convert target columns to class indices and align prediction columns to them (see {py:func}`~sdm.evaluation.to_class_indices` and {py:func}`~sdm.evaluation.to_binary_class`). -Similarly, {py:func}`~sdm.evaluation.to_quantiles` converts quantile predictions of one or more targets into a single tensor together with their quantile levels. ## Ensembling diff --git a/sdm/evaluation/__init__.py b/sdm/evaluation/__init__.py index 08304db084..7d3b78251e 100644 --- a/sdm/evaluation/__init__.py +++ b/sdm/evaluation/__init__.py @@ -4,10 +4,8 @@ """Evaluation utilities.""" from sdm.evaluation.classification import to_class_indices, to_binary_class -from sdm.evaluation.regression import to_quantiles __all__ = [ "to_class_indices", "to_binary_class", - "to_quantiles", ] diff --git a/sdm/evaluation/regression.py b/sdm/evaluation/regression.py deleted file mode 100644 index 8ff24bc101..0000000000 --- a/sdm/evaluation/regression.py +++ /dev/null @@ -1,61 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -from torch import Tensor - -from sdm import Stype, TableTensor - - -def to_quantiles(pred: TableTensor) -> tuple[Tensor, Tensor]: - r"""Convert quantile predictions to a per-target quantile tensor. - - Numerical columns must be named ``"q"`` for a single target or - ``"__q"`` for multiple targets, where the digits encode the - quantile level as a decimal fraction, *e.g.*, ``"q500"`` and ``"q50"`` both - denote the median. - Columns are expected to be grouped by target, with the same quantile levels - in the same order for every target. - - Args: - pred: Prediction table whose numerical columns contain quantiles. - - Returns: - The quantile predictions with shape ``[..., V, Q]`` for ``V`` targets - and ``Q`` quantile levels, and the quantile levels with shape ``[Q]``. - - >>> import torch - >>> import sdm - >>> from sdm.evaluation import to_quantiles - >>> pred = sdm.TableTensor.from_tensor( - ... torch.tensor([[1.0, 2.0, 3.0, 4.0]]), - ... columns=["q100", "q400", "q700", "q900"], - ... ) - >>> values, levels = to_quantiles(pred) - >>> values - tensor([[[1., 2., 3., 4.]]]) - >>> levels - tensor([0.1000, 0.4000, 0.7000, 0.9000]) - >>> pred = sdm.TableTensor.from_tensor( - ... torch.tensor([[1.0, 2.0, 3.0, 4.0]]), - ... columns=["a__q10", "a__q90", "b__q10", "b__q90"], - ... ) - >>> values, levels = to_quantiles(pred) - >>> values - tensor([[[1., 2.], - [3., 4.]]]) - >>> levels - tensor([0.1000, 0.9000]) - """ - columns = pred.columns[Stype.numerical] - prefixes, _, digits = zip(*(column.rpartition("q") for column in columns)) - targets = dict.fromkeys(prefixes) - digits = digits[: len(columns) // len(targets)] - if columns != tuple(f"{t}q{d}" for t in targets for d in digits): - raise ValueError( - "Expected quantile columns grouped by target with the same " - f"quantile levels (got {list(columns)})" - ) - - levels = [int(d) / 10 ** len(d) for d in digits] - values = pred.numerical.unflatten(-1, (len(targets), -1)) # [..., V, Q] - return values, values.new_tensor(levels) diff --git a/test/evaluation/test_regression.py b/test/evaluation/test_regression.py deleted file mode 100644 index 9ae0941a8f..0000000000 --- a/test/evaluation/test_regression.py +++ /dev/null @@ -1,26 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -import pytest -import torch - -import sdm - - -def test_to_quantiles() -> None: - pred = sdm.TableTensor.from_tensor( - torch.randn(2, 5, 4), - columns=["b__q10", "b__q90", "a__q10", "a__q90"], - ) - values, levels = sdm.evaluation.to_quantiles(pred) - torch.testing.assert_close(values, pred.numerical.view(2, 5, 2, 2)) - torch.testing.assert_close(levels, torch.tensor([0.1, 0.9])) - - -def test_to_quantiles_invalid() -> None: - pred = sdm.TableTensor.from_tensor( - torch.randn(5, 4), - columns=["a__q10", "b__q10", "a__q90", "b__q90"], - ) - with pytest.raises(ValueError, match="grouped by target"): - sdm.evaluation.to_quantiles(pred) From 633c823990fd17f2169457759d0fbbaa313bea5e Mon Sep 17 00:00:00 2001 From: rusty1s Date: Mon, 26 Oct 2026 04:53:42 +0000 Subject: [PATCH 6/7] update --- benchmark/timeseries/gift_eval/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/benchmark/timeseries/gift_eval/README.md b/benchmark/timeseries/gift_eval/README.md index fb1cd669d3..b0117fad6f 100644 --- a/benchmark/timeseries/gift_eval/README.md +++ b/benchmark/timeseries/gift_eval/README.md @@ -19,12 +19,12 @@ pip install --no-deps "git+https://github.com/SalesforceAIResearch/gift-eval.git python -m benchmark.timeseries.gift_eval.main --model timesfm3 ``` - > [!NOTE] - > Weights of `TimesFM3` are distributed under the [TimesFM Non-Commercial License v1.0](https://huggingface.co/google/timesfm-3.0-pytorch/blob/main/LICENSE). - > Review the license before running the `TimesFM3` benchmark, which will download its weights noninteractively. - `TimesFM3` follows official evaluation with symmetric averaging over two estimators, contexts truncated to 15,360 time steps and up to 32 variates per forward pass. +> [!NOTE] +> Weights of `TimesFM3` are distributed under the [TimesFM Non-Commercial License v1.0](https://huggingface.co/google/timesfm-3.0-pytorch/blob/main/LICENSE). +> Review the license before running the `TimesFM3` benchmark, which will download its weights noninteractively. + Results will be written to `benchmark/timeseries/gift_eval_out//`, and already completed configurations are skipped. ## Evaluate From e60c95993ac6f0d4867d9d3c29a502ca0b4f4f25 Mon Sep 17 00:00:00 2001 From: rusty1s Date: Mon, 26 Oct 2026 05:09:55 +0000 Subject: [PATCH 7/7] update --- benchmark/timeseries/gift_eval/predictor.py | 63 +++++++++------------ 1 file changed, 27 insertions(+), 36 deletions(-) diff --git a/benchmark/timeseries/gift_eval/predictor.py b/benchmark/timeseries/gift_eval/predictor.py index 67a8be9785..52d801bc94 100644 --- a/benchmark/timeseries/gift_eval/predictor.py +++ b/benchmark/timeseries/gift_eval/predictor.py @@ -3,7 +3,6 @@ from collections.abc import Iterator -import numpy as np import torch from gluonts.dataset import Dataset from gluonts.itertools import batcher @@ -37,12 +36,13 @@ def predict( ) -> Iterator[QuantileForecast]: for batch in batcher(dataset, batch_size=self.batch_size): + cpu_generator = torch.Generator().manual_seed(42) + ys = [torch.from_numpy(entry["target"]) for entry in batch] ys = [torch.atleast_2d(y) for y in ys] xs = [] if ( self.model.supports_context_only_columns - and sdm.Stype.numerical in self.model.supported_feature_stypes and "past_feat_dynamic_real" in batch[0] ): xs = [ @@ -69,18 +69,12 @@ def predict( for i, x_i in enumerate(xs): x[i, :, -x_i.size(-1) :] = x_i - size = y.size(1) - if self.config.max_variates is not None: - # Keep room for at least one target, matching upstream's - # fixed-seed selection of past covariates. - count = min(x.size(1), self.config.max_variates - 1) - if count < x.size(1): - rng = np.random.default_rng(42) - index = np.sort( - rng.choice(x.size(1), count, replace=False) - ) - x = x[:, index] - size = min(size, self.config.max_variates - count) + if ( + self.config.max_variates is not None + and x.size(1) > self.config.max_variates - 1 + ): # Keep room for at least one target: + perm = torch.randperm(x.size(1), generator=cpu_generator) + x = x[:, perm[: self.config.max_variates - 1]] y = y.transpose(-1, -2).to( self.device, @@ -91,33 +85,30 @@ def predict( memory_format=torch.contiguous_format, ) - x_query = x.new_empty( - (x.size(0), self.prediction_length, 0), - ) - predictions = [] + size = y.size(-1) + if self.config.max_variates is not None: + size = min(size, self.config.max_variates - x.size(-1)) + + preds = [] for start in range(0, y.size(-1), size): - chunk = y[..., start : start + size] - # Upstream fills the last chunk with leading target channels. - if chunk.size(-1) < size: - chunk = torch.cat( - [chunk, y[..., : size - chunk.size(-1)]], dim=-1 - ) + y_i = y[..., start : start + size] + if y_i.size(-1) < size: + y_i = torch.cat([y_i, y[..., : size - y_i.size(-1)]], -1) + out = self.model( x_context=x, - y_context=chunk, - x_query=x_query, + y_context=y_i, + x_query=torch.empty( + (x.size(0), self.prediction_length, 0), + device=self.device, + ), num_estimators=self.config.num_estimators, ) - predictions.append( - out.numerical.view( - y.size(0), self.prediction_length, size, -1 - ) - ) - pred = ( - predictions[0] - if len(predictions) == 1 - else torch.cat(predictions, dim=-2)[..., : y.size(-1), :] - ) + pred = out.numerical + pred = pred.view(y.size(0), self.prediction_length, size, -1) + preds.append(pred) + + pred = torch.cat(preds, dim=-2)[..., : y.size(-1), :] pred = pred.permute(0, 3, 1, 2).cpu().numpy() # [B, Q, R, C] levels = out.numerical_columns[: pred.shape[1]] levels = [str(float(f"0.{q.split('q')[-1]}")) for q in levels]