From 8c25ce03d405f62179966c603c9a5449104eca51 Mon Sep 17 00:00:00 2001 From: Filippo Guerranti Date: Tue, 13 Oct 2026 09:02:43 +0000 Subject: [PATCH 01/11] Add `sdm.evaluation.to_quantiles` --- docs/source/icl.md | 1 + sdm/evaluation/__init__.py | 2 + sdm/evaluation/regression.py | 61 ++++++++++++++++++++++++++++++ test/evaluation/test_regression.py | 26 +++++++++++++ 4 files changed, 90 insertions(+) create mode 100644 sdm/evaluation/regression.py create mode 100644 test/evaluation/test_regression.py diff --git a/docs/source/icl.md b/docs/source/icl.md index c8174aee4..d8e19fd70 100644 --- a/docs/source/icl.md +++ b/docs/source/icl.md @@ -98,6 +98,7 @@ Predictions are returned as a general {py:class}`~sdm.tensor.TableTensor`, where Since predictions are returned as {py:class}`~sdm.tensor.TableTensor`, you can zero-copy them to [`pandas`](https://pandas.pydata.org/docs), [`arrow`](https://arrow.apache.org/docs), or [`cudf`](https://docs.rapids.ai/api/cudf) via {py:meth}`~sdm.tensor.TableTensor.to_pandas`, {py:meth}`~sdm.tensor.TableTensor.to_arrow`, and {py:meth}`~sdm.tensor.TableTensor.to_cudf` for further downstream processing. In order to simplify metric calculation (*e.g.*, via [`torchmetrics`](https://lightning.ai/docs/torchmetrics)), we provide helper functions in the [`sdm.evaluation`](api/evaluation) package to convert target columns to class indices and align prediction columns to them (see {py:func}`~sdm.evaluation.to_class_indices` and {py:func}`~sdm.evaluation.to_binary_class`). +Similarly, {py:func}`~sdm.evaluation.to_quantiles` converts quantile predictions of one or more targets into a single tensor together with their quantile levels. ## Ensembling diff --git a/sdm/evaluation/__init__.py b/sdm/evaluation/__init__.py index 7d3b78251..08304db08 100644 --- a/sdm/evaluation/__init__.py +++ b/sdm/evaluation/__init__.py @@ -4,8 +4,10 @@ """Evaluation utilities.""" from sdm.evaluation.classification import to_class_indices, to_binary_class +from sdm.evaluation.regression import to_quantiles __all__ = [ "to_class_indices", "to_binary_class", + "to_quantiles", ] diff --git a/sdm/evaluation/regression.py b/sdm/evaluation/regression.py new file mode 100644 index 000000000..8ff24bc10 --- /dev/null +++ b/sdm/evaluation/regression.py @@ -0,0 +1,61 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from torch import Tensor + +from sdm import Stype, TableTensor + + +def to_quantiles(pred: TableTensor) -> tuple[Tensor, Tensor]: + r"""Convert quantile predictions to a per-target quantile tensor. + + Numerical columns must be named ``"q"`` for a single target or + ``"__q"`` for multiple targets, where the digits encode the + quantile level as a decimal fraction, *e.g.*, ``"q500"`` and ``"q50"`` both + denote the median. + Columns are expected to be grouped by target, with the same quantile levels + in the same order for every target. + + Args: + pred: Prediction table whose numerical columns contain quantiles. + + Returns: + The quantile predictions with shape ``[..., V, Q]`` for ``V`` targets + and ``Q`` quantile levels, and the quantile levels with shape ``[Q]``. + + >>> import torch + >>> import sdm + >>> from sdm.evaluation import to_quantiles + >>> pred = sdm.TableTensor.from_tensor( + ... torch.tensor([[1.0, 2.0, 3.0, 4.0]]), + ... columns=["q100", "q400", "q700", "q900"], + ... ) + >>> values, levels = to_quantiles(pred) + >>> values + tensor([[[1., 2., 3., 4.]]]) + >>> levels + tensor([0.1000, 0.4000, 0.7000, 0.9000]) + >>> pred = sdm.TableTensor.from_tensor( + ... torch.tensor([[1.0, 2.0, 3.0, 4.0]]), + ... columns=["a__q10", "a__q90", "b__q10", "b__q90"], + ... ) + >>> values, levels = to_quantiles(pred) + >>> values + tensor([[[1., 2.], + [3., 4.]]]) + >>> levels + tensor([0.1000, 0.9000]) + """ + columns = pred.columns[Stype.numerical] + prefixes, _, digits = zip(*(column.rpartition("q") for column in columns)) + targets = dict.fromkeys(prefixes) + digits = digits[: len(columns) // len(targets)] + if columns != tuple(f"{t}q{d}" for t in targets for d in digits): + raise ValueError( + "Expected quantile columns grouped by target with the same " + f"quantile levels (got {list(columns)})" + ) + + levels = [int(d) / 10 ** len(d) for d in digits] + values = pred.numerical.unflatten(-1, (len(targets), -1)) # [..., V, Q] + return values, values.new_tensor(levels) diff --git a/test/evaluation/test_regression.py b/test/evaluation/test_regression.py new file mode 100644 index 000000000..9ae0941a8 --- /dev/null +++ b/test/evaluation/test_regression.py @@ -0,0 +1,26 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import pytest +import torch + +import sdm + + +def test_to_quantiles() -> None: + pred = sdm.TableTensor.from_tensor( + torch.randn(2, 5, 4), + columns=["b__q10", "b__q90", "a__q10", "a__q90"], + ) + values, levels = sdm.evaluation.to_quantiles(pred) + torch.testing.assert_close(values, pred.numerical.view(2, 5, 2, 2)) + torch.testing.assert_close(levels, torch.tensor([0.1, 0.9])) + + +def test_to_quantiles_invalid() -> None: + pred = sdm.TableTensor.from_tensor( + torch.randn(5, 4), + columns=["a__q10", "b__q10", "a__q90", "b__q90"], + ) + with pytest.raises(ValueError, match="grouped by target"): + sdm.evaluation.to_quantiles(pred) From 1f4c5bbcf9c51867214ed78a9c596f4fb213b363 Mon Sep 17 00:00:00 2001 From: Filippo Guerranti Date: Tue, 13 Oct 2026 09:33:10 +0000 Subject: [PATCH 02/11] Add GIFT-Eval forecasting benchmark --- .gitignore | 3 + benchmark/README.md | 2 + benchmark/forecasting/README.md | 51 ++++++ benchmark/forecasting/__init__.py | 2 + benchmark/forecasting/gift_eval/__init__.py | 8 + benchmark/forecasting/gift_eval/evaluate.py | 62 +++++++ benchmark/forecasting/gift_eval/main.py | 172 ++++++++++++++++++++ benchmark/forecasting/gift_eval/models.py | 85 ++++++++++ benchmark/forecasting/model.py | 118 ++++++++++++++ 9 files changed, 503 insertions(+) create mode 100644 benchmark/forecasting/README.md create mode 100644 benchmark/forecasting/__init__.py create mode 100644 benchmark/forecasting/gift_eval/__init__.py create mode 100644 benchmark/forecasting/gift_eval/evaluate.py create mode 100644 benchmark/forecasting/gift_eval/main.py create mode 100644 benchmark/forecasting/gift_eval/models.py create mode 100644 benchmark/forecasting/model.py diff --git a/.gitignore b/.gitignore index 73fa2fdd2..8790ab32a 100644 --- a/.gitignore +++ b/.gitignore @@ -221,5 +221,8 @@ scoringbench_out/ evals/ datasets.json +# Forecasting benchmarks +gift_eval_out/ + # Checkpoint files *.pt diff --git a/benchmark/README.md b/benchmark/README.md index 964c325f6..36232fd7c 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -9,3 +9,5 @@ This folder contains benchmark integrations for `structured-data-models`: - [**`talent/`**](tabular/talent/): [TALENT](https://github.com/LAMDA-Tabular/TALENT) - [**`relational/`**](relational/): Relational benchmarks - [**`relarena/`**](relational/relarena): [RelArena](https://github.com/PriorLabs/relarena) +- [**`forecasting/`**](forecasting/): Time-series forecasting benchmarks + - [**`gift_eval/`**](forecasting/): [GIFT-Eval](https://github.com/SalesforceAIResearch/gift-eval) diff --git a/benchmark/forecasting/README.md b/benchmark/forecasting/README.md new file mode 100644 index 000000000..aee5f50a8 --- /dev/null +++ b/benchmark/forecasting/README.md @@ -0,0 +1,51 @@ +# Time-Series Forecasting Benchmarks + +This directory contains benchmarks of `structured-data-models` on [GIFT-Eval](https://github.com/SalesforceAIResearch/gift-eval). + +All benchmarks share one model adapter (see `benchmark/forecasting/model.py`) that maps each forecasting task onto the `ICLModel` interface: time steps become rows, past targets become `y_context`, all covariates and timestamps of the context become `x_context`, and the forecast horizon becomes `x_query` with one row of known covariates and timestamps per future time step. Columns keep their semantic types, so each model decides which ones it consumes. Series in a batch are left-padded with missing values, and multivariate series are flattened for models without multi-target support. + +______________________________________________________________________ + +> [!NOTE] +> Weights of `TimesFM3` are distributed under the [TimesFM Non-Commercial License v1.0](https://huggingface.co/google/timesfm-3.0-pytorch/blob/main/LICENSE). +> Review the license before running the `TimesFM3` benchmark, which will download its weights noninteractively. + +## GIFT-Eval + +### Setup + +Run the commands below from the repository root: + +```bash +pip install . gluonts datasets python-dotenv +pip install --no-deps "git+https://github.com/SalesforceAIResearch/gift-eval.git@577a65cfa3acb812405c00d989230bf59f7f8a3d" + +hf download Salesforce/GiftEval --repo-type dataset --local-dir /path/to/gift_eval +export GIFT_EVAL=/path/to/gift_eval +``` + +GIFT-Eval is not released on PyPI and is therefore pinned to a commit. It is installed without dependencies since it pins outdated `numpy` and `datasets` versions. + +### Run + +```bash +python -m benchmark.forecasting.gift_eval.main --model timesfm3 +``` + +Pass `--dataset` to run only selected GIFT-Eval datasets, *e.g.*, `--dataset hospital --dataset ett1/H`. +Use `--batch_size` to set the number of series per model call and `--max_context_size` to truncate long contexts. +Results are written to `benchmark/forecasting/gift_eval_out//`, and already completed configurations are skipped. + +### Evaluate + +```bash +python -m benchmark.forecasting.gift_eval.evaluate +``` + +This compares all available SDM results against published GIFT-Eval results on the same configurations, normalized by a seasonal naive baseline and aggregated by geometric mean. + +`TimesFM3` follows the published `TimesFM-3` evaluation with symmetric averaging over two estimators and contexts truncated to 15,360 time steps. Its results can still differ from the published ones: + +- Published results batch series of different context lengths together, which shifts their positional encodings on datasets with varying context lengths. +- Published results clip forecasts of non-negative series at zero. +- Published results forecast at most 32 variates jointly, subsampling covariates of `bizitobs_application` and `bizitobs_service`. diff --git a/benchmark/forecasting/__init__.py b/benchmark/forecasting/__init__.py new file mode 100644 index 000000000..d51c4fe1e --- /dev/null +++ b/benchmark/forecasting/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/benchmark/forecasting/gift_eval/__init__.py b/benchmark/forecasting/gift_eval/__init__.py new file mode 100644 index 000000000..417150a40 --- /dev/null +++ b/benchmark/forecasting/gift_eval/__init__.py @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Pinned GIFT-Eval revision for dataset properties and reference results: +GIFT_EVAL_URL = ( + "https://raw.githubusercontent.com/SalesforceAIResearch/gift-eval/" + "577a65cfa3acb812405c00d989230bf59f7f8a3d" +) diff --git a/benchmark/forecasting/gift_eval/evaluate.py b/benchmark/forecasting/gift_eval/evaluate.py new file mode 100644 index 000000000..d6176985d --- /dev/null +++ b/benchmark/forecasting/gift_eval/evaluate.py @@ -0,0 +1,62 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +r"""Evaluate SDM forecasting model results on GIFT-Eval.""" + +from pathlib import Path + +import numpy as np +import pandas as pd + +from benchmark.forecasting.gift_eval import GIFT_EVAL_URL +from benchmark.forecasting.model import MODEL_CONFIGS + +BASELINE = "Seasonal_Naive" +REFERENCES = ("TimesFM-3", "TimesFM-2.5", "Chronos-2", BASELINE) +METRICS = { + "eval_metrics/MASE[0.5]": "MASE", + "eval_metrics/mean_weighted_sum_quantile_loss": "CRPS", +} + + +def main() -> None: + benchmark_dir = Path(__file__).parent.parent + result_root = benchmark_dir / "gift_eval_out" + output_root = benchmark_dir / "evals" / "gift_eval" + + results = [] + for model_config in MODEL_CONFIGS.values(): + path = result_root / model_config.name / "all_results.csv" + if path.exists(): + results.append(pd.read_csv(path)) + + if not results: + raise FileNotFoundError( + f"No GIFT-Eval results found under {result_root}" + ) + + # Compare with published results on configurations run by all models: + configs = set.intersection(*(set(result["dataset"]) for result in results)) + for name in REFERENCES: + path = f"{GIFT_EVAL_URL}/results/{name}/all_results.csv" + results.append(pd.read_csv(path)) + + df = pd.concat(results) + df = df[df["dataset"].isin(configs)].set_index(["model", "dataset"]) + df = df[list(METRICS)].rename(columns=METRICS) + + # Normalize by the seasonal naive baseline and aggregate by geometric mean: + log_scores = np.log(df.div(df.loc[BASELINE], level="dataset")) + terms = log_scores.index.get_level_values("dataset").str.split("/").str[-1] + + output_root.mkdir(parents=True, exist_ok=True) + for term, scores in [("overall", log_scores), *log_scores.groupby(terms)]: + leaderboard = np.exp(scores.groupby("model").mean()) + leaderboard = leaderboard.sort_values("CRPS") + leaderboard.to_csv(output_root / f"leaderboard_{term}.csv") + print(f"\nGIFT-Eval leaderboard ({term}):") + print(leaderboard.to_string(float_format="%.4f")) + + +if __name__ == "__main__": + main() diff --git a/benchmark/forecasting/gift_eval/main.py b/benchmark/forecasting/gift_eval/main.py new file mode 100644 index 000000000..513815328 --- /dev/null +++ b/benchmark/forecasting/gift_eval/main.py @@ -0,0 +1,172 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +r"""Run an SDM forecasting model on GIFT-Eval.""" + +import argparse +from dataclasses import replace +from pathlib import Path + +import pandas as pd +import torch +from gift_eval.data import Dataset +from gluonts.ev.metrics import ( + MAE, + MAPE, + MASE, + MSE, + MSIS, + ND, + NRMSE, + RMSE, + SMAPE, + MeanWeightedSumQuantileLoss, +) +from gluonts.model import evaluate_model +from gluonts.time_feature import get_seasonality + +from benchmark.forecasting.gift_eval import GIFT_EVAL_URL +from benchmark.forecasting.gift_eval.models import ( + QUANTILE_LEVELS, + SDMPredictor, +) +from benchmark.forecasting.model import MODEL_CONFIGS, add_model_args + +# Datasets and terms from GIFT-Eval's notebooks: +SHORT_DATASETS = ( # noqa: SIM905 + "m4_yearly m4_quarterly m4_monthly m4_weekly m4_daily m4_hourly " + "electricity/15T electricity/H electricity/D electricity/W " + "solar/10T solar/H solar/D solar/W hospital covid_deaths " + "us_births/D us_births/M us_births/W " + "saugeenday/D saugeenday/M saugeenday/W " + "temperature_rain_with_missing " + "kdd_cup_2018_with_missing/H kdd_cup_2018_with_missing/D " + "car_parts_with_missing restaurant " + "hierarchical_sales/D hierarchical_sales/W " + "LOOP_SEATTLE/5T LOOP_SEATTLE/H LOOP_SEATTLE/D SZ_TAXI/15T SZ_TAXI/H " + "M_DENSE/H M_DENSE/D ett1/15T ett1/H ett1/D ett1/W " + "ett2/15T ett2/H ett2/D ett2/W jena_weather/10T jena_weather/H " + "jena_weather/D bitbrains_fast_storage/5T bitbrains_fast_storage/H " + "bitbrains_rnd/5T bitbrains_rnd/H bizitobs_application bizitobs_service " + "bizitobs_l2c/5T bizitobs_l2c/H" +).split() +MED_LONG_DATASETS = ( # noqa: SIM905 + "electricity/15T electricity/H solar/10T solar/H " + "kdd_cup_2018_with_missing/H LOOP_SEATTLE/5T LOOP_SEATTLE/H SZ_TAXI/15T " + "M_DENSE/H ett1/15T ett1/H ett2/15T ett2/H jena_weather/10T " + "jena_weather/H bitbrains_fast_storage/5T bitbrains_rnd/5T " + "bizitobs_application bizitobs_service bizitobs_l2c/5T bizitobs_l2c/H" +).split() +PRETTY_NAMES = { + "saugeenday": "saugeen", + "temperature_rain_with_missing": "temperature_rain", + "kdd_cup_2018_with_missing": "kdd_cup_2018", + "car_parts_with_missing": "car_parts", +} +METRICS = [ + MSE(forecast_type="mean"), + MSE(forecast_type="0.5"), + MAE(), + MASE(), + MAPE(), + SMAPE(), + MSIS(), + RMSE(), + NRMSE(), + ND(), + MeanWeightedSumQuantileLoss(quantile_levels=QUANTILE_LEVELS), +] + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + add_model_args(parser) + parser.add_argument( + "--dataset", + action="append", + help="Run only the selected GIFT-Eval dataset; repeat to combine.", + ) + args = parser.parse_args() + + model_config = MODEL_CONFIGS[args.model] + if args.max_context_size is not None: + model_config = replace( + model_config, max_context_size=args.max_context_size + ) + result_path = ( + Path(__file__).parent.parent + / "gift_eval_out" + / model_config.name + / "all_results.csv" + ) + result_path.parent.mkdir(parents=True, exist_ok=True) + completed = ( + set(pd.read_csv(result_path)["dataset"]) + if result_path.exists() + else set() + ) + properties = pd.read_json( + f"{GIFT_EVAL_URL}/notebooks/dataset_properties.json" + ) + + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + model = model_config.factory(device) + + for name in args.dataset or SHORT_DATASETS: + key, _, freq = name.partition("/") + key = PRETTY_NAMES.get(key.lower(), key.lower()) + freq = freq or properties[key]["frequency"] + for term in ("short", "medium", "long"): + if term != "short" and name not in MED_LONG_DATASETS: + continue + config = f"{key}/{freq}/{term}" + if config in completed: + continue + + # Flatten multivariate series for models without joint multi-target + # forecasting: + dataset = Dataset( + name=name, + term=term, + to_univariate=( + not model.supports_multi_target + and properties[key]["num_variates"] > 1 + ), + ) + predictor = SDMPredictor( + model, + config=model_config, + prediction_length=dataset.prediction_length, + batch_size=args.batch_size, + ) + result = evaluate_model( + predictor, + test_data=dataset.test_data, + metrics=METRICS, + seasonality=get_seasonality(dataset.freq), + ) + row = pd.DataFrame( + [ + { + "dataset": config, + "model": f"[SDM] {model_config.name}", + **{ + f"eval_metrics/{k}": v + for k, v in result.iloc[0].items() + }, + "domain": properties[key]["domain"], + "num_variates": properties[key]["num_variates"], + } + ] + ) + row.to_csv( + result_path, + mode="a", + header=not result_path.exists(), + index=False, + ) + print(row.to_string(index=False)) + + +if __name__ == "__main__": + main() diff --git a/benchmark/forecasting/gift_eval/models.py b/benchmark/forecasting/gift_eval/models.py new file mode 100644 index 000000000..c14b7f254 --- /dev/null +++ b/benchmark/forecasting/gift_eval/models.py @@ -0,0 +1,85 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""SDM forecasting adapter for GIFT-Eval.""" + +from collections.abc import Iterator + +import numpy as np +import pandas as pd +from gluonts.dataset import Dataset +from gluonts.itertools import batcher +from gluonts.model.forecast import QuantileForecast +from gluonts.model.predictor import Predictor + +import sdm +from benchmark.forecasting.model import ModelConfig, forecast, select_levels + +QUANTILE_LEVELS = (0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9) + + +class SDMPredictor(Predictor): + """GluonTS predictor forecasting batches of series with an SDM model.""" + + def __init__( + self, + model: sdm.models.ICLModel, + config: ModelConfig, + prediction_length: int, + batch_size: int, + ) -> None: + super().__init__(prediction_length=prediction_length) + self.model = model + self.config = config + self.batch_size = batch_size + + def predict( + self, + dataset: Dataset, + **kwargs: object, + ) -> Iterator[QuantileForecast]: + for batch in batcher(dataset, batch_size=self.batch_size): + # GluonTS stores series as [V, T] (or [T] if univariate): + num_variates = np.atleast_2d(batch[0]["target"]).shape[0] + targets = [f"target_{i}" for i in range(num_variates)] + past, future = [], [] + for entry in batch: + target = np.atleast_2d(entry["target"]) + length = target.shape[-1] + timestamp = pd.period_range( + entry["start"], + periods=length + self.prediction_length, + ).to_timestamp() + columns = { + "item_id": entry["item_id"], + "timestamp": timestamp[:length], + } + columns |= dict(zip(targets, target, strict=True)) + for i, x in enumerate(entry.get("past_feat_dynamic_real", [])): + columns[f"past_feat_dynamic_real_{i}"] = x + past.append(pd.DataFrame(columns)) + future.append( + pd.DataFrame( + { + "item_id": entry["item_id"], + "timestamp": timestamp[length:], + } + ) + ) + values, levels = forecast( + self.model, + config=self.config, + past=past, + future=future, + targets=targets, + ) + values = select_levels(values, levels, QUANTILE_LEVELS) + for entry, value in zip(batch, values, strict=True): + value = value.transpose(2, 0, 1) # [Q, H, V] + if entry["target"].ndim == 1: + value = value[..., 0] + yield QuantileForecast( + forecast_arrays=value, + forecast_keys=[str(q) for q in QUANTILE_LEVELS], + start_date=entry["start"] + entry["target"].shape[-1], + ) diff --git a/benchmark/forecasting/model.py b/benchmark/forecasting/model.py new file mode 100644 index 000000000..13304709b --- /dev/null +++ b/benchmark/forecasting/model.py @@ -0,0 +1,118 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""SDM model adapter shared by the time-series forecasting benchmarks.""" + +import argparse +from collections.abc import Callable, Sequence +from dataclasses import dataclass + +import numpy as np +import pandas as pd +import torch + +import sdm +from sdm.models.timesfm3 import TimesFM3 + + +@dataclass(frozen=True) +class ModelConfig: + name: str + factory: Callable[[torch.device], sdm.models.ICLModel] + num_estimators: int = 1 + max_context_size: int | None = None + + +MODEL_CONFIGS = { + "timesfm3": ModelConfig( + name="TimesFM3", + factory=lambda device: TimesFM3(accept_license=True, device=device), + num_estimators=2, + max_context_size=15360, + ), +} + + +def add_model_args(parser: argparse.ArgumentParser) -> None: + """Add the model flags shared by all forecasting benchmarks.""" + parser.add_argument( + "--model", + choices=tuple(MODEL_CONFIGS), + default="timesfm3", + help="SDM model to benchmark.", + ) + parser.add_argument( + "--batch_size", + type=int, + default=64, + help="Number of series forecast per model call.", + ) + parser.add_argument( + "--max_context_size", + type=int, + help="Truncate each context to its most recent time steps " + "(defaults to the model's maximum context size).", + ) + + +def forecast( + model: sdm.models.ICLModel, + config: ModelConfig, + past: Sequence[pd.DataFrame], # B x [T_i, V + C] + future: Sequence[pd.DataFrame], # B x [H, K] + targets: Sequence[str], +) -> tuple[np.ndarray, np.ndarray]: # [B, H, V, Q], [Q] + """Forecast the quantiles of a batch of (multivariate) time series. + + Time steps map to rows: ``past`` holds the targets and all covariates of + the context, while ``future`` holds the covariates known over the + forecast horizon. Columns keep their semantic types, so that the model + decides which ones to consume. Missing values are passed through. + """ + device = next(model.parameters()).device + num_series, horizon = len(past), len(future[0]) + size = max(len(df) for df in past) + if config.max_context_size is not None: + size = min(size, config.max_context_size) + + # Left-pad shorter series with missing values so that all series end at + # the forecast start: + past = [df.iloc[-size:].reset_index(drop=True) for df in past] + past = [df.reindex(range(len(df) - size, len(df))) for df in past] + + # Infer semantic types jointly so that context and query share them: + df = pd.concat([*past, *future], ignore_index=True) + table = sdm.TableTensor.from_pandas( + df=df, + stypes=sdm.infer_stypes(df, id="infer"), + device=device, + ) + # Split rows into the context [B, T, V + C] and the query [B, H, V + C]: + context = table[: num_series * size].view(num_series, size, -1) + query = table[num_series * size :].view(num_series, horizon, -1) + assert isinstance(context, sdm.TableTensor) + assert isinstance(query, sdm.TableTensor) + + out = model( + x_context=context.drop_columns(targets), + y_context=context.select_columns(targets), + x_query=query.select_columns(future[0].columns), + num_estimators=config.num_estimators, + ) + values, levels = sdm.evaluation.to_quantiles(out) + return values.cpu().numpy(), levels.cpu().numpy() + + +def select_levels( + values: np.ndarray, # [..., Q] + levels: np.ndarray, # [Q] + wanted: Sequence[float], +) -> np.ndarray: # [..., len(wanted)] + """Select the quantile levels requested by a benchmark.""" + index = [int(np.abs(levels - level).argmin()) for level in wanted] + if not np.allclose(levels[index], wanted): + raise ValueError( + f"Model does not predict quantile levels {list(wanted)} " + f"(got {levels.tolist()})" + ) + return values[..., index] From 7497cf1d92cc435890cb9017c9075235bcfe73b0 Mon Sep 17 00:00:00 2001 From: Filippo Guerranti Date: Tue, 13 Oct 2026 15:59:29 +0000 Subject: [PATCH 03/11] Add fev-bench forecasting benchmark --- .gitignore | 1 + benchmark/README.md | 1 + benchmark/forecasting/README.md | 35 ++++++++- benchmark/forecasting/fev_bench/__init__.py | 5 ++ benchmark/forecasting/fev_bench/evaluate.py | 60 ++++++++++++++ benchmark/forecasting/fev_bench/main.py | 87 +++++++++++++++++++++ benchmark/forecasting/fev_bench/models.py | 87 +++++++++++++++++++++ benchmark/forecasting/model.py | 4 +- 8 files changed, 278 insertions(+), 2 deletions(-) create mode 100644 benchmark/forecasting/fev_bench/__init__.py create mode 100644 benchmark/forecasting/fev_bench/evaluate.py create mode 100644 benchmark/forecasting/fev_bench/main.py create mode 100644 benchmark/forecasting/fev_bench/models.py diff --git a/.gitignore b/.gitignore index 8790ab32a..aea92019b 100644 --- a/.gitignore +++ b/.gitignore @@ -223,6 +223,7 @@ datasets.json # Forecasting benchmarks gift_eval_out/ +fev_bench_out/ # Checkpoint files *.pt diff --git a/benchmark/README.md b/benchmark/README.md index 36232fd7c..8ce0c1c59 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -11,3 +11,4 @@ This folder contains benchmark integrations for `structured-data-models`: - [**`relarena/`**](relational/relarena): [RelArena](https://github.com/PriorLabs/relarena) - [**`forecasting/`**](forecasting/): Time-series forecasting benchmarks - [**`gift_eval/`**](forecasting/): [GIFT-Eval](https://github.com/SalesforceAIResearch/gift-eval) + - [**`fev_bench/`**](forecasting/): [fev-bench](https://github.com/autogluon/fev) diff --git a/benchmark/forecasting/README.md b/benchmark/forecasting/README.md index aee5f50a8..e15986cd2 100644 --- a/benchmark/forecasting/README.md +++ b/benchmark/forecasting/README.md @@ -1,6 +1,6 @@ # Time-Series Forecasting Benchmarks -This directory contains benchmarks of `structured-data-models` on [GIFT-Eval](https://github.com/SalesforceAIResearch/gift-eval). +This directory contains benchmarks of `structured-data-models` on [GIFT-Eval](https://github.com/SalesforceAIResearch/gift-eval) and [fev-bench](https://github.com/autogluon/fev). All benchmarks share one model adapter (see `benchmark/forecasting/model.py`) that maps each forecasting task onto the `ICLModel` interface: time steps become rows, past targets become `y_context`, all covariates and timestamps of the context become `x_context`, and the forecast horizon becomes `x_query` with one row of known covariates and timestamps per future time step. Columns keep their semantic types, so each model decides which ones it consumes. Series in a batch are left-padded with missing values, and multivariate series are flattened for models without multi-target support. @@ -49,3 +49,36 @@ This compares all available SDM results against published GIFT-Eval results on t - Published results batch series of different context lengths together, which shifts their positional encodings on datasets with varying context lengths. - Published results clip forecasts of non-negative series at zero. - Published results forecast at most 32 variates jointly, subsampling covariates of `bizitobs_application` and `bizitobs_service`. + +______________________________________________________________________ + +## fev-bench + +### Setup + +Run the command below from the repository root: + +```bash +pip install . fev==0.10.0 +``` + +Tasks are downloaded from Hugging Face on first use. + +### Run + +```bash +python -m benchmark.forecasting.fev_bench.main --model timesfm3 +``` + +Pass `--task` to run only selected fev-bench tasks, *e.g.*, `--task hospital --task proenfo_gfc12`. +Results are written to `benchmark/forecasting/fev_bench_out//`, and already completed tasks are skipped. + +### Evaluate + +```bash +python -m benchmark.forecasting.fev_bench.evaluate +``` + +This compares all available SDM results against published fev-bench results on the same tasks with `fev.leaderboard`, using a seasonal naive baseline. + +In addition to the differences listed for GIFT-Eval, published `TimesFM-3` results impute missing values with the last observed value, convert non-numerical covariates to numbers, and ignore static covariates. diff --git a/benchmark/forecasting/fev_bench/__init__.py b/benchmark/forecasting/fev_bench/__init__.py new file mode 100644 index 000000000..6fdaa7520 --- /dev/null +++ b/benchmark/forecasting/fev_bench/__init__.py @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Task definitions and reference results of the installed fev release: +FEV_URL = "https://raw.githubusercontent.com/autogluon/fev/v0.10.0" diff --git a/benchmark/forecasting/fev_bench/evaluate.py b/benchmark/forecasting/fev_bench/evaluate.py new file mode 100644 index 000000000..3b9c278f4 --- /dev/null +++ b/benchmark/forecasting/fev_bench/evaluate.py @@ -0,0 +1,60 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +r"""Evaluate SDM forecasting model results on fev-bench.""" + +from pathlib import Path + +import fev +import pandas as pd + +from benchmark.forecasting.fev_bench import FEV_URL +from benchmark.forecasting.model import MODEL_CONFIGS + +BASELINE = "Seasonal Naive" +REFERENCES = ("timesfm-3", "timesfm-2_5", "chronos-2", "seasonal_naive") +METRICS = ("SQL", "MASE") + + +def main() -> None: + benchmark_dir = Path(__file__).parent.parent + result_root = benchmark_dir / "fev_bench_out" + output_root = benchmark_dir / "evals" / "fev_bench" + + summaries = [] + for model_config in MODEL_CONFIGS.values(): + path = result_root / model_config.name / "results.csv" + if path.exists(): + summaries.append(pd.read_csv(path)) + + if not summaries: + raise FileNotFoundError( + f"No fev-bench results found under {result_root}" + ) + + # Compare with published results on tasks run by all models: + tasks = set.intersection(*(set(df["task_name"]) for df in summaries)) + for name in REFERENCES: + path = f"{FEV_URL}/benchmarks/fev_bench/results/{name}.csv" + summaries.append(pd.read_csv(path)) + summaries = pd.concat(summaries) + summaries = summaries[summaries["task_name"].isin(tasks)] + + output_root.mkdir(parents=True, exist_ok=True) + for metric in METRICS: + leaderboard = fev.leaderboard( + summaries, + metric_column=metric, + baseline_model=BASELINE, + ) + leaderboard.to_csv(output_root / f"leaderboard_{metric}.csv") + print(f"\nfev-bench leaderboard ({metric}, {len(tasks)} tasks):") + print( + leaderboard[["skill_score", "win_rate"]].to_string( + float_format="%.4f" + ) + ) + + +if __name__ == "__main__": + main() diff --git a/benchmark/forecasting/fev_bench/main.py b/benchmark/forecasting/fev_bench/main.py new file mode 100644 index 000000000..ba0c44fb4 --- /dev/null +++ b/benchmark/forecasting/fev_bench/main.py @@ -0,0 +1,87 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +r"""Run an SDM forecasting model on fev-bench.""" + +import argparse +import json +from dataclasses import replace +from pathlib import Path + +import fev +import pandas as pd +import torch + +from benchmark.forecasting.fev_bench import FEV_URL +from benchmark.forecasting.fev_bench.models import SDMModel +from benchmark.forecasting.model import MODEL_CONFIGS, add_model_args + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + add_model_args(parser) + parser.add_argument( + "--task", + action="append", + help="Run only the selected fev-bench task; repeat to combine.", + ) + args = parser.parse_args() + + model_config = MODEL_CONFIGS[args.model] + if args.max_context_size is not None: + model_config = replace( + model_config, max_context_size=args.max_context_size + ) + result_path = ( + Path(__file__).parent.parent + / "fev_bench_out" + / model_config.name + / "results.csv" + ) + result_path.parent.mkdir(parents=True, exist_ok=True) + completed = ( + set(pd.read_csv(result_path)["task_name"]) + if result_path.exists() + else set() + ) + + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + model = SDMModel( + model_config.factory(device), + config=model_config, + batch_size=args.batch_size, + ) + + # Mirrors fev's official `models/evaluate.py` runner: + benchmark = fev.Benchmark.from_yaml( + f"{FEV_URL}/benchmarks/fev_bench/tasks.yaml" + ) + for task in benchmark.tasks: + if args.task is not None and task.task_name not in args.task: + continue + if task.task_name in completed: + continue + + predictions = model.fit_predict(task) + summary = task.evaluation_summary( + predictions, + model_name=f"[SDM] {model_config.name}", + training_time_s=model.training_time, + inference_time_s=model.inference_time, + trained_on_this_dataset=( + task.dataset_config in model.trained_on_datasets + ), + extra_info={"model_kwargs": json.dumps(vars(args))}, + ) + row = pd.DataFrame([summary]) + row.to_csv( + result_path, + mode="a", + header=not result_path.exists(), + index=False, + ) + print(row[["task_name", "test_error"]].to_string(index=False)) + + +if __name__ == "__main__": + main() diff --git a/benchmark/forecasting/fev_bench/models.py b/benchmark/forecasting/fev_bench/models.py new file mode 100644 index 000000000..eba1ac6ee --- /dev/null +++ b/benchmark/forecasting/fev_bench/models.py @@ -0,0 +1,87 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""SDM forecasting adapter for fev-bench.""" + +from collections.abc import Iterator + +import datasets +import fev +import numpy as np +import pandas as pd + +import sdm +from benchmark.forecasting.model import ModelConfig, forecast, select_levels + + +class SDMModel(fev.ForecastingModel): + """fev model forecasting batches of series with an SDM model.""" + + def __init__( + self, + model: sdm.models.ICLModel, + config: ModelConfig, + batch_size: int, + ) -> None: + super().__init__() + self.model = model + self.config = config + self.batch_size = batch_size + + def _fit_predict(self, task: fev.Task) -> list[datasets.DatasetDict]: + predictions = [] + for window in task.iter_windows(): + past, future = window.get_input_data() + + values = [] + with self._record_inference_time(): + for past_batch, future_batch in zip( + _to_frames(past, self.batch_size), + _to_frames(future, self.batch_size), + strict=True, + ): + out, levels = forecast( + self.model, + config=self.config, + past=past_batch, + future=future_batch, + targets=task.target_columns, + ) + # Median point forecast followed by the task quantiles: + wanted = [0.5, *task.quantile_levels] + values.append(select_levels(out, levels, wanted)) + values = np.concatenate(values) # [N, H, V, 1 + Q] + + window_predictions = datasets.DatasetDict() + for i, column in enumerate(task.target_columns): + window_predictions[column] = datasets.Dataset.from_dict( + { + "predictions": values[:, :, i, 0], + **{ + str(level): values[:, :, i, j + 1] + for j, level in enumerate(task.quantile_levels) + }, + } + ) + predictions.append(window_predictions) + return predictions + + +def _to_frames( + dataset: datasets.Dataset, + batch_size: int, +) -> Iterator[list[pd.DataFrame]]: + # One frame per series, with static values repeated over time steps and + # the column types declared by the benchmark: + dtypes = { + name: pd.ArrowDtype(getattr(feature, "feature", feature).pa_type) + for name, feature in dataset.features.items() + } + for batch in dataset.with_format("numpy").iter(batch_size): + num_series = len(next(iter(batch.values()))) + yield [ + pd.DataFrame({name: x[i] for name, x in batch.items()}).astype( + dtypes + ) + for i in range(num_series) + ] diff --git a/benchmark/forecasting/model.py b/benchmark/forecasting/model.py index 13304709b..0d2b4b1fb 100644 --- a/benchmark/forecasting/model.py +++ b/benchmark/forecasting/model.py @@ -80,8 +80,10 @@ def forecast( past = [df.iloc[-size:].reset_index(drop=True) for df in past] past = [df.reindex(range(len(df) - size, len(df))) for df in past] - # Infer semantic types jointly so that context and query share them: + # Infer semantic types jointly so that context and query share them, and + # order targets first so that forecasts follow ``targets``: df = pd.concat([*past, *future], ignore_index=True) + df = df[[*targets, *df.columns.drop(targets)]] table = sdm.TableTensor.from_pandas( df=df, stypes=sdm.infer_stypes(df, id="infer"), From 6074233f89ea7f48d3f3aa4f088195bcc758470d Mon Sep 17 00:00:00 2001 From: rusty1s Date: Mon, 26 Oct 2026 09:52:03 +0000 Subject: [PATCH 04/11] update --- benchmark/forecasting/gift_eval/__init__.py | 8 - benchmark/forecasting/gift_eval/evaluate.py | 62 ------- benchmark/forecasting/gift_eval/main.py | 172 ------------------ benchmark/forecasting/gift_eval/models.py | 85 --------- .../fev_bench/__init__.py | 0 .../fev_bench/evaluate.py | 0 .../fev_bench/main.py | 0 .../fev_bench/models.py | 0 8 files changed, 327 deletions(-) delete mode 100644 benchmark/forecasting/gift_eval/__init__.py delete mode 100644 benchmark/forecasting/gift_eval/evaluate.py delete mode 100644 benchmark/forecasting/gift_eval/main.py delete mode 100644 benchmark/forecasting/gift_eval/models.py rename benchmark/{forecasting => timeseries}/fev_bench/__init__.py (100%) rename benchmark/{forecasting => timeseries}/fev_bench/evaluate.py (100%) rename benchmark/{forecasting => timeseries}/fev_bench/main.py (100%) rename benchmark/{forecasting => timeseries}/fev_bench/models.py (100%) diff --git a/benchmark/forecasting/gift_eval/__init__.py b/benchmark/forecasting/gift_eval/__init__.py deleted file mode 100644 index 417150a40..000000000 --- a/benchmark/forecasting/gift_eval/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# Pinned GIFT-Eval revision for dataset properties and reference results: -GIFT_EVAL_URL = ( - "https://raw.githubusercontent.com/SalesforceAIResearch/gift-eval/" - "577a65cfa3acb812405c00d989230bf59f7f8a3d" -) diff --git a/benchmark/forecasting/gift_eval/evaluate.py b/benchmark/forecasting/gift_eval/evaluate.py deleted file mode 100644 index d6176985d..000000000 --- a/benchmark/forecasting/gift_eval/evaluate.py +++ /dev/null @@ -1,62 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -r"""Evaluate SDM forecasting model results on GIFT-Eval.""" - -from pathlib import Path - -import numpy as np -import pandas as pd - -from benchmark.forecasting.gift_eval import GIFT_EVAL_URL -from benchmark.forecasting.model import MODEL_CONFIGS - -BASELINE = "Seasonal_Naive" -REFERENCES = ("TimesFM-3", "TimesFM-2.5", "Chronos-2", BASELINE) -METRICS = { - "eval_metrics/MASE[0.5]": "MASE", - "eval_metrics/mean_weighted_sum_quantile_loss": "CRPS", -} - - -def main() -> None: - benchmark_dir = Path(__file__).parent.parent - result_root = benchmark_dir / "gift_eval_out" - output_root = benchmark_dir / "evals" / "gift_eval" - - results = [] - for model_config in MODEL_CONFIGS.values(): - path = result_root / model_config.name / "all_results.csv" - if path.exists(): - results.append(pd.read_csv(path)) - - if not results: - raise FileNotFoundError( - f"No GIFT-Eval results found under {result_root}" - ) - - # Compare with published results on configurations run by all models: - configs = set.intersection(*(set(result["dataset"]) for result in results)) - for name in REFERENCES: - path = f"{GIFT_EVAL_URL}/results/{name}/all_results.csv" - results.append(pd.read_csv(path)) - - df = pd.concat(results) - df = df[df["dataset"].isin(configs)].set_index(["model", "dataset"]) - df = df[list(METRICS)].rename(columns=METRICS) - - # Normalize by the seasonal naive baseline and aggregate by geometric mean: - log_scores = np.log(df.div(df.loc[BASELINE], level="dataset")) - terms = log_scores.index.get_level_values("dataset").str.split("/").str[-1] - - output_root.mkdir(parents=True, exist_ok=True) - for term, scores in [("overall", log_scores), *log_scores.groupby(terms)]: - leaderboard = np.exp(scores.groupby("model").mean()) - leaderboard = leaderboard.sort_values("CRPS") - leaderboard.to_csv(output_root / f"leaderboard_{term}.csv") - print(f"\nGIFT-Eval leaderboard ({term}):") - print(leaderboard.to_string(float_format="%.4f")) - - -if __name__ == "__main__": - main() diff --git a/benchmark/forecasting/gift_eval/main.py b/benchmark/forecasting/gift_eval/main.py deleted file mode 100644 index 513815328..000000000 --- a/benchmark/forecasting/gift_eval/main.py +++ /dev/null @@ -1,172 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -r"""Run an SDM forecasting model on GIFT-Eval.""" - -import argparse -from dataclasses import replace -from pathlib import Path - -import pandas as pd -import torch -from gift_eval.data import Dataset -from gluonts.ev.metrics import ( - MAE, - MAPE, - MASE, - MSE, - MSIS, - ND, - NRMSE, - RMSE, - SMAPE, - MeanWeightedSumQuantileLoss, -) -from gluonts.model import evaluate_model -from gluonts.time_feature import get_seasonality - -from benchmark.forecasting.gift_eval import GIFT_EVAL_URL -from benchmark.forecasting.gift_eval.models import ( - QUANTILE_LEVELS, - SDMPredictor, -) -from benchmark.forecasting.model import MODEL_CONFIGS, add_model_args - -# Datasets and terms from GIFT-Eval's notebooks: -SHORT_DATASETS = ( # noqa: SIM905 - "m4_yearly m4_quarterly m4_monthly m4_weekly m4_daily m4_hourly " - "electricity/15T electricity/H electricity/D electricity/W " - "solar/10T solar/H solar/D solar/W hospital covid_deaths " - "us_births/D us_births/M us_births/W " - "saugeenday/D saugeenday/M saugeenday/W " - "temperature_rain_with_missing " - "kdd_cup_2018_with_missing/H kdd_cup_2018_with_missing/D " - "car_parts_with_missing restaurant " - "hierarchical_sales/D hierarchical_sales/W " - "LOOP_SEATTLE/5T LOOP_SEATTLE/H LOOP_SEATTLE/D SZ_TAXI/15T SZ_TAXI/H " - "M_DENSE/H M_DENSE/D ett1/15T ett1/H ett1/D ett1/W " - "ett2/15T ett2/H ett2/D ett2/W jena_weather/10T jena_weather/H " - "jena_weather/D bitbrains_fast_storage/5T bitbrains_fast_storage/H " - "bitbrains_rnd/5T bitbrains_rnd/H bizitobs_application bizitobs_service " - "bizitobs_l2c/5T bizitobs_l2c/H" -).split() -MED_LONG_DATASETS = ( # noqa: SIM905 - "electricity/15T electricity/H solar/10T solar/H " - "kdd_cup_2018_with_missing/H LOOP_SEATTLE/5T LOOP_SEATTLE/H SZ_TAXI/15T " - "M_DENSE/H ett1/15T ett1/H ett2/15T ett2/H jena_weather/10T " - "jena_weather/H bitbrains_fast_storage/5T bitbrains_rnd/5T " - "bizitobs_application bizitobs_service bizitobs_l2c/5T bizitobs_l2c/H" -).split() -PRETTY_NAMES = { - "saugeenday": "saugeen", - "temperature_rain_with_missing": "temperature_rain", - "kdd_cup_2018_with_missing": "kdd_cup_2018", - "car_parts_with_missing": "car_parts", -} -METRICS = [ - MSE(forecast_type="mean"), - MSE(forecast_type="0.5"), - MAE(), - MASE(), - MAPE(), - SMAPE(), - MSIS(), - RMSE(), - NRMSE(), - ND(), - MeanWeightedSumQuantileLoss(quantile_levels=QUANTILE_LEVELS), -] - - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - add_model_args(parser) - parser.add_argument( - "--dataset", - action="append", - help="Run only the selected GIFT-Eval dataset; repeat to combine.", - ) - args = parser.parse_args() - - model_config = MODEL_CONFIGS[args.model] - if args.max_context_size is not None: - model_config = replace( - model_config, max_context_size=args.max_context_size - ) - result_path = ( - Path(__file__).parent.parent - / "gift_eval_out" - / model_config.name - / "all_results.csv" - ) - result_path.parent.mkdir(parents=True, exist_ok=True) - completed = ( - set(pd.read_csv(result_path)["dataset"]) - if result_path.exists() - else set() - ) - properties = pd.read_json( - f"{GIFT_EVAL_URL}/notebooks/dataset_properties.json" - ) - - device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - model = model_config.factory(device) - - for name in args.dataset or SHORT_DATASETS: - key, _, freq = name.partition("/") - key = PRETTY_NAMES.get(key.lower(), key.lower()) - freq = freq or properties[key]["frequency"] - for term in ("short", "medium", "long"): - if term != "short" and name not in MED_LONG_DATASETS: - continue - config = f"{key}/{freq}/{term}" - if config in completed: - continue - - # Flatten multivariate series for models without joint multi-target - # forecasting: - dataset = Dataset( - name=name, - term=term, - to_univariate=( - not model.supports_multi_target - and properties[key]["num_variates"] > 1 - ), - ) - predictor = SDMPredictor( - model, - config=model_config, - prediction_length=dataset.prediction_length, - batch_size=args.batch_size, - ) - result = evaluate_model( - predictor, - test_data=dataset.test_data, - metrics=METRICS, - seasonality=get_seasonality(dataset.freq), - ) - row = pd.DataFrame( - [ - { - "dataset": config, - "model": f"[SDM] {model_config.name}", - **{ - f"eval_metrics/{k}": v - for k, v in result.iloc[0].items() - }, - "domain": properties[key]["domain"], - "num_variates": properties[key]["num_variates"], - } - ] - ) - row.to_csv( - result_path, - mode="a", - header=not result_path.exists(), - index=False, - ) - print(row.to_string(index=False)) - - -if __name__ == "__main__": - main() diff --git a/benchmark/forecasting/gift_eval/models.py b/benchmark/forecasting/gift_eval/models.py deleted file mode 100644 index c14b7f254..000000000 --- a/benchmark/forecasting/gift_eval/models.py +++ /dev/null @@ -1,85 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""SDM forecasting adapter for GIFT-Eval.""" - -from collections.abc import Iterator - -import numpy as np -import pandas as pd -from gluonts.dataset import Dataset -from gluonts.itertools import batcher -from gluonts.model.forecast import QuantileForecast -from gluonts.model.predictor import Predictor - -import sdm -from benchmark.forecasting.model import ModelConfig, forecast, select_levels - -QUANTILE_LEVELS = (0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9) - - -class SDMPredictor(Predictor): - """GluonTS predictor forecasting batches of series with an SDM model.""" - - def __init__( - self, - model: sdm.models.ICLModel, - config: ModelConfig, - prediction_length: int, - batch_size: int, - ) -> None: - super().__init__(prediction_length=prediction_length) - self.model = model - self.config = config - self.batch_size = batch_size - - def predict( - self, - dataset: Dataset, - **kwargs: object, - ) -> Iterator[QuantileForecast]: - for batch in batcher(dataset, batch_size=self.batch_size): - # GluonTS stores series as [V, T] (or [T] if univariate): - num_variates = np.atleast_2d(batch[0]["target"]).shape[0] - targets = [f"target_{i}" for i in range(num_variates)] - past, future = [], [] - for entry in batch: - target = np.atleast_2d(entry["target"]) - length = target.shape[-1] - timestamp = pd.period_range( - entry["start"], - periods=length + self.prediction_length, - ).to_timestamp() - columns = { - "item_id": entry["item_id"], - "timestamp": timestamp[:length], - } - columns |= dict(zip(targets, target, strict=True)) - for i, x in enumerate(entry.get("past_feat_dynamic_real", [])): - columns[f"past_feat_dynamic_real_{i}"] = x - past.append(pd.DataFrame(columns)) - future.append( - pd.DataFrame( - { - "item_id": entry["item_id"], - "timestamp": timestamp[length:], - } - ) - ) - values, levels = forecast( - self.model, - config=self.config, - past=past, - future=future, - targets=targets, - ) - values = select_levels(values, levels, QUANTILE_LEVELS) - for entry, value in zip(batch, values, strict=True): - value = value.transpose(2, 0, 1) # [Q, H, V] - if entry["target"].ndim == 1: - value = value[..., 0] - yield QuantileForecast( - forecast_arrays=value, - forecast_keys=[str(q) for q in QUANTILE_LEVELS], - start_date=entry["start"] + entry["target"].shape[-1], - ) diff --git a/benchmark/forecasting/fev_bench/__init__.py b/benchmark/timeseries/fev_bench/__init__.py similarity index 100% rename from benchmark/forecasting/fev_bench/__init__.py rename to benchmark/timeseries/fev_bench/__init__.py diff --git a/benchmark/forecasting/fev_bench/evaluate.py b/benchmark/timeseries/fev_bench/evaluate.py similarity index 100% rename from benchmark/forecasting/fev_bench/evaluate.py rename to benchmark/timeseries/fev_bench/evaluate.py diff --git a/benchmark/forecasting/fev_bench/main.py b/benchmark/timeseries/fev_bench/main.py similarity index 100% rename from benchmark/forecasting/fev_bench/main.py rename to benchmark/timeseries/fev_bench/main.py diff --git a/benchmark/forecasting/fev_bench/models.py b/benchmark/timeseries/fev_bench/models.py similarity index 100% rename from benchmark/forecasting/fev_bench/models.py rename to benchmark/timeseries/fev_bench/models.py From 9570d625c2aca6a5b2e21a0fcb1783dd828eecd6 Mon Sep 17 00:00:00 2001 From: rusty1s Date: Mon, 26 Oct 2026 09:53:32 +0000 Subject: [PATCH 05/11] update --- benchmark/forecasting/README.md | 84 --------------------- benchmark/forecasting/__init__.py | 2 - benchmark/forecasting/model.py | 120 ------------------------------ 3 files changed, 206 deletions(-) delete mode 100644 benchmark/forecasting/README.md delete mode 100644 benchmark/forecasting/__init__.py delete mode 100644 benchmark/forecasting/model.py diff --git a/benchmark/forecasting/README.md b/benchmark/forecasting/README.md deleted file mode 100644 index e15986cd2..000000000 --- a/benchmark/forecasting/README.md +++ /dev/null @@ -1,84 +0,0 @@ -# Time-Series Forecasting Benchmarks - -This directory contains benchmarks of `structured-data-models` on [GIFT-Eval](https://github.com/SalesforceAIResearch/gift-eval) and [fev-bench](https://github.com/autogluon/fev). - -All benchmarks share one model adapter (see `benchmark/forecasting/model.py`) that maps each forecasting task onto the `ICLModel` interface: time steps become rows, past targets become `y_context`, all covariates and timestamps of the context become `x_context`, and the forecast horizon becomes `x_query` with one row of known covariates and timestamps per future time step. Columns keep their semantic types, so each model decides which ones it consumes. Series in a batch are left-padded with missing values, and multivariate series are flattened for models without multi-target support. - -______________________________________________________________________ - -> [!NOTE] -> Weights of `TimesFM3` are distributed under the [TimesFM Non-Commercial License v1.0](https://huggingface.co/google/timesfm-3.0-pytorch/blob/main/LICENSE). -> Review the license before running the `TimesFM3` benchmark, which will download its weights noninteractively. - -## GIFT-Eval - -### Setup - -Run the commands below from the repository root: - -```bash -pip install . gluonts datasets python-dotenv -pip install --no-deps "git+https://github.com/SalesforceAIResearch/gift-eval.git@577a65cfa3acb812405c00d989230bf59f7f8a3d" - -hf download Salesforce/GiftEval --repo-type dataset --local-dir /path/to/gift_eval -export GIFT_EVAL=/path/to/gift_eval -``` - -GIFT-Eval is not released on PyPI and is therefore pinned to a commit. It is installed without dependencies since it pins outdated `numpy` and `datasets` versions. - -### Run - -```bash -python -m benchmark.forecasting.gift_eval.main --model timesfm3 -``` - -Pass `--dataset` to run only selected GIFT-Eval datasets, *e.g.*, `--dataset hospital --dataset ett1/H`. -Use `--batch_size` to set the number of series per model call and `--max_context_size` to truncate long contexts. -Results are written to `benchmark/forecasting/gift_eval_out//`, and already completed configurations are skipped. - -### Evaluate - -```bash -python -m benchmark.forecasting.gift_eval.evaluate -``` - -This compares all available SDM results against published GIFT-Eval results on the same configurations, normalized by a seasonal naive baseline and aggregated by geometric mean. - -`TimesFM3` follows the published `TimesFM-3` evaluation with symmetric averaging over two estimators and contexts truncated to 15,360 time steps. Its results can still differ from the published ones: - -- Published results batch series of different context lengths together, which shifts their positional encodings on datasets with varying context lengths. -- Published results clip forecasts of non-negative series at zero. -- Published results forecast at most 32 variates jointly, subsampling covariates of `bizitobs_application` and `bizitobs_service`. - -______________________________________________________________________ - -## fev-bench - -### Setup - -Run the command below from the repository root: - -```bash -pip install . fev==0.10.0 -``` - -Tasks are downloaded from Hugging Face on first use. - -### Run - -```bash -python -m benchmark.forecasting.fev_bench.main --model timesfm3 -``` - -Pass `--task` to run only selected fev-bench tasks, *e.g.*, `--task hospital --task proenfo_gfc12`. -Results are written to `benchmark/forecasting/fev_bench_out//`, and already completed tasks are skipped. - -### Evaluate - -```bash -python -m benchmark.forecasting.fev_bench.evaluate -``` - -This compares all available SDM results against published fev-bench results on the same tasks with `fev.leaderboard`, using a seasonal naive baseline. - -In addition to the differences listed for GIFT-Eval, published `TimesFM-3` results impute missing values with the last observed value, convert non-numerical covariates to numbers, and ignore static covariates. diff --git a/benchmark/forecasting/__init__.py b/benchmark/forecasting/__init__.py deleted file mode 100644 index d51c4fe1e..000000000 --- a/benchmark/forecasting/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 diff --git a/benchmark/forecasting/model.py b/benchmark/forecasting/model.py deleted file mode 100644 index 0d2b4b1fb..000000000 --- a/benchmark/forecasting/model.py +++ /dev/null @@ -1,120 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""SDM model adapter shared by the time-series forecasting benchmarks.""" - -import argparse -from collections.abc import Callable, Sequence -from dataclasses import dataclass - -import numpy as np -import pandas as pd -import torch - -import sdm -from sdm.models.timesfm3 import TimesFM3 - - -@dataclass(frozen=True) -class ModelConfig: - name: str - factory: Callable[[torch.device], sdm.models.ICLModel] - num_estimators: int = 1 - max_context_size: int | None = None - - -MODEL_CONFIGS = { - "timesfm3": ModelConfig( - name="TimesFM3", - factory=lambda device: TimesFM3(accept_license=True, device=device), - num_estimators=2, - max_context_size=15360, - ), -} - - -def add_model_args(parser: argparse.ArgumentParser) -> None: - """Add the model flags shared by all forecasting benchmarks.""" - parser.add_argument( - "--model", - choices=tuple(MODEL_CONFIGS), - default="timesfm3", - help="SDM model to benchmark.", - ) - parser.add_argument( - "--batch_size", - type=int, - default=64, - help="Number of series forecast per model call.", - ) - parser.add_argument( - "--max_context_size", - type=int, - help="Truncate each context to its most recent time steps " - "(defaults to the model's maximum context size).", - ) - - -def forecast( - model: sdm.models.ICLModel, - config: ModelConfig, - past: Sequence[pd.DataFrame], # B x [T_i, V + C] - future: Sequence[pd.DataFrame], # B x [H, K] - targets: Sequence[str], -) -> tuple[np.ndarray, np.ndarray]: # [B, H, V, Q], [Q] - """Forecast the quantiles of a batch of (multivariate) time series. - - Time steps map to rows: ``past`` holds the targets and all covariates of - the context, while ``future`` holds the covariates known over the - forecast horizon. Columns keep their semantic types, so that the model - decides which ones to consume. Missing values are passed through. - """ - device = next(model.parameters()).device - num_series, horizon = len(past), len(future[0]) - size = max(len(df) for df in past) - if config.max_context_size is not None: - size = min(size, config.max_context_size) - - # Left-pad shorter series with missing values so that all series end at - # the forecast start: - past = [df.iloc[-size:].reset_index(drop=True) for df in past] - past = [df.reindex(range(len(df) - size, len(df))) for df in past] - - # Infer semantic types jointly so that context and query share them, and - # order targets first so that forecasts follow ``targets``: - df = pd.concat([*past, *future], ignore_index=True) - df = df[[*targets, *df.columns.drop(targets)]] - table = sdm.TableTensor.from_pandas( - df=df, - stypes=sdm.infer_stypes(df, id="infer"), - device=device, - ) - # Split rows into the context [B, T, V + C] and the query [B, H, V + C]: - context = table[: num_series * size].view(num_series, size, -1) - query = table[num_series * size :].view(num_series, horizon, -1) - assert isinstance(context, sdm.TableTensor) - assert isinstance(query, sdm.TableTensor) - - out = model( - x_context=context.drop_columns(targets), - y_context=context.select_columns(targets), - x_query=query.select_columns(future[0].columns), - num_estimators=config.num_estimators, - ) - values, levels = sdm.evaluation.to_quantiles(out) - return values.cpu().numpy(), levels.cpu().numpy() - - -def select_levels( - values: np.ndarray, # [..., Q] - levels: np.ndarray, # [Q] - wanted: Sequence[float], -) -> np.ndarray: # [..., len(wanted)] - """Select the quantile levels requested by a benchmark.""" - index = [int(np.abs(levels - level).argmin()) for level in wanted] - if not np.allclose(levels[index], wanted): - raise ValueError( - f"Model does not predict quantile levels {list(wanted)} " - f"(got {levels.tolist()})" - ) - return values[..., index] From a2439ba2e45c9bd99c7d6f1bc5c9b5aa276b36e3 Mon Sep 17 00:00:00 2001 From: rusty1s Date: Mon, 26 Oct 2026 09:55:59 +0000 Subject: [PATCH 06/11] update --- benchmark/timeseries/README.md | 2 +- benchmark/timeseries/fev_bench/__init__.py | 3 +- docs/source/icl.md | 1 - sdm/evaluation/__init__.py | 2 - sdm/evaluation/regression.py | 61 ---------------------- test/evaluation/test_regression.py | 26 --------- 6 files changed, 2 insertions(+), 93 deletions(-) delete mode 100644 sdm/evaluation/regression.py delete mode 100644 test/evaluation/test_regression.py diff --git a/benchmark/timeseries/README.md b/benchmark/timeseries/README.md index abd9b02a7..1f2955fc8 100644 --- a/benchmark/timeseries/README.md +++ b/benchmark/timeseries/README.md @@ -1,3 +1,3 @@ # Time Series Benchmarks -This directory contains time series benchmarks of `structured-data-models` on [GIFT-Eval](gift_eval/). +This directory contains time series benchmarks of `structured-data-models` on [GIFT-Eval](gift_eval/), and [fev-bench](fev_bench/). diff --git a/benchmark/timeseries/fev_bench/__init__.py b/benchmark/timeseries/fev_bench/__init__.py index 6fdaa7520..4a93993ce 100644 --- a/benchmark/timeseries/fev_bench/__init__.py +++ b/benchmark/timeseries/fev_bench/__init__.py @@ -1,5 +1,4 @@ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# Task definitions and reference results of the installed fev release: -FEV_URL = "https://raw.githubusercontent.com/autogluon/fev/v0.10.0" +FEV_BENCH_URL = "https://raw.githubusercontent.com/autogluon/fev/v0.10.0" diff --git a/docs/source/icl.md b/docs/source/icl.md index 8a34ba05d..b4a1e1e58 100644 --- a/docs/source/icl.md +++ b/docs/source/icl.md @@ -98,7 +98,6 @@ Predictions are returned as a general {py:class}`~sdm.tensor.TableTensor`, where Since predictions are returned as a {py:class}`~sdm.tensor.TableTensor`, you can zero-copy them to [`pandas`](https://pandas.pydata.org/docs), [`arrow`](https://arrow.apache.org/docs), or [`cudf`](https://docs.rapids.ai/api/cudf) via {py:meth}`~sdm.tensor.TableTensor.to_pandas`, {py:meth}`~sdm.tensor.TableTensor.to_arrow`, and {py:meth}`~sdm.tensor.TableTensor.to_cudf` for further downstream processing. In order to simplify metric calculation (*e.g.*, via [`torchmetrics`](https://lightning.ai/docs/torchmetrics)), we provide helper functions in the [`sdm.evaluation`](api/evaluation) package to convert target columns to class indices and align prediction columns to them (see {py:func}`~sdm.evaluation.to_class_indices` and {py:func}`~sdm.evaluation.to_binary_class`). -Similarly, {py:func}`~sdm.evaluation.to_quantiles` converts quantile predictions of one or more targets into a single tensor together with their quantile levels. ## Ensembling diff --git a/sdm/evaluation/__init__.py b/sdm/evaluation/__init__.py index 08304db08..7d3b78251 100644 --- a/sdm/evaluation/__init__.py +++ b/sdm/evaluation/__init__.py @@ -4,10 +4,8 @@ """Evaluation utilities.""" from sdm.evaluation.classification import to_class_indices, to_binary_class -from sdm.evaluation.regression import to_quantiles __all__ = [ "to_class_indices", "to_binary_class", - "to_quantiles", ] diff --git a/sdm/evaluation/regression.py b/sdm/evaluation/regression.py deleted file mode 100644 index 8ff24bc10..000000000 --- a/sdm/evaluation/regression.py +++ /dev/null @@ -1,61 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -from torch import Tensor - -from sdm import Stype, TableTensor - - -def to_quantiles(pred: TableTensor) -> tuple[Tensor, Tensor]: - r"""Convert quantile predictions to a per-target quantile tensor. - - Numerical columns must be named ``"q"`` for a single target or - ``"__q"`` for multiple targets, where the digits encode the - quantile level as a decimal fraction, *e.g.*, ``"q500"`` and ``"q50"`` both - denote the median. - Columns are expected to be grouped by target, with the same quantile levels - in the same order for every target. - - Args: - pred: Prediction table whose numerical columns contain quantiles. - - Returns: - The quantile predictions with shape ``[..., V, Q]`` for ``V`` targets - and ``Q`` quantile levels, and the quantile levels with shape ``[Q]``. - - >>> import torch - >>> import sdm - >>> from sdm.evaluation import to_quantiles - >>> pred = sdm.TableTensor.from_tensor( - ... torch.tensor([[1.0, 2.0, 3.0, 4.0]]), - ... columns=["q100", "q400", "q700", "q900"], - ... ) - >>> values, levels = to_quantiles(pred) - >>> values - tensor([[[1., 2., 3., 4.]]]) - >>> levels - tensor([0.1000, 0.4000, 0.7000, 0.9000]) - >>> pred = sdm.TableTensor.from_tensor( - ... torch.tensor([[1.0, 2.0, 3.0, 4.0]]), - ... columns=["a__q10", "a__q90", "b__q10", "b__q90"], - ... ) - >>> values, levels = to_quantiles(pred) - >>> values - tensor([[[1., 2.], - [3., 4.]]]) - >>> levels - tensor([0.1000, 0.9000]) - """ - columns = pred.columns[Stype.numerical] - prefixes, _, digits = zip(*(column.rpartition("q") for column in columns)) - targets = dict.fromkeys(prefixes) - digits = digits[: len(columns) // len(targets)] - if columns != tuple(f"{t}q{d}" for t in targets for d in digits): - raise ValueError( - "Expected quantile columns grouped by target with the same " - f"quantile levels (got {list(columns)})" - ) - - levels = [int(d) / 10 ** len(d) for d in digits] - values = pred.numerical.unflatten(-1, (len(targets), -1)) # [..., V, Q] - return values, values.new_tensor(levels) diff --git a/test/evaluation/test_regression.py b/test/evaluation/test_regression.py deleted file mode 100644 index 9ae0941a8..000000000 --- a/test/evaluation/test_regression.py +++ /dev/null @@ -1,26 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -import pytest -import torch - -import sdm - - -def test_to_quantiles() -> None: - pred = sdm.TableTensor.from_tensor( - torch.randn(2, 5, 4), - columns=["b__q10", "b__q90", "a__q10", "a__q90"], - ) - values, levels = sdm.evaluation.to_quantiles(pred) - torch.testing.assert_close(values, pred.numerical.view(2, 5, 2, 2)) - torch.testing.assert_close(levels, torch.tensor([0.1, 0.9])) - - -def test_to_quantiles_invalid() -> None: - pred = sdm.TableTensor.from_tensor( - torch.randn(5, 4), - columns=["a__q10", "b__q10", "a__q90", "b__q90"], - ) - with pytest.raises(ValueError, match="grouped by target"): - sdm.evaluation.to_quantiles(pred) From 7038f8e1fc5caa65e4c49e8b7cab84e9b3dab9c4 Mon Sep 17 00:00:00 2001 From: rusty1s Date: Mon, 26 Oct 2026 10:00:26 +0000 Subject: [PATCH 07/11] update --- benchmark/timeseries/fev_bench/README.md | 35 ++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 benchmark/timeseries/fev_bench/README.md diff --git a/benchmark/timeseries/fev_bench/README.md b/benchmark/timeseries/fev_bench/README.md new file mode 100644 index 000000000..407c939b1 --- /dev/null +++ b/benchmark/timeseries/fev_bench/README.md @@ -0,0 +1,35 @@ +# Time Series Forecasting Foundation Models on fev-bench + +This benchmark evaluates `structured-data-models` on the [fev-bench](https://github.com/autogluon/fev) benchmark. + +## Setup + +Run the command below from the repository root: + +```bash +pip install . fev==0.10.0 +``` + +## Run + +- **`TimesFM3`**: + + ```bash + python -m benchmark.timeseries.fev_bench.main --model timesfm3 + ``` + + `TimesFM3` follows official evaluation with symmetric averaging over two estimators, contexts truncated to 15,360 time steps and up to 32 variates per forward pass. + +> [!NOTE] +> Weights of `TimesFM3` are distributed under the [TimesFM Non-Commercial License v1.0](https://huggingface.co/google/timesfm-3.0-pytorch/blob/main/LICENSE). +> Review the license before running the `TimesFM3` benchmark, which will download its weights noninteractively. + +Results will be written to `benchmark/timeseries/fev_bench_out//`, and already completed configurations are skipped. + +## Evaluate + +Evaluate all available model results with: + +```bash +python -m benchmark.timeseries.fev_bench.evaluate +``` From 74b2d7ef766a4d7a93f43c4c62e223239151415e Mon Sep 17 00:00:00 2001 From: rusty1s Date: Mon, 26 Oct 2026 10:09:06 +0000 Subject: [PATCH 08/11] update --- benchmark/timeseries/fev_bench/README.md | 3 +++ benchmark/timeseries/fev_bench/evaluate.py | 2 +- benchmark/timeseries/fev_bench/main.py | 12 +++++++++--- benchmark/timeseries/gift_eval/README.md | 2 ++ 4 files changed, 15 insertions(+), 4 deletions(-) diff --git a/benchmark/timeseries/fev_bench/README.md b/benchmark/timeseries/fev_bench/README.md index 407c939b1..30701cdf4 100644 --- a/benchmark/timeseries/fev_bench/README.md +++ b/benchmark/timeseries/fev_bench/README.md @@ -26,6 +26,9 @@ pip install . fev==0.10.0 Results will be written to `benchmark/timeseries/fev_bench_out//`, and already completed configurations are skipped. +Pass `--task` to run only selected fev-bench tasks, *e.g.*, `--task hospital --task proenfo_gfc12`. +Pass `--mini` to run the 20-task fev-bench-mini subset: + ## Evaluate Evaluate all available model results with: diff --git a/benchmark/timeseries/fev_bench/evaluate.py b/benchmark/timeseries/fev_bench/evaluate.py index 3b9c278f4..c508e5fc1 100644 --- a/benchmark/timeseries/fev_bench/evaluate.py +++ b/benchmark/timeseries/fev_bench/evaluate.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -r"""Evaluate SDM forecasting model results on fev-bench.""" +r"""Evaluate SDM time series forecasting results on fev-bench.""" from pathlib import Path diff --git a/benchmark/timeseries/fev_bench/main.py b/benchmark/timeseries/fev_bench/main.py index ba0c44fb4..a90d8447c 100644 --- a/benchmark/timeseries/fev_bench/main.py +++ b/benchmark/timeseries/fev_bench/main.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -r"""Run an SDM forecasting model on fev-bench.""" +r"""Run an SDM time series forecasting model on fev-bench.""" import argparse import json @@ -12,14 +12,19 @@ import pandas as pd import torch -from benchmark.forecasting.fev_bench import FEV_URL from benchmark.forecasting.fev_bench.models import SDMModel from benchmark.forecasting.model import MODEL_CONFIGS, add_model_args +from benchmark.timeseries.fev_bench import FEV_BENCH_URL def main() -> None: parser = argparse.ArgumentParser(description=__doc__) add_model_args(parser) + parser.add_argument( + "--mini", + action="store_true", + help="Run the 20-task fev-bench-mini subset.", + ) parser.add_argument( "--task", action="append", @@ -53,8 +58,9 @@ def main() -> None: ) # Mirrors fev's official `models/evaluate.py` runner: + filename = "tasks_mini.yaml" if args.mini else "tasks.yaml" benchmark = fev.Benchmark.from_yaml( - f"{FEV_URL}/benchmarks/fev_bench/tasks.yaml" + f"{FEV_BENCH_URL}/benchmarks/fev_bench/{filename}" ) for task in benchmark.tasks: if args.task is not None and task.task_name not in args.task: diff --git a/benchmark/timeseries/gift_eval/README.md b/benchmark/timeseries/gift_eval/README.md index b0117fad6..ef0711fd9 100644 --- a/benchmark/timeseries/gift_eval/README.md +++ b/benchmark/timeseries/gift_eval/README.md @@ -27,6 +27,8 @@ pip install --no-deps "git+https://github.com/SalesforceAIResearch/gift-eval.git Results will be written to `benchmark/timeseries/gift_eval_out//`, and already completed configurations are skipped. +Pass `--dataset` to run only selected GIFT-Eval datasets, *e.g.*, `--dataset hospital --dataset ett1/H`. + ## Evaluate Evaluate all available model results with: From 26a5b532ddcb9f3a18070558ed40e1ff109fc850 Mon Sep 17 00:00:00 2001 From: rusty1s Date: Mon, 26 Oct 2026 12:02:16 +0000 Subject: [PATCH 09/11] update --- benchmark/timeseries/fev_bench/main.py | 126 ++++++++---------- .../fev_bench/{models.py => predictor.py} | 8 +- benchmark/timeseries/gift_eval/main.py | 5 +- benchmark/timeseries/gift_eval/predictor.py | 8 +- 4 files changed, 61 insertions(+), 86 deletions(-) rename benchmark/timeseries/fev_bench/{models.py => predictor.py} (92%) diff --git a/benchmark/timeseries/fev_bench/main.py b/benchmark/timeseries/fev_bench/main.py index a90d8447c..db945fefa 100644 --- a/benchmark/timeseries/fev_bench/main.py +++ b/benchmark/timeseries/fev_bench/main.py @@ -5,89 +5,69 @@ import argparse import json -from dataclasses import replace from pathlib import Path import fev import pandas as pd import torch -from benchmark.forecasting.fev_bench.models import SDMModel -from benchmark.forecasting.model import MODEL_CONFIGS, add_model_args from benchmark.timeseries.fev_bench import FEV_BENCH_URL +from benchmark.timeseries.model import MODEL_CONFIGS +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument( + "--model", + choices=tuple(MODEL_CONFIGS), + default="timesfm3", + help="SDM model to benchmark.", +) +parser.add_argument( + "--mini", + action="store_true", + help="Run the 20-task fev-bench-mini subset.", +) +parser.add_argument( + "--task", + action="append", + help="Run only the selected fev-bench task; repeat to combine.", +) +args = parser.parse_args() -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - add_model_args(parser) - parser.add_argument( - "--mini", - action="store_true", - help="Run the 20-task fev-bench-mini subset.", - ) - parser.add_argument( - "--task", - action="append", - help="Run only the selected fev-bench task; repeat to combine.", - ) - args = parser.parse_args() +config = MODEL_CONFIGS[args.model] +path = Path(__file__).parent.parent +path = path / "fev_bench_out" / config.name / "results.csv" +path.parent.mkdir(parents=True, exist_ok=True) - model_config = MODEL_CONFIGS[args.model] - if args.max_context_size is not None: - model_config = replace( - model_config, max_context_size=args.max_context_size - ) - result_path = ( - Path(__file__).parent.parent - / "fev_bench_out" - / model_config.name - / "results.csv" - ) - result_path.parent.mkdir(parents=True, exist_ok=True) - completed = ( - set(pd.read_csv(result_path)["task_name"]) - if result_path.exists() - else set() - ) +device = torch.device("cuda" if torch.cuda.is_available() else "cpu") +model = config.factory(device) - device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - model = SDMModel( - model_config.factory(device), - config=model_config, - batch_size=args.batch_size, - ) +completed = set(pd.read_csv(path)["task_name"]) if path.exists() else set() +filename = "tasks_mini.yaml" if args.mini else "tasks.yaml" +benchmark = fev.Benchmark.from_yaml( + f"{FEV_BENCH_URL}/benchmarks/fev_bench/{filename}" +) +for task in benchmark.tasks: + if args.task is not None and task.task_name not in args.task: + continue + if task.task_name in completed: + continue - # Mirrors fev's official `models/evaluate.py` runner: - filename = "tasks_mini.yaml" if args.mini else "tasks.yaml" - benchmark = fev.Benchmark.from_yaml( - f"{FEV_BENCH_URL}/benchmarks/fev_bench/{filename}" + predictions = model.fit_predict(task) + summary = task.evaluation_summary( + predictions, + model_name=f"[SDM] {config.name}", + training_time_s=model.training_time, + inference_time_s=model.inference_time, + trained_on_this_dataset=( + task.dataset_config in model.trained_on_datasets + ), + extra_info={"model_kwargs": json.dumps(vars(args))}, ) - for task in benchmark.tasks: - if args.task is not None and task.task_name not in args.task: - continue - if task.task_name in completed: - continue - - predictions = model.fit_predict(task) - summary = task.evaluation_summary( - predictions, - model_name=f"[SDM] {model_config.name}", - training_time_s=model.training_time, - inference_time_s=model.inference_time, - trained_on_this_dataset=( - task.dataset_config in model.trained_on_datasets - ), - extra_info={"model_kwargs": json.dumps(vars(args))}, - ) - row = pd.DataFrame([summary]) - row.to_csv( - result_path, - mode="a", - header=not result_path.exists(), - index=False, - ) - print(row[["task_name", "test_error"]].to_string(index=False)) - - -if __name__ == "__main__": - main() + row = pd.DataFrame([summary]) + row.to_csv( + result_path, + mode="a", + header=not result_path.exists(), + index=False, + ) + print(row[["task_name", "test_error"]].to_string(index=False)) diff --git a/benchmark/timeseries/fev_bench/models.py b/benchmark/timeseries/fev_bench/predictor.py similarity index 92% rename from benchmark/timeseries/fev_bench/models.py rename to benchmark/timeseries/fev_bench/predictor.py index eba1ac6ee..ce2dd6856 100644 --- a/benchmark/timeseries/fev_bench/models.py +++ b/benchmark/timeseries/fev_bench/predictor.py @@ -1,8 +1,6 @@ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""SDM forecasting adapter for fev-bench.""" - from collections.abc import Iterator import datasets @@ -11,17 +9,15 @@ import pandas as pd import sdm -from benchmark.forecasting.model import ModelConfig, forecast, select_levels +from benchmark.timeseries.model import ModelConfig class SDMModel(fev.ForecastingModel): - """fev model forecasting batches of series with an SDM model.""" - def __init__( self, model: sdm.models.ICLModel, config: ModelConfig, - batch_size: int, + batch_size: int = 64, ) -> None: super().__init__() self.model = model diff --git a/benchmark/timeseries/gift_eval/main.py b/benchmark/timeseries/gift_eval/main.py index 23a14b832..cc028a591 100644 --- a/benchmark/timeseries/gift_eval/main.py +++ b/benchmark/timeseries/gift_eval/main.py @@ -69,9 +69,6 @@ path = path / "gift_eval_out" / config.name / "all_results.csv" path.parent.mkdir(parents=True, exist_ok=True) -completed = set(pd.read_csv(path)["dataset"]) if path.exists() else set() -properties = pd.read_json(f"{GIFT_EVAL_URL}/notebooks/dataset_properties.json") - names = args.dataset or DATASETS if not os.getenv("GIFT_EVAL"): os.environ["GIFT_EVAL"] = snapshot_download( @@ -83,6 +80,8 @@ device = torch.device("cuda" if torch.cuda.is_available() else "cpu") model = config.factory(device) +completed = set(pd.read_csv(path)["dataset"]) if path.exists() else set() +properties = pd.read_json(f"{GIFT_EVAL_URL}/notebooks/dataset_properties.json") for name in names: key, _, freq = name.partition("/") key = PRETTY_NAMES.get(key.lower(), key.lower()) diff --git a/benchmark/timeseries/gift_eval/predictor.py b/benchmark/timeseries/gift_eval/predictor.py index 52d801bc9..67e3ad6fe 100644 --- a/benchmark/timeseries/gift_eval/predictor.py +++ b/benchmark/timeseries/gift_eval/predictor.py @@ -96,10 +96,10 @@ def predict( y_i = torch.cat([y_i, y[..., : size - y_i.size(-1)]], -1) out = self.model( - x_context=x, - y_context=y_i, - x_query=torch.empty( - (x.size(0), self.prediction_length, 0), + x_context=sdm.TableTensor.from_tensor(x), + y_context=sdm.TableTensor.from_tensor(y_i), + x_query=sdm.TableTensor( + size=(x.size(0), self.prediction_length), device=self.device, ), num_estimators=self.config.num_estimators, From 53c028b270bec00a5f9022e945d42b6d6ef9ab01 Mon Sep 17 00:00:00 2001 From: rusty1s Date: Mon, 26 Oct 2026 14:18:46 +0000 Subject: [PATCH 10/11] update --- benchmark/timeseries/fev_bench/main.py | 27 ++-- benchmark/timeseries/fev_bench/predictor.py | 161 ++++++++++++++++---- benchmark/timeseries/gift_eval/predictor.py | 10 +- 3 files changed, 151 insertions(+), 47 deletions(-) diff --git a/benchmark/timeseries/fev_bench/main.py b/benchmark/timeseries/fev_bench/main.py index db945fefa..828c016da 100644 --- a/benchmark/timeseries/fev_bench/main.py +++ b/benchmark/timeseries/fev_bench/main.py @@ -4,7 +4,6 @@ r"""Run an SDM time series forecasting model on fev-bench.""" import argparse -import json from pathlib import Path import fev @@ -12,6 +11,7 @@ import torch from benchmark.timeseries.fev_bench import FEV_BENCH_URL +from benchmark.timeseries.fev_bench.predictor import SDMPredictor from benchmark.timeseries.model import MODEL_CONFIGS parser = argparse.ArgumentParser(description=__doc__) @@ -39,7 +39,10 @@ path.parent.mkdir(parents=True, exist_ok=True) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") -model = config.factory(device) +model = SDMPredictor( + model=config.factory(device), + config=config, +) completed = set(pd.read_csv(path)["task_name"]) if path.exists() else set() filename = "tasks_mini.yaml" if args.mini else "tasks.yaml" @@ -52,22 +55,16 @@ if task.task_name in completed: continue - predictions = model.fit_predict(task) summary = task.evaluation_summary( - predictions, + model.fit_predict(task), model_name=f"[SDM] {config.name}", training_time_s=model.training_time, inference_time_s=model.inference_time, - trained_on_this_dataset=( - task.dataset_config in model.trained_on_datasets - ), - extra_info={"model_kwargs": json.dumps(vars(args))}, ) - row = pd.DataFrame([summary]) - row.to_csv( - result_path, - mode="a", - header=not result_path.exists(), - index=False, + print( + f"{task.task_name}: Error={summary['test_error']:6f}, " + f"Time={model.inference_time:.2f}s" ) - print(row[["task_name", "test_error"]].to_string(index=False)) + row = pd.DataFrame([summary]) + row.to_csv(path, mode="a", header=not path.exists(), index=False) + completed.add(task.task_name) diff --git a/benchmark/timeseries/fev_bench/predictor.py b/benchmark/timeseries/fev_bench/predictor.py index ce2dd6856..bd5a48983 100644 --- a/benchmark/timeseries/fev_bench/predictor.py +++ b/benchmark/timeseries/fev_bench/predictor.py @@ -6,13 +6,14 @@ import datasets import fev import numpy as np -import pandas as pd +import torch +import tqdm import sdm from benchmark.timeseries.model import ModelConfig -class SDMModel(fev.ForecastingModel): +class SDMPredictor(fev.ForecastingModel): def __init__( self, model: sdm.models.ICLModel, @@ -23,31 +24,135 @@ def __init__( self.model = model self.config = config self.batch_size = batch_size + self.device = next(model.parameters()).device def _fit_predict(self, task: fev.Task) -> list[datasets.DatasetDict]: predictions = [] - for window in task.iter_windows(): + for window in tqdm.tqdm( + task.iter_windows(), + total=task.num_windows, + desc=task.task_name, + ): past, future = window.get_input_data() + all_columns = task.known_dynamic_columns.copy() + if self.model.supports_context_only_columns: + all_columns += task.past_dynamic_columns + values = [] with self._record_inference_time(): - for past_batch, future_batch in zip( - _to_frames(past, self.batch_size), - _to_frames(future, self.batch_size), + for ys, xs_context, xs_query in zip( + _from(past, task.target_columns, self.batch_size), + _from(past, all_columns, self.batch_size), + _from(future, task.known_dynamic_columns, self.batch_size), strict=True, ): - out, levels = forecast( - self.model, - config=self.config, - past=past_batch, - future=future_batch, - targets=task.target_columns, + generator = torch.Generator().manual_seed(42) + + if self.config.max_context_size is not None: + length = self.config.max_context_size + ys = [y[..., -length:] for y in ys] + xs_context = [x[..., -length:] for x in xs_context] + + length = max(y.size(-1) for y in ys) + y = torch.full( + (len(ys), ys[0].size(0), length), + fill_value=torch.nan, + ) + for i, y_i in enumerate(ys): + y[i, :, -y_i.size(-1) :] = y_i + x_context = torch.full( + ( + len(ys), + xs_context[0].size(0) + if len(xs_context) > 0 + else 0, + length, + ), + fill_value=torch.nan, + ) + for i, x_context_i in enumerate(xs_context): + x_context[i, :, -x_context_i.size(-1) :] = x_context_i + x_query = torch.empty( + len(ys), + xs_query[0].size(0) if len(xs_query) > 0 else 0, + task.horizon, + ) + for i, x_query_i in enumerate(xs_query): + x_query[i, :, :] = x_query_i + + if ( # Keep room for at least one target: + self.config.max_variates is not None + and x_query.size(1) >= self.config.max_variates + ): + perm = torch.randperm( + x_query.size(1), generator=generator + )[: self.config.max_variates - 1] + x_context = x_context[:, perm] + x_query = x_query[:, perm] + elif ( + self.config.max_variates is not None + and x_context.size(1) >= self.config.max_variates + ): + perm = torch.randperm( + x_context.size(1) - x_query.size(1), + generator=generator, + )[: self.config.max_variates - 1 - x_query.size(1)] + x_context = torch.cat( + [ + x_context[:, : x_query.size(1)], + x_context[:, perm + x_query.size(1)], + ], + dim=1, + ) + + y = y.transpose(-1, -2).to( + self.device, + memory_format=torch.contiguous_format, + ) + x_context = x_context.transpose(-1, -2).to( + self.device, + memory_format=torch.contiguous_format, ) - # Median point forecast followed by the task quantiles: + x_query = x_query.transpose(-1, -2).to( + self.device, + memory_format=torch.contiguous_format, + ) + + size = y.size(-1) + if self.config.max_variates is not None: + size = min( + size, + self.config.max_variates - x_context.size(-1), + ) + + preds = [] + for start in range(0, y.size(-1), size): + y_i = y[..., start : start + size] + if y_i.size(-1) < size: + y_i = torch.cat( + [y_i, y[..., : size - y_i.size(-1)]], dim=-1 + ) + + out = self.model( + x_context=sdm.TableTensor.from_tensor(x_context), + y_context=sdm.TableTensor.from_tensor(y_i), + x_query=sdm.TableTensor.from_tensor(x_query), + num_estimators=self.config.num_estimators, + ) + pred = out.numerical + pred = pred.view(y.size(0), task.horizon, size, -1) + preds.append(pred) + + pred = torch.cat(preds, dim=-2)[..., : y.size(-1), :] + pred = pred.cpu().numpy() # [B, R, C, Q] + levels = out.numerical_columns[: pred.shape[1]] + levels = [float(f"0.{q.split('q')[-1]}") for q in levels] wanted = [0.5, *task.quantile_levels] - values.append(select_levels(out, levels, wanted)) - values = np.concatenate(values) # [N, H, V, 1 + Q] + index = [levels.index(q) for q in wanted] + values.append(pred[..., index]) + values = np.concatenate(values) # [B, R, C, 1 + Q] window_predictions = datasets.DatasetDict() for i, column in enumerate(task.target_columns): window_predictions[column] = datasets.Dataset.from_dict( @@ -60,24 +165,26 @@ def _fit_predict(self, task: fev.Task) -> list[datasets.DatasetDict]: } ) predictions.append(window_predictions) + return predictions -def _to_frames( +def _from( dataset: datasets.Dataset, + columns: list[str], batch_size: int, -) -> Iterator[list[pd.DataFrame]]: - # One frame per series, with static values repeated over time steps and - # the column types declared by the benchmark: - dtypes = { - name: pd.ArrowDtype(getattr(feature, "feature", feature).pa_type) - for name, feature in dataset.features.items() - } +) -> Iterator[list[torch.Tensor]]: for batch in dataset.with_format("numpy").iter(batch_size): - num_series = len(next(iter(batch.values()))) + if not columns: + yield [] + continue + yield [ - pd.DataFrame({name: x[i] for name, x in batch.items()}).astype( - dtypes + torch.from_numpy( + np.asarray( + [batch[column][i] for column in columns], + dtype=np.float32, + ) ) - for i in range(num_series) + for i in range(len(batch[columns[0]])) ] diff --git a/benchmark/timeseries/gift_eval/predictor.py b/benchmark/timeseries/gift_eval/predictor.py index 67e3ad6fe..3174b5e95 100644 --- a/benchmark/timeseries/gift_eval/predictor.py +++ b/benchmark/timeseries/gift_eval/predictor.py @@ -36,7 +36,7 @@ def predict( ) -> Iterator[QuantileForecast]: for batch in batcher(dataset, batch_size=self.batch_size): - cpu_generator = torch.Generator().manual_seed(42) + generator = torch.Generator().manual_seed(42) ys = [torch.from_numpy(entry["target"]) for entry in batch] ys = [torch.atleast_2d(y) for y in ys] @@ -69,11 +69,11 @@ def predict( for i, x_i in enumerate(xs): x[i, :, -x_i.size(-1) :] = x_i - if ( + if ( # Keep room for at least one target: self.config.max_variates is not None - and x.size(1) > self.config.max_variates - 1 - ): # Keep room for at least one target: - perm = torch.randperm(x.size(1), generator=cpu_generator) + and x.size(1) >= self.config.max_variates + ): + perm = torch.randperm(x.size(1), generator=generator) x = x[:, perm[: self.config.max_variates - 1]] y = y.transpose(-1, -2).to( From 3fb4abc38ae1a8580f6f5094d1001969a21724a5 Mon Sep 17 00:00:00 2001 From: rusty1s Date: Mon, 26 Oct 2026 14:20:48 +0000 Subject: [PATCH 11/11] update --- benchmark/timeseries/fev_bench/evaluate.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/benchmark/timeseries/fev_bench/evaluate.py b/benchmark/timeseries/fev_bench/evaluate.py index c508e5fc1..70ed2e6cb 100644 --- a/benchmark/timeseries/fev_bench/evaluate.py +++ b/benchmark/timeseries/fev_bench/evaluate.py @@ -8,8 +8,8 @@ import fev import pandas as pd -from benchmark.forecasting.fev_bench import FEV_URL -from benchmark.forecasting.model import MODEL_CONFIGS +from benchmark.timeseries.fev_bench import FEV_BENCH_URL +from benchmark.timeseries.model import MODEL_CONFIGS BASELINE = "Seasonal Naive" REFERENCES = ("timesfm-3", "timesfm-2_5", "chronos-2", "seasonal_naive") @@ -35,7 +35,7 @@ def main() -> None: # Compare with published results on tasks run by all models: tasks = set.intersection(*(set(df["task_name"]) for df in summaries)) for name in REFERENCES: - path = f"{FEV_URL}/benchmarks/fev_bench/results/{name}.csv" + path = f"{FEV_BENCH_URL}/benchmarks/fev_bench/results/{name}.csv" summaries.append(pd.read_csv(path)) summaries = pd.concat(summaries) summaries = summaries[summaries["task_name"].isin(tasks)]