From 8c25ce03d405f62179966c603c9a5449104eca51 Mon Sep 17 00:00:00 2001 From: Filippo Guerranti Date: Tue, 13 Oct 2026 09:02:43 +0000 Subject: [PATCH 01/14] Add `sdm.evaluation.to_quantiles` --- docs/source/icl.md | 1 + sdm/evaluation/__init__.py | 2 + sdm/evaluation/regression.py | 61 ++++++++++++++++++++++++++++++ test/evaluation/test_regression.py | 26 +++++++++++++ 4 files changed, 90 insertions(+) create mode 100644 sdm/evaluation/regression.py create mode 100644 test/evaluation/test_regression.py diff --git a/docs/source/icl.md b/docs/source/icl.md index c8174aee4..d8e19fd70 100644 --- a/docs/source/icl.md +++ b/docs/source/icl.md @@ -98,6 +98,7 @@ Predictions are returned as a general {py:class}`~sdm.tensor.TableTensor`, where Since predictions are returned as {py:class}`~sdm.tensor.TableTensor`, you can zero-copy them to [`pandas`](https://pandas.pydata.org/docs), [`arrow`](https://arrow.apache.org/docs), or [`cudf`](https://docs.rapids.ai/api/cudf) via {py:meth}`~sdm.tensor.TableTensor.to_pandas`, {py:meth}`~sdm.tensor.TableTensor.to_arrow`, and {py:meth}`~sdm.tensor.TableTensor.to_cudf` for further downstream processing. In order to simplify metric calculation (*e.g.*, via [`torchmetrics`](https://lightning.ai/docs/torchmetrics)), we provide helper functions in the [`sdm.evaluation`](api/evaluation) package to convert target columns to class indices and align prediction columns to them (see {py:func}`~sdm.evaluation.to_class_indices` and {py:func}`~sdm.evaluation.to_binary_class`). +Similarly, {py:func}`~sdm.evaluation.to_quantiles` converts quantile predictions of one or more targets into a single tensor together with their quantile levels. ## Ensembling diff --git a/sdm/evaluation/__init__.py b/sdm/evaluation/__init__.py index 7d3b78251..08304db08 100644 --- a/sdm/evaluation/__init__.py +++ b/sdm/evaluation/__init__.py @@ -4,8 +4,10 @@ """Evaluation utilities.""" from sdm.evaluation.classification import to_class_indices, to_binary_class +from sdm.evaluation.regression import to_quantiles __all__ = [ "to_class_indices", "to_binary_class", + "to_quantiles", ] diff --git a/sdm/evaluation/regression.py b/sdm/evaluation/regression.py new file mode 100644 index 000000000..8ff24bc10 --- /dev/null +++ b/sdm/evaluation/regression.py @@ -0,0 +1,61 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from torch import Tensor + +from sdm import Stype, TableTensor + + +def to_quantiles(pred: TableTensor) -> tuple[Tensor, Tensor]: + r"""Convert quantile predictions to a per-target quantile tensor. + + Numerical columns must be named ``"q"`` for a single target or + ``"__q"`` for multiple targets, where the digits encode the + quantile level as a decimal fraction, *e.g.*, ``"q500"`` and ``"q50"`` both + denote the median. + Columns are expected to be grouped by target, with the same quantile levels + in the same order for every target. + + Args: + pred: Prediction table whose numerical columns contain quantiles. + + Returns: + The quantile predictions with shape ``[..., V, Q]`` for ``V`` targets + and ``Q`` quantile levels, and the quantile levels with shape ``[Q]``. + + >>> import torch + >>> import sdm + >>> from sdm.evaluation import to_quantiles + >>> pred = sdm.TableTensor.from_tensor( + ... torch.tensor([[1.0, 2.0, 3.0, 4.0]]), + ... columns=["q100", "q400", "q700", "q900"], + ... ) + >>> values, levels = to_quantiles(pred) + >>> values + tensor([[[1., 2., 3., 4.]]]) + >>> levels + tensor([0.1000, 0.4000, 0.7000, 0.9000]) + >>> pred = sdm.TableTensor.from_tensor( + ... torch.tensor([[1.0, 2.0, 3.0, 4.0]]), + ... columns=["a__q10", "a__q90", "b__q10", "b__q90"], + ... ) + >>> values, levels = to_quantiles(pred) + >>> values + tensor([[[1., 2.], + [3., 4.]]]) + >>> levels + tensor([0.1000, 0.9000]) + """ + columns = pred.columns[Stype.numerical] + prefixes, _, digits = zip(*(column.rpartition("q") for column in columns)) + targets = dict.fromkeys(prefixes) + digits = digits[: len(columns) // len(targets)] + if columns != tuple(f"{t}q{d}" for t in targets for d in digits): + raise ValueError( + "Expected quantile columns grouped by target with the same " + f"quantile levels (got {list(columns)})" + ) + + levels = [int(d) / 10 ** len(d) for d in digits] + values = pred.numerical.unflatten(-1, (len(targets), -1)) # [..., V, Q] + return values, values.new_tensor(levels) diff --git a/test/evaluation/test_regression.py b/test/evaluation/test_regression.py new file mode 100644 index 000000000..9ae0941a8 --- /dev/null +++ b/test/evaluation/test_regression.py @@ -0,0 +1,26 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import pytest +import torch + +import sdm + + +def test_to_quantiles() -> None: + pred = sdm.TableTensor.from_tensor( + torch.randn(2, 5, 4), + columns=["b__q10", "b__q90", "a__q10", "a__q90"], + ) + values, levels = sdm.evaluation.to_quantiles(pred) + torch.testing.assert_close(values, pred.numerical.view(2, 5, 2, 2)) + torch.testing.assert_close(levels, torch.tensor([0.1, 0.9])) + + +def test_to_quantiles_invalid() -> None: + pred = sdm.TableTensor.from_tensor( + torch.randn(5, 4), + columns=["a__q10", "b__q10", "a__q90", "b__q90"], + ) + with pytest.raises(ValueError, match="grouped by target"): + sdm.evaluation.to_quantiles(pred) From 1f4c5bbcf9c51867214ed78a9c596f4fb213b363 Mon Sep 17 00:00:00 2001 From: Filippo Guerranti Date: Tue, 13 Oct 2026 09:33:10 +0000 Subject: [PATCH 02/14] Add GIFT-Eval forecasting benchmark --- .gitignore | 3 + benchmark/README.md | 2 + benchmark/forecasting/README.md | 51 ++++++ benchmark/forecasting/__init__.py | 2 + benchmark/forecasting/gift_eval/__init__.py | 8 + benchmark/forecasting/gift_eval/evaluate.py | 62 +++++++ benchmark/forecasting/gift_eval/main.py | 172 ++++++++++++++++++++ benchmark/forecasting/gift_eval/models.py | 85 ++++++++++ benchmark/forecasting/model.py | 118 ++++++++++++++ 9 files changed, 503 insertions(+) create mode 100644 benchmark/forecasting/README.md create mode 100644 benchmark/forecasting/__init__.py create mode 100644 benchmark/forecasting/gift_eval/__init__.py create mode 100644 benchmark/forecasting/gift_eval/evaluate.py create mode 100644 benchmark/forecasting/gift_eval/main.py create mode 100644 benchmark/forecasting/gift_eval/models.py create mode 100644 benchmark/forecasting/model.py diff --git a/.gitignore b/.gitignore index 73fa2fdd2..8790ab32a 100644 --- a/.gitignore +++ b/.gitignore @@ -221,5 +221,8 @@ scoringbench_out/ evals/ datasets.json +# Forecasting benchmarks +gift_eval_out/ + # Checkpoint files *.pt diff --git a/benchmark/README.md b/benchmark/README.md index 964c325f6..36232fd7c 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -9,3 +9,5 @@ This folder contains benchmark integrations for `structured-data-models`: - [**`talent/`**](tabular/talent/): [TALENT](https://github.com/LAMDA-Tabular/TALENT) - [**`relational/`**](relational/): Relational benchmarks - [**`relarena/`**](relational/relarena): [RelArena](https://github.com/PriorLabs/relarena) +- [**`forecasting/`**](forecasting/): Time-series forecasting benchmarks + - [**`gift_eval/`**](forecasting/): [GIFT-Eval](https://github.com/SalesforceAIResearch/gift-eval) diff --git a/benchmark/forecasting/README.md b/benchmark/forecasting/README.md new file mode 100644 index 000000000..aee5f50a8 --- /dev/null +++ b/benchmark/forecasting/README.md @@ -0,0 +1,51 @@ +# Time-Series Forecasting Benchmarks + +This directory contains benchmarks of `structured-data-models` on [GIFT-Eval](https://github.com/SalesforceAIResearch/gift-eval). + +All benchmarks share one model adapter (see `benchmark/forecasting/model.py`) that maps each forecasting task onto the `ICLModel` interface: time steps become rows, past targets become `y_context`, all covariates and timestamps of the context become `x_context`, and the forecast horizon becomes `x_query` with one row of known covariates and timestamps per future time step. Columns keep their semantic types, so each model decides which ones it consumes. Series in a batch are left-padded with missing values, and multivariate series are flattened for models without multi-target support. + +______________________________________________________________________ + +> [!NOTE] +> Weights of `TimesFM3` are distributed under the [TimesFM Non-Commercial License v1.0](https://huggingface.co/google/timesfm-3.0-pytorch/blob/main/LICENSE). +> Review the license before running the `TimesFM3` benchmark, which will download its weights noninteractively. + +## GIFT-Eval + +### Setup + +Run the commands below from the repository root: + +```bash +pip install . gluonts datasets python-dotenv +pip install --no-deps "git+https://github.com/SalesforceAIResearch/gift-eval.git@577a65cfa3acb812405c00d989230bf59f7f8a3d" + +hf download Salesforce/GiftEval --repo-type dataset --local-dir /path/to/gift_eval +export GIFT_EVAL=/path/to/gift_eval +``` + +GIFT-Eval is not released on PyPI and is therefore pinned to a commit. It is installed without dependencies since it pins outdated `numpy` and `datasets` versions. + +### Run + +```bash +python -m benchmark.forecasting.gift_eval.main --model timesfm3 +``` + +Pass `--dataset` to run only selected GIFT-Eval datasets, *e.g.*, `--dataset hospital --dataset ett1/H`. +Use `--batch_size` to set the number of series per model call and `--max_context_size` to truncate long contexts. +Results are written to `benchmark/forecasting/gift_eval_out//`, and already completed configurations are skipped. + +### Evaluate + +```bash +python -m benchmark.forecasting.gift_eval.evaluate +``` + +This compares all available SDM results against published GIFT-Eval results on the same configurations, normalized by a seasonal naive baseline and aggregated by geometric mean. + +`TimesFM3` follows the published `TimesFM-3` evaluation with symmetric averaging over two estimators and contexts truncated to 15,360 time steps. Its results can still differ from the published ones: + +- Published results batch series of different context lengths together, which shifts their positional encodings on datasets with varying context lengths. +- Published results clip forecasts of non-negative series at zero. +- Published results forecast at most 32 variates jointly, subsampling covariates of `bizitobs_application` and `bizitobs_service`. diff --git a/benchmark/forecasting/__init__.py b/benchmark/forecasting/__init__.py new file mode 100644 index 000000000..d51c4fe1e --- /dev/null +++ b/benchmark/forecasting/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/benchmark/forecasting/gift_eval/__init__.py b/benchmark/forecasting/gift_eval/__init__.py new file mode 100644 index 000000000..417150a40 --- /dev/null +++ b/benchmark/forecasting/gift_eval/__init__.py @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Pinned GIFT-Eval revision for dataset properties and reference results: +GIFT_EVAL_URL = ( + "https://raw.githubusercontent.com/SalesforceAIResearch/gift-eval/" + "577a65cfa3acb812405c00d989230bf59f7f8a3d" +) diff --git a/benchmark/forecasting/gift_eval/evaluate.py b/benchmark/forecasting/gift_eval/evaluate.py new file mode 100644 index 000000000..d6176985d --- /dev/null +++ b/benchmark/forecasting/gift_eval/evaluate.py @@ -0,0 +1,62 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +r"""Evaluate SDM forecasting model results on GIFT-Eval.""" + +from pathlib import Path + +import numpy as np +import pandas as pd + +from benchmark.forecasting.gift_eval import GIFT_EVAL_URL +from benchmark.forecasting.model import MODEL_CONFIGS + +BASELINE = "Seasonal_Naive" +REFERENCES = ("TimesFM-3", "TimesFM-2.5", "Chronos-2", BASELINE) +METRICS = { + "eval_metrics/MASE[0.5]": "MASE", + "eval_metrics/mean_weighted_sum_quantile_loss": "CRPS", +} + + +def main() -> None: + benchmark_dir = Path(__file__).parent.parent + result_root = benchmark_dir / "gift_eval_out" + output_root = benchmark_dir / "evals" / "gift_eval" + + results = [] + for model_config in MODEL_CONFIGS.values(): + path = result_root / model_config.name / "all_results.csv" + if path.exists(): + results.append(pd.read_csv(path)) + + if not results: + raise FileNotFoundError( + f"No GIFT-Eval results found under {result_root}" + ) + + # Compare with published results on configurations run by all models: + configs = set.intersection(*(set(result["dataset"]) for result in results)) + for name in REFERENCES: + path = f"{GIFT_EVAL_URL}/results/{name}/all_results.csv" + results.append(pd.read_csv(path)) + + df = pd.concat(results) + df = df[df["dataset"].isin(configs)].set_index(["model", "dataset"]) + df = df[list(METRICS)].rename(columns=METRICS) + + # Normalize by the seasonal naive baseline and aggregate by geometric mean: + log_scores = np.log(df.div(df.loc[BASELINE], level="dataset")) + terms = log_scores.index.get_level_values("dataset").str.split("/").str[-1] + + output_root.mkdir(parents=True, exist_ok=True) + for term, scores in [("overall", log_scores), *log_scores.groupby(terms)]: + leaderboard = np.exp(scores.groupby("model").mean()) + leaderboard = leaderboard.sort_values("CRPS") + leaderboard.to_csv(output_root / f"leaderboard_{term}.csv") + print(f"\nGIFT-Eval leaderboard ({term}):") + print(leaderboard.to_string(float_format="%.4f")) + + +if __name__ == "__main__": + main() diff --git a/benchmark/forecasting/gift_eval/main.py b/benchmark/forecasting/gift_eval/main.py new file mode 100644 index 000000000..513815328 --- /dev/null +++ b/benchmark/forecasting/gift_eval/main.py @@ -0,0 +1,172 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +r"""Run an SDM forecasting model on GIFT-Eval.""" + +import argparse +from dataclasses import replace +from pathlib import Path + +import pandas as pd +import torch +from gift_eval.data import Dataset +from gluonts.ev.metrics import ( + MAE, + MAPE, + MASE, + MSE, + MSIS, + ND, + NRMSE, + RMSE, + SMAPE, + MeanWeightedSumQuantileLoss, +) +from gluonts.model import evaluate_model +from gluonts.time_feature import get_seasonality + +from benchmark.forecasting.gift_eval import GIFT_EVAL_URL +from benchmark.forecasting.gift_eval.models import ( + QUANTILE_LEVELS, + SDMPredictor, +) +from benchmark.forecasting.model import MODEL_CONFIGS, add_model_args + +# Datasets and terms from GIFT-Eval's notebooks: +SHORT_DATASETS = ( # noqa: SIM905 + "m4_yearly m4_quarterly m4_monthly m4_weekly m4_daily m4_hourly " + "electricity/15T electricity/H electricity/D electricity/W " + "solar/10T solar/H solar/D solar/W hospital covid_deaths " + "us_births/D us_births/M us_births/W " + "saugeenday/D saugeenday/M saugeenday/W " + "temperature_rain_with_missing " + "kdd_cup_2018_with_missing/H kdd_cup_2018_with_missing/D " + "car_parts_with_missing restaurant " + "hierarchical_sales/D hierarchical_sales/W " + "LOOP_SEATTLE/5T LOOP_SEATTLE/H LOOP_SEATTLE/D SZ_TAXI/15T SZ_TAXI/H " + "M_DENSE/H M_DENSE/D ett1/15T ett1/H ett1/D ett1/W " + "ett2/15T ett2/H ett2/D ett2/W jena_weather/10T jena_weather/H " + "jena_weather/D bitbrains_fast_storage/5T bitbrains_fast_storage/H " + "bitbrains_rnd/5T bitbrains_rnd/H bizitobs_application bizitobs_service " + "bizitobs_l2c/5T bizitobs_l2c/H" +).split() +MED_LONG_DATASETS = ( # noqa: SIM905 + "electricity/15T electricity/H solar/10T solar/H " + "kdd_cup_2018_with_missing/H LOOP_SEATTLE/5T LOOP_SEATTLE/H SZ_TAXI/15T " + "M_DENSE/H ett1/15T ett1/H ett2/15T ett2/H jena_weather/10T " + "jena_weather/H bitbrains_fast_storage/5T bitbrains_rnd/5T " + "bizitobs_application bizitobs_service bizitobs_l2c/5T bizitobs_l2c/H" +).split() +PRETTY_NAMES = { + "saugeenday": "saugeen", + "temperature_rain_with_missing": "temperature_rain", + "kdd_cup_2018_with_missing": "kdd_cup_2018", + "car_parts_with_missing": "car_parts", +} +METRICS = [ + MSE(forecast_type="mean"), + MSE(forecast_type="0.5"), + MAE(), + MASE(), + MAPE(), + SMAPE(), + MSIS(), + RMSE(), + NRMSE(), + ND(), + MeanWeightedSumQuantileLoss(quantile_levels=QUANTILE_LEVELS), +] + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + add_model_args(parser) + parser.add_argument( + "--dataset", + action="append", + help="Run only the selected GIFT-Eval dataset; repeat to combine.", + ) + args = parser.parse_args() + + model_config = MODEL_CONFIGS[args.model] + if args.max_context_size is not None: + model_config = replace( + model_config, max_context_size=args.max_context_size + ) + result_path = ( + Path(__file__).parent.parent + / "gift_eval_out" + / model_config.name + / "all_results.csv" + ) + result_path.parent.mkdir(parents=True, exist_ok=True) + completed = ( + set(pd.read_csv(result_path)["dataset"]) + if result_path.exists() + else set() + ) + properties = pd.read_json( + f"{GIFT_EVAL_URL}/notebooks/dataset_properties.json" + ) + + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + model = model_config.factory(device) + + for name in args.dataset or SHORT_DATASETS: + key, _, freq = name.partition("/") + key = PRETTY_NAMES.get(key.lower(), key.lower()) + freq = freq or properties[key]["frequency"] + for term in ("short", "medium", "long"): + if term != "short" and name not in MED_LONG_DATASETS: + continue + config = f"{key}/{freq}/{term}" + if config in completed: + continue + + # Flatten multivariate series for models without joint multi-target + # forecasting: + dataset = Dataset( + name=name, + term=term, + to_univariate=( + not model.supports_multi_target + and properties[key]["num_variates"] > 1 + ), + ) + predictor = SDMPredictor( + model, + config=model_config, + prediction_length=dataset.prediction_length, + batch_size=args.batch_size, + ) + result = evaluate_model( + predictor, + test_data=dataset.test_data, + metrics=METRICS, + seasonality=get_seasonality(dataset.freq), + ) + row = pd.DataFrame( + [ + { + "dataset": config, + "model": f"[SDM] {model_config.name}", + **{ + f"eval_metrics/{k}": v + for k, v in result.iloc[0].items() + }, + "domain": properties[key]["domain"], + "num_variates": properties[key]["num_variates"], + } + ] + ) + row.to_csv( + result_path, + mode="a", + header=not result_path.exists(), + index=False, + ) + print(row.to_string(index=False)) + + +if __name__ == "__main__": + main() diff --git a/benchmark/forecasting/gift_eval/models.py b/benchmark/forecasting/gift_eval/models.py new file mode 100644 index 000000000..c14b7f254 --- /dev/null +++ b/benchmark/forecasting/gift_eval/models.py @@ -0,0 +1,85 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""SDM forecasting adapter for GIFT-Eval.""" + +from collections.abc import Iterator + +import numpy as np +import pandas as pd +from gluonts.dataset import Dataset +from gluonts.itertools import batcher +from gluonts.model.forecast import QuantileForecast +from gluonts.model.predictor import Predictor + +import sdm +from benchmark.forecasting.model import ModelConfig, forecast, select_levels + +QUANTILE_LEVELS = (0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9) + + +class SDMPredictor(Predictor): + """GluonTS predictor forecasting batches of series with an SDM model.""" + + def __init__( + self, + model: sdm.models.ICLModel, + config: ModelConfig, + prediction_length: int, + batch_size: int, + ) -> None: + super().__init__(prediction_length=prediction_length) + self.model = model + self.config = config + self.batch_size = batch_size + + def predict( + self, + dataset: Dataset, + **kwargs: object, + ) -> Iterator[QuantileForecast]: + for batch in batcher(dataset, batch_size=self.batch_size): + # GluonTS stores series as [V, T] (or [T] if univariate): + num_variates = np.atleast_2d(batch[0]["target"]).shape[0] + targets = [f"target_{i}" for i in range(num_variates)] + past, future = [], [] + for entry in batch: + target = np.atleast_2d(entry["target"]) + length = target.shape[-1] + timestamp = pd.period_range( + entry["start"], + periods=length + self.prediction_length, + ).to_timestamp() + columns = { + "item_id": entry["item_id"], + "timestamp": timestamp[:length], + } + columns |= dict(zip(targets, target, strict=True)) + for i, x in enumerate(entry.get("past_feat_dynamic_real", [])): + columns[f"past_feat_dynamic_real_{i}"] = x + past.append(pd.DataFrame(columns)) + future.append( + pd.DataFrame( + { + "item_id": entry["item_id"], + "timestamp": timestamp[length:], + } + ) + ) + values, levels = forecast( + self.model, + config=self.config, + past=past, + future=future, + targets=targets, + ) + values = select_levels(values, levels, QUANTILE_LEVELS) + for entry, value in zip(batch, values, strict=True): + value = value.transpose(2, 0, 1) # [Q, H, V] + if entry["target"].ndim == 1: + value = value[..., 0] + yield QuantileForecast( + forecast_arrays=value, + forecast_keys=[str(q) for q in QUANTILE_LEVELS], + start_date=entry["start"] + entry["target"].shape[-1], + ) diff --git a/benchmark/forecasting/model.py b/benchmark/forecasting/model.py new file mode 100644 index 000000000..13304709b --- /dev/null +++ b/benchmark/forecasting/model.py @@ -0,0 +1,118 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""SDM model adapter shared by the time-series forecasting benchmarks.""" + +import argparse +from collections.abc import Callable, Sequence +from dataclasses import dataclass + +import numpy as np +import pandas as pd +import torch + +import sdm +from sdm.models.timesfm3 import TimesFM3 + + +@dataclass(frozen=True) +class ModelConfig: + name: str + factory: Callable[[torch.device], sdm.models.ICLModel] + num_estimators: int = 1 + max_context_size: int | None = None + + +MODEL_CONFIGS = { + "timesfm3": ModelConfig( + name="TimesFM3", + factory=lambda device: TimesFM3(accept_license=True, device=device), + num_estimators=2, + max_context_size=15360, + ), +} + + +def add_model_args(parser: argparse.ArgumentParser) -> None: + """Add the model flags shared by all forecasting benchmarks.""" + parser.add_argument( + "--model", + choices=tuple(MODEL_CONFIGS), + default="timesfm3", + help="SDM model to benchmark.", + ) + parser.add_argument( + "--batch_size", + type=int, + default=64, + help="Number of series forecast per model call.", + ) + parser.add_argument( + "--max_context_size", + type=int, + help="Truncate each context to its most recent time steps " + "(defaults to the model's maximum context size).", + ) + + +def forecast( + model: sdm.models.ICLModel, + config: ModelConfig, + past: Sequence[pd.DataFrame], # B x [T_i, V + C] + future: Sequence[pd.DataFrame], # B x [H, K] + targets: Sequence[str], +) -> tuple[np.ndarray, np.ndarray]: # [B, H, V, Q], [Q] + """Forecast the quantiles of a batch of (multivariate) time series. + + Time steps map to rows: ``past`` holds the targets and all covariates of + the context, while ``future`` holds the covariates known over the + forecast horizon. Columns keep their semantic types, so that the model + decides which ones to consume. Missing values are passed through. + """ + device = next(model.parameters()).device + num_series, horizon = len(past), len(future[0]) + size = max(len(df) for df in past) + if config.max_context_size is not None: + size = min(size, config.max_context_size) + + # Left-pad shorter series with missing values so that all series end at + # the forecast start: + past = [df.iloc[-size:].reset_index(drop=True) for df in past] + past = [df.reindex(range(len(df) - size, len(df))) for df in past] + + # Infer semantic types jointly so that context and query share them: + df = pd.concat([*past, *future], ignore_index=True) + table = sdm.TableTensor.from_pandas( + df=df, + stypes=sdm.infer_stypes(df, id="infer"), + device=device, + ) + # Split rows into the context [B, T, V + C] and the query [B, H, V + C]: + context = table[: num_series * size].view(num_series, size, -1) + query = table[num_series * size :].view(num_series, horizon, -1) + assert isinstance(context, sdm.TableTensor) + assert isinstance(query, sdm.TableTensor) + + out = model( + x_context=context.drop_columns(targets), + y_context=context.select_columns(targets), + x_query=query.select_columns(future[0].columns), + num_estimators=config.num_estimators, + ) + values, levels = sdm.evaluation.to_quantiles(out) + return values.cpu().numpy(), levels.cpu().numpy() + + +def select_levels( + values: np.ndarray, # [..., Q] + levels: np.ndarray, # [Q] + wanted: Sequence[float], +) -> np.ndarray: # [..., len(wanted)] + """Select the quantile levels requested by a benchmark.""" + index = [int(np.abs(levels - level).argmin()) for level in wanted] + if not np.allclose(levels[index], wanted): + raise ValueError( + f"Model does not predict quantile levels {list(wanted)} " + f"(got {levels.tolist()})" + ) + return values[..., index] From 7497cf1d92cc435890cb9017c9075235bcfe73b0 Mon Sep 17 00:00:00 2001 From: Filippo Guerranti Date: Tue, 13 Oct 2026 15:59:29 +0000 Subject: [PATCH 03/14] Add fev-bench forecasting benchmark --- .gitignore | 1 + benchmark/README.md | 1 + benchmark/forecasting/README.md | 35 ++++++++- benchmark/forecasting/fev_bench/__init__.py | 5 ++ benchmark/forecasting/fev_bench/evaluate.py | 60 ++++++++++++++ benchmark/forecasting/fev_bench/main.py | 87 +++++++++++++++++++++ benchmark/forecasting/fev_bench/models.py | 87 +++++++++++++++++++++ benchmark/forecasting/model.py | 4 +- 8 files changed, 278 insertions(+), 2 deletions(-) create mode 100644 benchmark/forecasting/fev_bench/__init__.py create mode 100644 benchmark/forecasting/fev_bench/evaluate.py create mode 100644 benchmark/forecasting/fev_bench/main.py create mode 100644 benchmark/forecasting/fev_bench/models.py diff --git a/.gitignore b/.gitignore index 8790ab32a..aea92019b 100644 --- a/.gitignore +++ b/.gitignore @@ -223,6 +223,7 @@ datasets.json # Forecasting benchmarks gift_eval_out/ +fev_bench_out/ # Checkpoint files *.pt diff --git a/benchmark/README.md b/benchmark/README.md index 36232fd7c..8ce0c1c59 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -11,3 +11,4 @@ This folder contains benchmark integrations for `structured-data-models`: - [**`relarena/`**](relational/relarena): [RelArena](https://github.com/PriorLabs/relarena) - [**`forecasting/`**](forecasting/): Time-series forecasting benchmarks - [**`gift_eval/`**](forecasting/): [GIFT-Eval](https://github.com/SalesforceAIResearch/gift-eval) + - [**`fev_bench/`**](forecasting/): [fev-bench](https://github.com/autogluon/fev) diff --git a/benchmark/forecasting/README.md b/benchmark/forecasting/README.md index aee5f50a8..e15986cd2 100644 --- a/benchmark/forecasting/README.md +++ b/benchmark/forecasting/README.md @@ -1,6 +1,6 @@ # Time-Series Forecasting Benchmarks -This directory contains benchmarks of `structured-data-models` on [GIFT-Eval](https://github.com/SalesforceAIResearch/gift-eval). +This directory contains benchmarks of `structured-data-models` on [GIFT-Eval](https://github.com/SalesforceAIResearch/gift-eval) and [fev-bench](https://github.com/autogluon/fev). All benchmarks share one model adapter (see `benchmark/forecasting/model.py`) that maps each forecasting task onto the `ICLModel` interface: time steps become rows, past targets become `y_context`, all covariates and timestamps of the context become `x_context`, and the forecast horizon becomes `x_query` with one row of known covariates and timestamps per future time step. Columns keep their semantic types, so each model decides which ones it consumes. Series in a batch are left-padded with missing values, and multivariate series are flattened for models without multi-target support. @@ -49,3 +49,36 @@ This compares all available SDM results against published GIFT-Eval results on t - Published results batch series of different context lengths together, which shifts their positional encodings on datasets with varying context lengths. - Published results clip forecasts of non-negative series at zero. - Published results forecast at most 32 variates jointly, subsampling covariates of `bizitobs_application` and `bizitobs_service`. + +______________________________________________________________________ + +## fev-bench + +### Setup + +Run the command below from the repository root: + +```bash +pip install . fev==0.10.0 +``` + +Tasks are downloaded from Hugging Face on first use. + +### Run + +```bash +python -m benchmark.forecasting.fev_bench.main --model timesfm3 +``` + +Pass `--task` to run only selected fev-bench tasks, *e.g.*, `--task hospital --task proenfo_gfc12`. +Results are written to `benchmark/forecasting/fev_bench_out//`, and already completed tasks are skipped. + +### Evaluate + +```bash +python -m benchmark.forecasting.fev_bench.evaluate +``` + +This compares all available SDM results against published fev-bench results on the same tasks with `fev.leaderboard`, using a seasonal naive baseline. + +In addition to the differences listed for GIFT-Eval, published `TimesFM-3` results impute missing values with the last observed value, convert non-numerical covariates to numbers, and ignore static covariates. diff --git a/benchmark/forecasting/fev_bench/__init__.py b/benchmark/forecasting/fev_bench/__init__.py new file mode 100644 index 000000000..6fdaa7520 --- /dev/null +++ b/benchmark/forecasting/fev_bench/__init__.py @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Task definitions and reference results of the installed fev release: +FEV_URL = "https://raw.githubusercontent.com/autogluon/fev/v0.10.0" diff --git a/benchmark/forecasting/fev_bench/evaluate.py b/benchmark/forecasting/fev_bench/evaluate.py new file mode 100644 index 000000000..3b9c278f4 --- /dev/null +++ b/benchmark/forecasting/fev_bench/evaluate.py @@ -0,0 +1,60 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +r"""Evaluate SDM forecasting model results on fev-bench.""" + +from pathlib import Path + +import fev +import pandas as pd + +from benchmark.forecasting.fev_bench import FEV_URL +from benchmark.forecasting.model import MODEL_CONFIGS + +BASELINE = "Seasonal Naive" +REFERENCES = ("timesfm-3", "timesfm-2_5", "chronos-2", "seasonal_naive") +METRICS = ("SQL", "MASE") + + +def main() -> None: + benchmark_dir = Path(__file__).parent.parent + result_root = benchmark_dir / "fev_bench_out" + output_root = benchmark_dir / "evals" / "fev_bench" + + summaries = [] + for model_config in MODEL_CONFIGS.values(): + path = result_root / model_config.name / "results.csv" + if path.exists(): + summaries.append(pd.read_csv(path)) + + if not summaries: + raise FileNotFoundError( + f"No fev-bench results found under {result_root}" + ) + + # Compare with published results on tasks run by all models: + tasks = set.intersection(*(set(df["task_name"]) for df in summaries)) + for name in REFERENCES: + path = f"{FEV_URL}/benchmarks/fev_bench/results/{name}.csv" + summaries.append(pd.read_csv(path)) + summaries = pd.concat(summaries) + summaries = summaries[summaries["task_name"].isin(tasks)] + + output_root.mkdir(parents=True, exist_ok=True) + for metric in METRICS: + leaderboard = fev.leaderboard( + summaries, + metric_column=metric, + baseline_model=BASELINE, + ) + leaderboard.to_csv(output_root / f"leaderboard_{metric}.csv") + print(f"\nfev-bench leaderboard ({metric}, {len(tasks)} tasks):") + print( + leaderboard[["skill_score", "win_rate"]].to_string( + float_format="%.4f" + ) + ) + + +if __name__ == "__main__": + main() diff --git a/benchmark/forecasting/fev_bench/main.py b/benchmark/forecasting/fev_bench/main.py new file mode 100644 index 000000000..ba0c44fb4 --- /dev/null +++ b/benchmark/forecasting/fev_bench/main.py @@ -0,0 +1,87 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +r"""Run an SDM forecasting model on fev-bench.""" + +import argparse +import json +from dataclasses import replace +from pathlib import Path + +import fev +import pandas as pd +import torch + +from benchmark.forecasting.fev_bench import FEV_URL +from benchmark.forecasting.fev_bench.models import SDMModel +from benchmark.forecasting.model import MODEL_CONFIGS, add_model_args + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + add_model_args(parser) + parser.add_argument( + "--task", + action="append", + help="Run only the selected fev-bench task; repeat to combine.", + ) + args = parser.parse_args() + + model_config = MODEL_CONFIGS[args.model] + if args.max_context_size is not None: + model_config = replace( + model_config, max_context_size=args.max_context_size + ) + result_path = ( + Path(__file__).parent.parent + / "fev_bench_out" + / model_config.name + / "results.csv" + ) + result_path.parent.mkdir(parents=True, exist_ok=True) + completed = ( + set(pd.read_csv(result_path)["task_name"]) + if result_path.exists() + else set() + ) + + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + model = SDMModel( + model_config.factory(device), + config=model_config, + batch_size=args.batch_size, + ) + + # Mirrors fev's official `models/evaluate.py` runner: + benchmark = fev.Benchmark.from_yaml( + f"{FEV_URL}/benchmarks/fev_bench/tasks.yaml" + ) + for task in benchmark.tasks: + if args.task is not None and task.task_name not in args.task: + continue + if task.task_name in completed: + continue + + predictions = model.fit_predict(task) + summary = task.evaluation_summary( + predictions, + model_name=f"[SDM] {model_config.name}", + training_time_s=model.training_time, + inference_time_s=model.inference_time, + trained_on_this_dataset=( + task.dataset_config in model.trained_on_datasets + ), + extra_info={"model_kwargs": json.dumps(vars(args))}, + ) + row = pd.DataFrame([summary]) + row.to_csv( + result_path, + mode="a", + header=not result_path.exists(), + index=False, + ) + print(row[["task_name", "test_error"]].to_string(index=False)) + + +if __name__ == "__main__": + main() diff --git a/benchmark/forecasting/fev_bench/models.py b/benchmark/forecasting/fev_bench/models.py new file mode 100644 index 000000000..eba1ac6ee --- /dev/null +++ b/benchmark/forecasting/fev_bench/models.py @@ -0,0 +1,87 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""SDM forecasting adapter for fev-bench.""" + +from collections.abc import Iterator + +import datasets +import fev +import numpy as np +import pandas as pd + +import sdm +from benchmark.forecasting.model import ModelConfig, forecast, select_levels + + +class SDMModel(fev.ForecastingModel): + """fev model forecasting batches of series with an SDM model.""" + + def __init__( + self, + model: sdm.models.ICLModel, + config: ModelConfig, + batch_size: int, + ) -> None: + super().__init__() + self.model = model + self.config = config + self.batch_size = batch_size + + def _fit_predict(self, task: fev.Task) -> list[datasets.DatasetDict]: + predictions = [] + for window in task.iter_windows(): + past, future = window.get_input_data() + + values = [] + with self._record_inference_time(): + for past_batch, future_batch in zip( + _to_frames(past, self.batch_size), + _to_frames(future, self.batch_size), + strict=True, + ): + out, levels = forecast( + self.model, + config=self.config, + past=past_batch, + future=future_batch, + targets=task.target_columns, + ) + # Median point forecast followed by the task quantiles: + wanted = [0.5, *task.quantile_levels] + values.append(select_levels(out, levels, wanted)) + values = np.concatenate(values) # [N, H, V, 1 + Q] + + window_predictions = datasets.DatasetDict() + for i, column in enumerate(task.target_columns): + window_predictions[column] = datasets.Dataset.from_dict( + { + "predictions": values[:, :, i, 0], + **{ + str(level): values[:, :, i, j + 1] + for j, level in enumerate(task.quantile_levels) + }, + } + ) + predictions.append(window_predictions) + return predictions + + +def _to_frames( + dataset: datasets.Dataset, + batch_size: int, +) -> Iterator[list[pd.DataFrame]]: + # One frame per series, with static values repeated over time steps and + # the column types declared by the benchmark: + dtypes = { + name: pd.ArrowDtype(getattr(feature, "feature", feature).pa_type) + for name, feature in dataset.features.items() + } + for batch in dataset.with_format("numpy").iter(batch_size): + num_series = len(next(iter(batch.values()))) + yield [ + pd.DataFrame({name: x[i] for name, x in batch.items()}).astype( + dtypes + ) + for i in range(num_series) + ] diff --git a/benchmark/forecasting/model.py b/benchmark/forecasting/model.py index 13304709b..0d2b4b1fb 100644 --- a/benchmark/forecasting/model.py +++ b/benchmark/forecasting/model.py @@ -80,8 +80,10 @@ def forecast( past = [df.iloc[-size:].reset_index(drop=True) for df in past] past = [df.reindex(range(len(df) - size, len(df))) for df in past] - # Infer semantic types jointly so that context and query share them: + # Infer semantic types jointly so that context and query share them, and + # order targets first so that forecasts follow ``targets``: df = pd.concat([*past, *future], ignore_index=True) + df = df[[*targets, *df.columns.drop(targets)]] table = sdm.TableTensor.from_pandas( df=df, stypes=sdm.infer_stypes(df, id="infer"), From e980c312d8fb345ce2bd49ad503f9d88b8dcd2b7 Mon Sep 17 00:00:00 2001 From: Filippo Guerranti Date: Tue, 13 Oct 2026 16:15:03 +0000 Subject: [PATCH 04/14] Limit variates per forward pass for TimesFM3 benchmarks --- benchmark/forecasting/README.md | 3 +- benchmark/forecasting/model.py | 79 +++++++++++++++++++++++++++++---- 2 files changed, 72 insertions(+), 10 deletions(-) diff --git a/benchmark/forecasting/README.md b/benchmark/forecasting/README.md index e15986cd2..a3df47e96 100644 --- a/benchmark/forecasting/README.md +++ b/benchmark/forecasting/README.md @@ -44,11 +44,10 @@ python -m benchmark.forecasting.gift_eval.evaluate This compares all available SDM results against published GIFT-Eval results on the same configurations, normalized by a seasonal naive baseline and aggregated by geometric mean. -`TimesFM3` follows the published `TimesFM-3` evaluation with symmetric averaging over two estimators and contexts truncated to 15,360 time steps. Its results can still differ from the published ones: +`TimesFM3` follows the published `TimesFM-3` evaluation with symmetric averaging over two estimators, contexts truncated to 15,360 time steps, and at most 32 variates per forward pass. Its results can still differ from the published ones: - Published results batch series of different context lengths together, which shifts their positional encodings on datasets with varying context lengths. - Published results clip forecasts of non-negative series at zero. -- Published results forecast at most 32 variates jointly, subsampling covariates of `bizitobs_application` and `bizitobs_service`. ______________________________________________________________________ diff --git a/benchmark/forecasting/model.py b/benchmark/forecasting/model.py index 0d2b4b1fb..e05fc902c 100644 --- a/benchmark/forecasting/model.py +++ b/benchmark/forecasting/model.py @@ -4,7 +4,7 @@ """SDM model adapter shared by the time-series forecasting benchmarks.""" import argparse -from collections.abc import Callable, Sequence +from collections.abc import Callable, Collection, Sequence from dataclasses import dataclass import numpy as np @@ -21,6 +21,7 @@ class ModelConfig: factory: Callable[[torch.device], sdm.models.ICLModel] num_estimators: int = 1 max_context_size: int | None = None + max_variates: int | None = None MODEL_CONFIGS = { @@ -29,6 +30,7 @@ class ModelConfig: factory=lambda device: TimesFM3(accept_license=True, device=device), num_estimators=2, max_context_size=15360, + max_variates=32, ), } @@ -95,16 +97,77 @@ def forecast( assert isinstance(context, sdm.TableTensor) assert isinstance(query, sdm.TableTensor) - out = model( - x_context=context.drop_columns(targets), - y_context=context.select_columns(targets), - x_query=query.select_columns(future[0].columns), - num_estimators=config.num_estimators, - ) - values, levels = sdm.evaluation.to_quantiles(out) + x_context = context.drop_columns(targets) + x_query = query.select_columns(future[0].columns) + y = context.select_columns(targets).numerical # [B, T, V] + + chunks = [list(range(len(targets)))] + if config.max_variates is not None: + x_context, x_query, chunks = _split_variates( + x_context, + x_query, + num_targets=len(targets), + max_variates=config.max_variates, + stypes=model.supported_feature_stypes, + ) + + values = [] + for index in chunks: + out = model( + x_context=x_context, + y_context=sdm.TableTensor.from_tensor(y[..., index]), + x_query=x_query, + num_estimators=config.num_estimators, + ) + chunk, levels = sdm.evaluation.to_quantiles(out) # [B, H, V', Q] + values.append(chunk) + # Only the last chunk can end with filler targets: + values = torch.cat(values, dim=-2)[..., : len(targets), :] # [B, H, V, Q] return values.cpu().numpy(), levels.cpu().numpy() +def _split_variates( + x_context: sdm.TableTensor, + x_query: sdm.TableTensor, + num_targets: int, + max_variates: int, + stypes: Collection[sdm.Stype], + seed: int = 42, +) -> tuple[sdm.TableTensor, sdm.TableTensor, list[list[int]]]: + """Split targets and covariates into passes of at most ``max_variates``. + + Covariates of the given semantic types count as variates. If there are + too many, covariates are randomly subsampled (future-known ones first) to + leave room for at least one target. Targets are then split into chunks of + equal size, padding the last chunk with leading targets. Returns the + remaining covariates and the target indices of each chunk. + """ + covariates = [ + c for c, stype in x_context.stypes.items() if stype in stypes + ] + known = [c for c in covariates if c in x_query.column_names] + past = [c for c in covariates if c not in x_query.column_names] + + rng = np.random.default_rng(seed) + keep: list[str] = [] + for columns in (known, past): + size = min(len(columns), max_variates - 1 - len(keep)) + if size < len(columns): + index = np.sort(rng.choice(len(columns), size, replace=False)) + columns = [columns[i] for i in index] + keep += columns + drop = set(covariates).difference(keep) + x_context = x_context.drop_columns(drop) + x_query = x_query.drop_columns(drop & x_query.column_names) + + size = min(max_variates - len(keep), num_targets) + chunks = [ + [i % num_targets for i in range(start, start + size)] + for start in range(0, num_targets, size) + ] + return x_context, x_query, chunks + + def select_levels( values: np.ndarray, # [..., Q] levels: np.ndarray, # [Q] From 7d29f0d466b8556119bd15342f7d20a64d8c4178 Mon Sep 17 00:00:00 2001 From: Filippo Guerranti Date: Tue, 13 Oct 2026 16:46:25 +0000 Subject: [PATCH 05/14] Prepare fev-bench data for timesfm3 as in upstream --- benchmark/forecasting/README.md | 2 +- benchmark/forecasting/fev_bench/main.py | 4 +-- benchmark/forecasting/fev_bench/models.py | 36 +++++++++++++++++++++-- 3 files changed, 37 insertions(+), 5 deletions(-) diff --git a/benchmark/forecasting/README.md b/benchmark/forecasting/README.md index a3df47e96..93fc1c336 100644 --- a/benchmark/forecasting/README.md +++ b/benchmark/forecasting/README.md @@ -80,4 +80,4 @@ python -m benchmark.forecasting.fev_bench.evaluate This compares all available SDM results against published fev-bench results on the same tasks with `fev.leaderboard`, using a seasonal naive baseline. -In addition to the differences listed for GIFT-Eval, published `TimesFM-3` results impute missing values with the last observed value, convert non-numerical covariates to numbers, and ignore static covariates. +On fev-bench, `TimesFM3` also imputes missing values with the last observed value, converts dynamic covariates to numbers, and ignores static covariates. In addition to the differences listed for GIFT-Eval, published results hash string covariates with a randomly seeded instead of a fixed hash function. diff --git a/benchmark/forecasting/fev_bench/main.py b/benchmark/forecasting/fev_bench/main.py index ba0c44fb4..d545e1f60 100644 --- a/benchmark/forecasting/fev_bench/main.py +++ b/benchmark/forecasting/fev_bench/main.py @@ -13,7 +13,7 @@ import torch from benchmark.forecasting.fev_bench import FEV_URL -from benchmark.forecasting.fev_bench.models import SDMModel +from benchmark.forecasting.fev_bench.models import MODELS from benchmark.forecasting.model import MODEL_CONFIGS, add_model_args @@ -46,7 +46,7 @@ def main() -> None: ) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - model = SDMModel( + model = MODELS[args.model]( model_config.factory(device), config=model_config, batch_size=args.batch_size, diff --git a/benchmark/forecasting/fev_bench/models.py b/benchmark/forecasting/fev_bench/models.py index eba1ac6ee..c8854ef59 100644 --- a/benchmark/forecasting/fev_bench/models.py +++ b/benchmark/forecasting/fev_bench/models.py @@ -3,6 +3,7 @@ """SDM forecasting adapter for fev-bench.""" +import zlib from collections.abc import Iterator import datasets @@ -43,8 +44,10 @@ def _fit_predict(self, task: fev.Task) -> list[datasets.DatasetDict]: out, levels = forecast( self.model, config=self.config, - past=past_batch, - future=future_batch, + past=[self._prepare(task, df) for df in past_batch], + future=[ + self._prepare(task, df) for df in future_batch + ], targets=task.target_columns, ) # Median point forecast followed by the task quantiles: @@ -66,6 +69,35 @@ def _fit_predict(self, task: fev.Task) -> list[datasets.DatasetDict]: predictions.append(window_predictions) return predictions + def _prepare(self, task: fev.Task, df: pd.DataFrame) -> pd.DataFrame: + """Prepare the past or future data of a series for the model.""" + return df + + +class TimesFM3Model(SDMModel): + """fev model for ``TimesFM3``. + + Static covariates are ignored, while targets and dynamic covariates are + converted to numbers (hashing strings into 100 buckets) and imputed with + the last observed value. + """ + + model_name = "sdm-timesfm3" + + def _prepare(self, task: fev.Task, df: pd.DataFrame) -> pd.DataFrame: + df = df.drop(columns=task.static_columns) + for column in df.columns.drop([task.id_column, task.timestamp_column]): + x = df[column] + if pd.api.types.is_string_dtype(x): + x = x.fillna("").map(lambda v: zlib.crc32(v.encode()) % 100) + df[column] = x.astype("float32").ffill().bfill().fillna(0) + return df + + +MODELS: dict[str, type[SDMModel]] = { + "timesfm3": TimesFM3Model, +} + def _to_frames( dataset: datasets.Dataset, From bc73d3b5ada93f008595210d3d2fdfb2739d8a3c Mon Sep 17 00:00:00 2001 From: Filippo Guerranti Date: Wed, 14 Oct 2026 11:33:18 +0000 Subject: [PATCH 06/14] Add TIME forecasting benchmark --- .gitignore | 1 + benchmark/README.md | 1 + benchmark/forecasting/README.md | 42 ++++++++- benchmark/forecasting/time_bench/__init__.py | 2 + benchmark/forecasting/time_bench/evaluate.py | 83 +++++++++++++++++ benchmark/forecasting/time_bench/main.py | 93 ++++++++++++++++++++ 6 files changed, 221 insertions(+), 1 deletion(-) create mode 100644 benchmark/forecasting/time_bench/__init__.py create mode 100644 benchmark/forecasting/time_bench/evaluate.py create mode 100644 benchmark/forecasting/time_bench/main.py diff --git a/.gitignore b/.gitignore index aea92019b..f64e13ac8 100644 --- a/.gitignore +++ b/.gitignore @@ -224,6 +224,7 @@ datasets.json # Forecasting benchmarks gift_eval_out/ fev_bench_out/ +time_bench_out/ # Checkpoint files *.pt diff --git a/benchmark/README.md b/benchmark/README.md index 8ce0c1c59..6ab854392 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -12,3 +12,4 @@ This folder contains benchmark integrations for `structured-data-models`: - [**`forecasting/`**](forecasting/): Time-series forecasting benchmarks - [**`gift_eval/`**](forecasting/): [GIFT-Eval](https://github.com/SalesforceAIResearch/gift-eval) - [**`fev_bench/`**](forecasting/): [fev-bench](https://github.com/autogluon/fev) + - [**`time_bench/`**](forecasting/): [TIME](https://github.com/zqiao11/TIME) diff --git a/benchmark/forecasting/README.md b/benchmark/forecasting/README.md index 93fc1c336..c529b7512 100644 --- a/benchmark/forecasting/README.md +++ b/benchmark/forecasting/README.md @@ -1,6 +1,6 @@ # Time-Series Forecasting Benchmarks -This directory contains benchmarks of `structured-data-models` on [GIFT-Eval](https://github.com/SalesforceAIResearch/gift-eval) and [fev-bench](https://github.com/autogluon/fev). +This directory contains benchmarks of `structured-data-models` on [GIFT-Eval](https://github.com/SalesforceAIResearch/gift-eval), [fev-bench](https://github.com/autogluon/fev), and [TIME](https://github.com/zqiao11/TIME). All benchmarks share one model adapter (see `benchmark/forecasting/model.py`) that maps each forecasting task onto the `ICLModel` interface: time steps become rows, past targets become `y_context`, all covariates and timestamps of the context become `x_context`, and the forecast horizon becomes `x_query` with one row of known covariates and timestamps per future time step. Columns keep their semantic types, so each model decides which ones it consumes. Series in a batch are left-padded with missing values, and multivariate series are flattened for models without multi-target support. @@ -81,3 +81,43 @@ python -m benchmark.forecasting.fev_bench.evaluate This compares all available SDM results against published fev-bench results on the same tasks with `fev.leaderboard`, using a seasonal naive baseline. On fev-bench, `TimesFM3` also imputes missing values with the last observed value, converts dynamic covariates to numbers, and ignores static covariates. In addition to the differences listed for GIFT-Eval, published results hash string covariates with a randomly seeded instead of a fixed hash function. + +______________________________________________________________________ + +## TIME + +### Setup + +Run the commands below from the repository root: + +```bash +pip install . gluonts datasets python-dotenv scipy pyyaml + +git clone https://github.com/zqiao11/TIME.git +git -C TIME checkout 3ca5c41d71c76f3c70632c0121142d325350ee32 +pip install --no-deps -e TIME + +hf download Real-TSF/TIME --repo-type dataset --local-dir /path/to/time +export TIME_DATASET=/path/to/time +``` + +TIME is not released on PyPI and is therefore pinned to a commit. It is installed without dependencies since it pins outdated `numpy`, `gluonts` and `datasets` versions. + +### Run + +```bash +python -m benchmark.forecasting.time_bench.main --model timesfm3 +``` + +Pass `--dataset` to run only selected TIME datasets, *e.g.*, `--dataset Job_Claims/W --dataset CPHL/15T`. +Results are written to `benchmark/forecasting/time_bench_out//`, and already completed configurations are skipped. + +### Evaluate + +```bash +python -m benchmark.forecasting.time_bench.evaluate --time-path /path/to/TIME +``` + +This compares all available SDM results against published TIME results on the same configurations with TIME's official leaderboard script, normalized by a seasonal naive baseline. + +The differences to published `TimesFM-3` results listed for GIFT-Eval apply to TIME as well. diff --git a/benchmark/forecasting/time_bench/__init__.py b/benchmark/forecasting/time_bench/__init__.py new file mode 100644 index 000000000..d51c4fe1e --- /dev/null +++ b/benchmark/forecasting/time_bench/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/benchmark/forecasting/time_bench/evaluate.py b/benchmark/forecasting/time_bench/evaluate.py new file mode 100644 index 000000000..f9c3b0406 --- /dev/null +++ b/benchmark/forecasting/time_bench/evaluate.py @@ -0,0 +1,83 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +r"""Evaluate SDM results with TIME's official leaderboard script.""" + +import argparse +import subprocess +import sys +import tempfile +from pathlib import Path + +from huggingface_hub import snapshot_download + +from benchmark.forecasting.model import MODEL_CONFIGS + +# Pinned revision of the published results. The official script adds the +# seasonal naive baseline itself: +REVISION = "1ebbe655f7fad151f5447ef94b2c371d71ba6d7e" +REFERENCES = ("TimesFM-3", "TimesFM-2.5", "Chronos-2") + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--time-path", + type=Path, + required=True, + help="Path to the pinned TIME checkout.", + ) + args = parser.parse_args() + + script = args.time_path / "scripts" / "compute_local_leaderboard.py" + if not script.is_file(): + raise FileNotFoundError(f"TIME checkout not found at {args.time_path}") + + result_root = Path(__file__).parent.parent / "time_bench_out" + runs = {} + for model_config in MODEL_CONFIGS.values(): + if (result_root / model_config.name).is_dir(): + runs[f"[SDM] {model_config.name}"] = result_root / model_config.name + + if not runs: + raise FileNotFoundError(f"No TIME results found under {result_root}") + + # Compare with published results on configurations run by all models: + configs = set.intersection( + *( + {path.parent.relative_to(run) for path in run.glob("**/metrics.npz")} + for run in runs.values() + ) + ) + reference_root = snapshot_download( + repo_id="Real-TSF/TIME-Output", + repo_type="dataset", + revision=REVISION, + allow_patterns=[ + f"results/{name}/**/{file}" + for name in REFERENCES + for file in ("metrics.npz", "config.json") + ], + ) + for name in REFERENCES: + runs[name] = Path(reference_root) / "results" / name + + # The official script reads all models under `output/results/`: + with tempfile.TemporaryDirectory(prefix="sdm-time-") as directory: + results = Path(directory) / "output" / "results" + for name, run in runs.items(): + for config in configs: + if (run / config / "metrics.npz").is_file(): + (results / name / config).parent.mkdir( + parents=True, exist_ok=True + ) + (results / name / config).symlink_to(run / config) + subprocess.run( + [sys.executable, str(script.resolve())], + cwd=directory, + check=True, + ) + + +if __name__ == "__main__": + main() diff --git a/benchmark/forecasting/time_bench/main.py b/benchmark/forecasting/time_bench/main.py new file mode 100644 index 000000000..7efa1c792 --- /dev/null +++ b/benchmark/forecasting/time_bench/main.py @@ -0,0 +1,93 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +r"""Run an SDM forecasting model on the TIME benchmark.""" + +import argparse +from dataclasses import replace +from pathlib import Path + +import numpy as np +import torch +from gluonts.time_feature import get_seasonality +from timebench.evaluation.data import ( + Dataset, + get_dataset_settings, + load_dataset_config, +) +from timebench.evaluation.saver import save_window_predictions +from timebench.evaluation.utils import get_available_terms + +from benchmark.forecasting.gift_eval.models import ( + QUANTILE_LEVELS, + SDMPredictor, +) +from benchmark.forecasting.model import MODEL_CONFIGS, add_model_args + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + add_model_args(parser) + parser.add_argument( + "--dataset", + action="append", + help="Run only the selected TIME dataset; repeat to combine.", + ) + args = parser.parse_args() + + model_config = MODEL_CONFIGS[args.model] + if args.max_context_size is not None: + model_config = replace( + model_config, max_context_size=args.max_context_size + ) + result_dir = ( + Path(__file__).parent.parent / "time_bench_out" / model_config.name + ) + + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + model = model_config.factory(device) + + # Mirrors TIME's official `experiments/*.py` runners: + config = load_dataset_config() + for name in args.dataset or config["datasets"]: + for term in get_available_terms(name, config): + if (result_dir / name / term / "metrics.npz").exists(): + continue + + settings = get_dataset_settings(name, term, config) + dataset = Dataset( + name=name, + term=term, + prediction_length=settings["prediction_length"], + test_length=settings["test_length"], + val_length=settings.get("val_length", 0), + # Flatten multivariate series for models without joint + # multi-target forecasting: + to_univariate=not model.supports_multi_target, + ) + + predictor = SDMPredictor( + model, + config=model_config, + prediction_length=dataset.prediction_length, + batch_size=args.batch_size, + ) + forecasts = predictor.predict(dataset.test_data.input) + # GluonTS stores quantiles as [Q, H, V] (or [Q, H] if univariate): + quantiles = np.stack([f.forecast_array for f in forecasts]) + if quantiles.ndim == 4: + quantiles = quantiles.swapaxes(-1, -2) # [N, Q, V, H] + + save_window_predictions( + dataset=dataset, + fc_quantiles=quantiles, + ds_config=f"{name}/{term}", + output_base_dir=str(result_dir), + seasonality=get_seasonality(dataset.freq), + model_hyperparams=vars(args), + quantile_levels=list(QUANTILE_LEVELS), + ) + + +if __name__ == "__main__": + main() From f54225125babe9c1a07492179c4dbc16b34c10a2 Mon Sep 17 00:00:00 2001 From: Filippo Guerranti Date: Wed, 14 Oct 2026 13:39:25 +0000 Subject: [PATCH 07/14] Fix line length in TIME evaluation script --- benchmark/forecasting/time_bench/evaluate.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/benchmark/forecasting/time_bench/evaluate.py b/benchmark/forecasting/time_bench/evaluate.py index f9c3b0406..26e8eb638 100644 --- a/benchmark/forecasting/time_bench/evaluate.py +++ b/benchmark/forecasting/time_bench/evaluate.py @@ -37,7 +37,9 @@ def main() -> None: runs = {} for model_config in MODEL_CONFIGS.values(): if (result_root / model_config.name).is_dir(): - runs[f"[SDM] {model_config.name}"] = result_root / model_config.name + runs[f"[SDM] {model_config.name}"] = ( + result_root / model_config.name + ) if not runs: raise FileNotFoundError(f"No TIME results found under {result_root}") @@ -45,7 +47,10 @@ def main() -> None: # Compare with published results on configurations run by all models: configs = set.intersection( *( - {path.parent.relative_to(run) for path in run.glob("**/metrics.npz")} + { + path.parent.relative_to(run) + for path in run.glob("**/metrics.npz") + } for run in runs.values() ) ) From 5a935c7d933fec23b2444c958540857d3dd396e5 Mon Sep 17 00:00:00 2001 From: rusty1s Date: Tue, 27 Oct 2026 07:26:19 +0000 Subject: [PATCH 08/14] update --- benchmark/forecasting/fev_bench/__init__.py | 5 - benchmark/forecasting/fev_bench/evaluate.py | 60 ------ benchmark/forecasting/fev_bench/main.py | 87 --------- benchmark/forecasting/fev_bench/models.py | 119 ------------ benchmark/forecasting/gift_eval/__init__.py | 8 - benchmark/forecasting/gift_eval/evaluate.py | 62 ------ benchmark/forecasting/gift_eval/main.py | 172 ---------------- benchmark/forecasting/gift_eval/models.py | 85 -------- benchmark/forecasting/model.py | 183 ------------------ benchmark/forecasting/time_bench/__init__.py | 2 - .../time_bench}/README.md | 0 .../time_bench}/__init__.py | 0 .../time_bench/evaluate.py | 0 .../time_bench/main.py | 0 docs/source/icl.md | 1 - sdm/evaluation/__init__.py | 2 - sdm/evaluation/regression.py | 61 ------ test/evaluation/test_regression.py | 26 --- 18 files changed, 873 deletions(-) delete mode 100644 benchmark/forecasting/fev_bench/__init__.py delete mode 100644 benchmark/forecasting/fev_bench/evaluate.py delete mode 100644 benchmark/forecasting/fev_bench/main.py delete mode 100644 benchmark/forecasting/fev_bench/models.py delete mode 100644 benchmark/forecasting/gift_eval/__init__.py delete mode 100644 benchmark/forecasting/gift_eval/evaluate.py delete mode 100644 benchmark/forecasting/gift_eval/main.py delete mode 100644 benchmark/forecasting/gift_eval/models.py delete mode 100644 benchmark/forecasting/model.py delete mode 100644 benchmark/forecasting/time_bench/__init__.py rename benchmark/{forecasting => timeseries/time_bench}/README.md (100%) rename benchmark/{forecasting => timeseries/time_bench}/__init__.py (100%) rename benchmark/{forecasting => timeseries}/time_bench/evaluate.py (100%) rename benchmark/{forecasting => timeseries}/time_bench/main.py (100%) delete mode 100644 sdm/evaluation/regression.py delete mode 100644 test/evaluation/test_regression.py diff --git a/benchmark/forecasting/fev_bench/__init__.py b/benchmark/forecasting/fev_bench/__init__.py deleted file mode 100644 index 6fdaa7520..000000000 --- a/benchmark/forecasting/fev_bench/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# Task definitions and reference results of the installed fev release: -FEV_URL = "https://raw.githubusercontent.com/autogluon/fev/v0.10.0" diff --git a/benchmark/forecasting/fev_bench/evaluate.py b/benchmark/forecasting/fev_bench/evaluate.py deleted file mode 100644 index 3b9c278f4..000000000 --- a/benchmark/forecasting/fev_bench/evaluate.py +++ /dev/null @@ -1,60 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -r"""Evaluate SDM forecasting model results on fev-bench.""" - -from pathlib import Path - -import fev -import pandas as pd - -from benchmark.forecasting.fev_bench import FEV_URL -from benchmark.forecasting.model import MODEL_CONFIGS - -BASELINE = "Seasonal Naive" -REFERENCES = ("timesfm-3", "timesfm-2_5", "chronos-2", "seasonal_naive") -METRICS = ("SQL", "MASE") - - -def main() -> None: - benchmark_dir = Path(__file__).parent.parent - result_root = benchmark_dir / "fev_bench_out" - output_root = benchmark_dir / "evals" / "fev_bench" - - summaries = [] - for model_config in MODEL_CONFIGS.values(): - path = result_root / model_config.name / "results.csv" - if path.exists(): - summaries.append(pd.read_csv(path)) - - if not summaries: - raise FileNotFoundError( - f"No fev-bench results found under {result_root}" - ) - - # Compare with published results on tasks run by all models: - tasks = set.intersection(*(set(df["task_name"]) for df in summaries)) - for name in REFERENCES: - path = f"{FEV_URL}/benchmarks/fev_bench/results/{name}.csv" - summaries.append(pd.read_csv(path)) - summaries = pd.concat(summaries) - summaries = summaries[summaries["task_name"].isin(tasks)] - - output_root.mkdir(parents=True, exist_ok=True) - for metric in METRICS: - leaderboard = fev.leaderboard( - summaries, - metric_column=metric, - baseline_model=BASELINE, - ) - leaderboard.to_csv(output_root / f"leaderboard_{metric}.csv") - print(f"\nfev-bench leaderboard ({metric}, {len(tasks)} tasks):") - print( - leaderboard[["skill_score", "win_rate"]].to_string( - float_format="%.4f" - ) - ) - - -if __name__ == "__main__": - main() diff --git a/benchmark/forecasting/fev_bench/main.py b/benchmark/forecasting/fev_bench/main.py deleted file mode 100644 index d545e1f60..000000000 --- a/benchmark/forecasting/fev_bench/main.py +++ /dev/null @@ -1,87 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -r"""Run an SDM forecasting model on fev-bench.""" - -import argparse -import json -from dataclasses import replace -from pathlib import Path - -import fev -import pandas as pd -import torch - -from benchmark.forecasting.fev_bench import FEV_URL -from benchmark.forecasting.fev_bench.models import MODELS -from benchmark.forecasting.model import MODEL_CONFIGS, add_model_args - - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - add_model_args(parser) - parser.add_argument( - "--task", - action="append", - help="Run only the selected fev-bench task; repeat to combine.", - ) - args = parser.parse_args() - - model_config = MODEL_CONFIGS[args.model] - if args.max_context_size is not None: - model_config = replace( - model_config, max_context_size=args.max_context_size - ) - result_path = ( - Path(__file__).parent.parent - / "fev_bench_out" - / model_config.name - / "results.csv" - ) - result_path.parent.mkdir(parents=True, exist_ok=True) - completed = ( - set(pd.read_csv(result_path)["task_name"]) - if result_path.exists() - else set() - ) - - device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - model = MODELS[args.model]( - model_config.factory(device), - config=model_config, - batch_size=args.batch_size, - ) - - # Mirrors fev's official `models/evaluate.py` runner: - benchmark = fev.Benchmark.from_yaml( - f"{FEV_URL}/benchmarks/fev_bench/tasks.yaml" - ) - for task in benchmark.tasks: - if args.task is not None and task.task_name not in args.task: - continue - if task.task_name in completed: - continue - - predictions = model.fit_predict(task) - summary = task.evaluation_summary( - predictions, - model_name=f"[SDM] {model_config.name}", - training_time_s=model.training_time, - inference_time_s=model.inference_time, - trained_on_this_dataset=( - task.dataset_config in model.trained_on_datasets - ), - extra_info={"model_kwargs": json.dumps(vars(args))}, - ) - row = pd.DataFrame([summary]) - row.to_csv( - result_path, - mode="a", - header=not result_path.exists(), - index=False, - ) - print(row[["task_name", "test_error"]].to_string(index=False)) - - -if __name__ == "__main__": - main() diff --git a/benchmark/forecasting/fev_bench/models.py b/benchmark/forecasting/fev_bench/models.py deleted file mode 100644 index c8854ef59..000000000 --- a/benchmark/forecasting/fev_bench/models.py +++ /dev/null @@ -1,119 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""SDM forecasting adapter for fev-bench.""" - -import zlib -from collections.abc import Iterator - -import datasets -import fev -import numpy as np -import pandas as pd - -import sdm -from benchmark.forecasting.model import ModelConfig, forecast, select_levels - - -class SDMModel(fev.ForecastingModel): - """fev model forecasting batches of series with an SDM model.""" - - def __init__( - self, - model: sdm.models.ICLModel, - config: ModelConfig, - batch_size: int, - ) -> None: - super().__init__() - self.model = model - self.config = config - self.batch_size = batch_size - - def _fit_predict(self, task: fev.Task) -> list[datasets.DatasetDict]: - predictions = [] - for window in task.iter_windows(): - past, future = window.get_input_data() - - values = [] - with self._record_inference_time(): - for past_batch, future_batch in zip( - _to_frames(past, self.batch_size), - _to_frames(future, self.batch_size), - strict=True, - ): - out, levels = forecast( - self.model, - config=self.config, - past=[self._prepare(task, df) for df in past_batch], - future=[ - self._prepare(task, df) for df in future_batch - ], - targets=task.target_columns, - ) - # Median point forecast followed by the task quantiles: - wanted = [0.5, *task.quantile_levels] - values.append(select_levels(out, levels, wanted)) - values = np.concatenate(values) # [N, H, V, 1 + Q] - - window_predictions = datasets.DatasetDict() - for i, column in enumerate(task.target_columns): - window_predictions[column] = datasets.Dataset.from_dict( - { - "predictions": values[:, :, i, 0], - **{ - str(level): values[:, :, i, j + 1] - for j, level in enumerate(task.quantile_levels) - }, - } - ) - predictions.append(window_predictions) - return predictions - - def _prepare(self, task: fev.Task, df: pd.DataFrame) -> pd.DataFrame: - """Prepare the past or future data of a series for the model.""" - return df - - -class TimesFM3Model(SDMModel): - """fev model for ``TimesFM3``. - - Static covariates are ignored, while targets and dynamic covariates are - converted to numbers (hashing strings into 100 buckets) and imputed with - the last observed value. - """ - - model_name = "sdm-timesfm3" - - def _prepare(self, task: fev.Task, df: pd.DataFrame) -> pd.DataFrame: - df = df.drop(columns=task.static_columns) - for column in df.columns.drop([task.id_column, task.timestamp_column]): - x = df[column] - if pd.api.types.is_string_dtype(x): - x = x.fillna("").map(lambda v: zlib.crc32(v.encode()) % 100) - df[column] = x.astype("float32").ffill().bfill().fillna(0) - return df - - -MODELS: dict[str, type[SDMModel]] = { - "timesfm3": TimesFM3Model, -} - - -def _to_frames( - dataset: datasets.Dataset, - batch_size: int, -) -> Iterator[list[pd.DataFrame]]: - # One frame per series, with static values repeated over time steps and - # the column types declared by the benchmark: - dtypes = { - name: pd.ArrowDtype(getattr(feature, "feature", feature).pa_type) - for name, feature in dataset.features.items() - } - for batch in dataset.with_format("numpy").iter(batch_size): - num_series = len(next(iter(batch.values()))) - yield [ - pd.DataFrame({name: x[i] for name, x in batch.items()}).astype( - dtypes - ) - for i in range(num_series) - ] diff --git a/benchmark/forecasting/gift_eval/__init__.py b/benchmark/forecasting/gift_eval/__init__.py deleted file mode 100644 index 417150a40..000000000 --- a/benchmark/forecasting/gift_eval/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# Pinned GIFT-Eval revision for dataset properties and reference results: -GIFT_EVAL_URL = ( - "https://raw.githubusercontent.com/SalesforceAIResearch/gift-eval/" - "577a65cfa3acb812405c00d989230bf59f7f8a3d" -) diff --git a/benchmark/forecasting/gift_eval/evaluate.py b/benchmark/forecasting/gift_eval/evaluate.py deleted file mode 100644 index d6176985d..000000000 --- a/benchmark/forecasting/gift_eval/evaluate.py +++ /dev/null @@ -1,62 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -r"""Evaluate SDM forecasting model results on GIFT-Eval.""" - -from pathlib import Path - -import numpy as np -import pandas as pd - -from benchmark.forecasting.gift_eval import GIFT_EVAL_URL -from benchmark.forecasting.model import MODEL_CONFIGS - -BASELINE = "Seasonal_Naive" -REFERENCES = ("TimesFM-3", "TimesFM-2.5", "Chronos-2", BASELINE) -METRICS = { - "eval_metrics/MASE[0.5]": "MASE", - "eval_metrics/mean_weighted_sum_quantile_loss": "CRPS", -} - - -def main() -> None: - benchmark_dir = Path(__file__).parent.parent - result_root = benchmark_dir / "gift_eval_out" - output_root = benchmark_dir / "evals" / "gift_eval" - - results = [] - for model_config in MODEL_CONFIGS.values(): - path = result_root / model_config.name / "all_results.csv" - if path.exists(): - results.append(pd.read_csv(path)) - - if not results: - raise FileNotFoundError( - f"No GIFT-Eval results found under {result_root}" - ) - - # Compare with published results on configurations run by all models: - configs = set.intersection(*(set(result["dataset"]) for result in results)) - for name in REFERENCES: - path = f"{GIFT_EVAL_URL}/results/{name}/all_results.csv" - results.append(pd.read_csv(path)) - - df = pd.concat(results) - df = df[df["dataset"].isin(configs)].set_index(["model", "dataset"]) - df = df[list(METRICS)].rename(columns=METRICS) - - # Normalize by the seasonal naive baseline and aggregate by geometric mean: - log_scores = np.log(df.div(df.loc[BASELINE], level="dataset")) - terms = log_scores.index.get_level_values("dataset").str.split("/").str[-1] - - output_root.mkdir(parents=True, exist_ok=True) - for term, scores in [("overall", log_scores), *log_scores.groupby(terms)]: - leaderboard = np.exp(scores.groupby("model").mean()) - leaderboard = leaderboard.sort_values("CRPS") - leaderboard.to_csv(output_root / f"leaderboard_{term}.csv") - print(f"\nGIFT-Eval leaderboard ({term}):") - print(leaderboard.to_string(float_format="%.4f")) - - -if __name__ == "__main__": - main() diff --git a/benchmark/forecasting/gift_eval/main.py b/benchmark/forecasting/gift_eval/main.py deleted file mode 100644 index 513815328..000000000 --- a/benchmark/forecasting/gift_eval/main.py +++ /dev/null @@ -1,172 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -r"""Run an SDM forecasting model on GIFT-Eval.""" - -import argparse -from dataclasses import replace -from pathlib import Path - -import pandas as pd -import torch -from gift_eval.data import Dataset -from gluonts.ev.metrics import ( - MAE, - MAPE, - MASE, - MSE, - MSIS, - ND, - NRMSE, - RMSE, - SMAPE, - MeanWeightedSumQuantileLoss, -) -from gluonts.model import evaluate_model -from gluonts.time_feature import get_seasonality - -from benchmark.forecasting.gift_eval import GIFT_EVAL_URL -from benchmark.forecasting.gift_eval.models import ( - QUANTILE_LEVELS, - SDMPredictor, -) -from benchmark.forecasting.model import MODEL_CONFIGS, add_model_args - -# Datasets and terms from GIFT-Eval's notebooks: -SHORT_DATASETS = ( # noqa: SIM905 - "m4_yearly m4_quarterly m4_monthly m4_weekly m4_daily m4_hourly " - "electricity/15T electricity/H electricity/D electricity/W " - "solar/10T solar/H solar/D solar/W hospital covid_deaths " - "us_births/D us_births/M us_births/W " - "saugeenday/D saugeenday/M saugeenday/W " - "temperature_rain_with_missing " - "kdd_cup_2018_with_missing/H kdd_cup_2018_with_missing/D " - "car_parts_with_missing restaurant " - "hierarchical_sales/D hierarchical_sales/W " - "LOOP_SEATTLE/5T LOOP_SEATTLE/H LOOP_SEATTLE/D SZ_TAXI/15T SZ_TAXI/H " - "M_DENSE/H M_DENSE/D ett1/15T ett1/H ett1/D ett1/W " - "ett2/15T ett2/H ett2/D ett2/W jena_weather/10T jena_weather/H " - "jena_weather/D bitbrains_fast_storage/5T bitbrains_fast_storage/H " - "bitbrains_rnd/5T bitbrains_rnd/H bizitobs_application bizitobs_service " - "bizitobs_l2c/5T bizitobs_l2c/H" -).split() -MED_LONG_DATASETS = ( # noqa: SIM905 - "electricity/15T electricity/H solar/10T solar/H " - "kdd_cup_2018_with_missing/H LOOP_SEATTLE/5T LOOP_SEATTLE/H SZ_TAXI/15T " - "M_DENSE/H ett1/15T ett1/H ett2/15T ett2/H jena_weather/10T " - "jena_weather/H bitbrains_fast_storage/5T bitbrains_rnd/5T " - "bizitobs_application bizitobs_service bizitobs_l2c/5T bizitobs_l2c/H" -).split() -PRETTY_NAMES = { - "saugeenday": "saugeen", - "temperature_rain_with_missing": "temperature_rain", - "kdd_cup_2018_with_missing": "kdd_cup_2018", - "car_parts_with_missing": "car_parts", -} -METRICS = [ - MSE(forecast_type="mean"), - MSE(forecast_type="0.5"), - MAE(), - MASE(), - MAPE(), - SMAPE(), - MSIS(), - RMSE(), - NRMSE(), - ND(), - MeanWeightedSumQuantileLoss(quantile_levels=QUANTILE_LEVELS), -] - - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - add_model_args(parser) - parser.add_argument( - "--dataset", - action="append", - help="Run only the selected GIFT-Eval dataset; repeat to combine.", - ) - args = parser.parse_args() - - model_config = MODEL_CONFIGS[args.model] - if args.max_context_size is not None: - model_config = replace( - model_config, max_context_size=args.max_context_size - ) - result_path = ( - Path(__file__).parent.parent - / "gift_eval_out" - / model_config.name - / "all_results.csv" - ) - result_path.parent.mkdir(parents=True, exist_ok=True) - completed = ( - set(pd.read_csv(result_path)["dataset"]) - if result_path.exists() - else set() - ) - properties = pd.read_json( - f"{GIFT_EVAL_URL}/notebooks/dataset_properties.json" - ) - - device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - model = model_config.factory(device) - - for name in args.dataset or SHORT_DATASETS: - key, _, freq = name.partition("/") - key = PRETTY_NAMES.get(key.lower(), key.lower()) - freq = freq or properties[key]["frequency"] - for term in ("short", "medium", "long"): - if term != "short" and name not in MED_LONG_DATASETS: - continue - config = f"{key}/{freq}/{term}" - if config in completed: - continue - - # Flatten multivariate series for models without joint multi-target - # forecasting: - dataset = Dataset( - name=name, - term=term, - to_univariate=( - not model.supports_multi_target - and properties[key]["num_variates"] > 1 - ), - ) - predictor = SDMPredictor( - model, - config=model_config, - prediction_length=dataset.prediction_length, - batch_size=args.batch_size, - ) - result = evaluate_model( - predictor, - test_data=dataset.test_data, - metrics=METRICS, - seasonality=get_seasonality(dataset.freq), - ) - row = pd.DataFrame( - [ - { - "dataset": config, - "model": f"[SDM] {model_config.name}", - **{ - f"eval_metrics/{k}": v - for k, v in result.iloc[0].items() - }, - "domain": properties[key]["domain"], - "num_variates": properties[key]["num_variates"], - } - ] - ) - row.to_csv( - result_path, - mode="a", - header=not result_path.exists(), - index=False, - ) - print(row.to_string(index=False)) - - -if __name__ == "__main__": - main() diff --git a/benchmark/forecasting/gift_eval/models.py b/benchmark/forecasting/gift_eval/models.py deleted file mode 100644 index c14b7f254..000000000 --- a/benchmark/forecasting/gift_eval/models.py +++ /dev/null @@ -1,85 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""SDM forecasting adapter for GIFT-Eval.""" - -from collections.abc import Iterator - -import numpy as np -import pandas as pd -from gluonts.dataset import Dataset -from gluonts.itertools import batcher -from gluonts.model.forecast import QuantileForecast -from gluonts.model.predictor import Predictor - -import sdm -from benchmark.forecasting.model import ModelConfig, forecast, select_levels - -QUANTILE_LEVELS = (0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9) - - -class SDMPredictor(Predictor): - """GluonTS predictor forecasting batches of series with an SDM model.""" - - def __init__( - self, - model: sdm.models.ICLModel, - config: ModelConfig, - prediction_length: int, - batch_size: int, - ) -> None: - super().__init__(prediction_length=prediction_length) - self.model = model - self.config = config - self.batch_size = batch_size - - def predict( - self, - dataset: Dataset, - **kwargs: object, - ) -> Iterator[QuantileForecast]: - for batch in batcher(dataset, batch_size=self.batch_size): - # GluonTS stores series as [V, T] (or [T] if univariate): - num_variates = np.atleast_2d(batch[0]["target"]).shape[0] - targets = [f"target_{i}" for i in range(num_variates)] - past, future = [], [] - for entry in batch: - target = np.atleast_2d(entry["target"]) - length = target.shape[-1] - timestamp = pd.period_range( - entry["start"], - periods=length + self.prediction_length, - ).to_timestamp() - columns = { - "item_id": entry["item_id"], - "timestamp": timestamp[:length], - } - columns |= dict(zip(targets, target, strict=True)) - for i, x in enumerate(entry.get("past_feat_dynamic_real", [])): - columns[f"past_feat_dynamic_real_{i}"] = x - past.append(pd.DataFrame(columns)) - future.append( - pd.DataFrame( - { - "item_id": entry["item_id"], - "timestamp": timestamp[length:], - } - ) - ) - values, levels = forecast( - self.model, - config=self.config, - past=past, - future=future, - targets=targets, - ) - values = select_levels(values, levels, QUANTILE_LEVELS) - for entry, value in zip(batch, values, strict=True): - value = value.transpose(2, 0, 1) # [Q, H, V] - if entry["target"].ndim == 1: - value = value[..., 0] - yield QuantileForecast( - forecast_arrays=value, - forecast_keys=[str(q) for q in QUANTILE_LEVELS], - start_date=entry["start"] + entry["target"].shape[-1], - ) diff --git a/benchmark/forecasting/model.py b/benchmark/forecasting/model.py deleted file mode 100644 index e05fc902c..000000000 --- a/benchmark/forecasting/model.py +++ /dev/null @@ -1,183 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""SDM model adapter shared by the time-series forecasting benchmarks.""" - -import argparse -from collections.abc import Callable, Collection, Sequence -from dataclasses import dataclass - -import numpy as np -import pandas as pd -import torch - -import sdm -from sdm.models.timesfm3 import TimesFM3 - - -@dataclass(frozen=True) -class ModelConfig: - name: str - factory: Callable[[torch.device], sdm.models.ICLModel] - num_estimators: int = 1 - max_context_size: int | None = None - max_variates: int | None = None - - -MODEL_CONFIGS = { - "timesfm3": ModelConfig( - name="TimesFM3", - factory=lambda device: TimesFM3(accept_license=True, device=device), - num_estimators=2, - max_context_size=15360, - max_variates=32, - ), -} - - -def add_model_args(parser: argparse.ArgumentParser) -> None: - """Add the model flags shared by all forecasting benchmarks.""" - parser.add_argument( - "--model", - choices=tuple(MODEL_CONFIGS), - default="timesfm3", - help="SDM model to benchmark.", - ) - parser.add_argument( - "--batch_size", - type=int, - default=64, - help="Number of series forecast per model call.", - ) - parser.add_argument( - "--max_context_size", - type=int, - help="Truncate each context to its most recent time steps " - "(defaults to the model's maximum context size).", - ) - - -def forecast( - model: sdm.models.ICLModel, - config: ModelConfig, - past: Sequence[pd.DataFrame], # B x [T_i, V + C] - future: Sequence[pd.DataFrame], # B x [H, K] - targets: Sequence[str], -) -> tuple[np.ndarray, np.ndarray]: # [B, H, V, Q], [Q] - """Forecast the quantiles of a batch of (multivariate) time series. - - Time steps map to rows: ``past`` holds the targets and all covariates of - the context, while ``future`` holds the covariates known over the - forecast horizon. Columns keep their semantic types, so that the model - decides which ones to consume. Missing values are passed through. - """ - device = next(model.parameters()).device - num_series, horizon = len(past), len(future[0]) - size = max(len(df) for df in past) - if config.max_context_size is not None: - size = min(size, config.max_context_size) - - # Left-pad shorter series with missing values so that all series end at - # the forecast start: - past = [df.iloc[-size:].reset_index(drop=True) for df in past] - past = [df.reindex(range(len(df) - size, len(df))) for df in past] - - # Infer semantic types jointly so that context and query share them, and - # order targets first so that forecasts follow ``targets``: - df = pd.concat([*past, *future], ignore_index=True) - df = df[[*targets, *df.columns.drop(targets)]] - table = sdm.TableTensor.from_pandas( - df=df, - stypes=sdm.infer_stypes(df, id="infer"), - device=device, - ) - # Split rows into the context [B, T, V + C] and the query [B, H, V + C]: - context = table[: num_series * size].view(num_series, size, -1) - query = table[num_series * size :].view(num_series, horizon, -1) - assert isinstance(context, sdm.TableTensor) - assert isinstance(query, sdm.TableTensor) - - x_context = context.drop_columns(targets) - x_query = query.select_columns(future[0].columns) - y = context.select_columns(targets).numerical # [B, T, V] - - chunks = [list(range(len(targets)))] - if config.max_variates is not None: - x_context, x_query, chunks = _split_variates( - x_context, - x_query, - num_targets=len(targets), - max_variates=config.max_variates, - stypes=model.supported_feature_stypes, - ) - - values = [] - for index in chunks: - out = model( - x_context=x_context, - y_context=sdm.TableTensor.from_tensor(y[..., index]), - x_query=x_query, - num_estimators=config.num_estimators, - ) - chunk, levels = sdm.evaluation.to_quantiles(out) # [B, H, V', Q] - values.append(chunk) - # Only the last chunk can end with filler targets: - values = torch.cat(values, dim=-2)[..., : len(targets), :] # [B, H, V, Q] - return values.cpu().numpy(), levels.cpu().numpy() - - -def _split_variates( - x_context: sdm.TableTensor, - x_query: sdm.TableTensor, - num_targets: int, - max_variates: int, - stypes: Collection[sdm.Stype], - seed: int = 42, -) -> tuple[sdm.TableTensor, sdm.TableTensor, list[list[int]]]: - """Split targets and covariates into passes of at most ``max_variates``. - - Covariates of the given semantic types count as variates. If there are - too many, covariates are randomly subsampled (future-known ones first) to - leave room for at least one target. Targets are then split into chunks of - equal size, padding the last chunk with leading targets. Returns the - remaining covariates and the target indices of each chunk. - """ - covariates = [ - c for c, stype in x_context.stypes.items() if stype in stypes - ] - known = [c for c in covariates if c in x_query.column_names] - past = [c for c in covariates if c not in x_query.column_names] - - rng = np.random.default_rng(seed) - keep: list[str] = [] - for columns in (known, past): - size = min(len(columns), max_variates - 1 - len(keep)) - if size < len(columns): - index = np.sort(rng.choice(len(columns), size, replace=False)) - columns = [columns[i] for i in index] - keep += columns - drop = set(covariates).difference(keep) - x_context = x_context.drop_columns(drop) - x_query = x_query.drop_columns(drop & x_query.column_names) - - size = min(max_variates - len(keep), num_targets) - chunks = [ - [i % num_targets for i in range(start, start + size)] - for start in range(0, num_targets, size) - ] - return x_context, x_query, chunks - - -def select_levels( - values: np.ndarray, # [..., Q] - levels: np.ndarray, # [Q] - wanted: Sequence[float], -) -> np.ndarray: # [..., len(wanted)] - """Select the quantile levels requested by a benchmark.""" - index = [int(np.abs(levels - level).argmin()) for level in wanted] - if not np.allclose(levels[index], wanted): - raise ValueError( - f"Model does not predict quantile levels {list(wanted)} " - f"(got {levels.tolist()})" - ) - return values[..., index] diff --git a/benchmark/forecasting/time_bench/__init__.py b/benchmark/forecasting/time_bench/__init__.py deleted file mode 100644 index d51c4fe1e..000000000 --- a/benchmark/forecasting/time_bench/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 diff --git a/benchmark/forecasting/README.md b/benchmark/timeseries/time_bench/README.md similarity index 100% rename from benchmark/forecasting/README.md rename to benchmark/timeseries/time_bench/README.md diff --git a/benchmark/forecasting/__init__.py b/benchmark/timeseries/time_bench/__init__.py similarity index 100% rename from benchmark/forecasting/__init__.py rename to benchmark/timeseries/time_bench/__init__.py diff --git a/benchmark/forecasting/time_bench/evaluate.py b/benchmark/timeseries/time_bench/evaluate.py similarity index 100% rename from benchmark/forecasting/time_bench/evaluate.py rename to benchmark/timeseries/time_bench/evaluate.py diff --git a/benchmark/forecasting/time_bench/main.py b/benchmark/timeseries/time_bench/main.py similarity index 100% rename from benchmark/forecasting/time_bench/main.py rename to benchmark/timeseries/time_bench/main.py diff --git a/docs/source/icl.md b/docs/source/icl.md index 8a34ba05d..b4a1e1e58 100644 --- a/docs/source/icl.md +++ b/docs/source/icl.md @@ -98,7 +98,6 @@ Predictions are returned as a general {py:class}`~sdm.tensor.TableTensor`, where Since predictions are returned as a {py:class}`~sdm.tensor.TableTensor`, you can zero-copy them to [`pandas`](https://pandas.pydata.org/docs), [`arrow`](https://arrow.apache.org/docs), or [`cudf`](https://docs.rapids.ai/api/cudf) via {py:meth}`~sdm.tensor.TableTensor.to_pandas`, {py:meth}`~sdm.tensor.TableTensor.to_arrow`, and {py:meth}`~sdm.tensor.TableTensor.to_cudf` for further downstream processing. In order to simplify metric calculation (*e.g.*, via [`torchmetrics`](https://lightning.ai/docs/torchmetrics)), we provide helper functions in the [`sdm.evaluation`](api/evaluation) package to convert target columns to class indices and align prediction columns to them (see {py:func}`~sdm.evaluation.to_class_indices` and {py:func}`~sdm.evaluation.to_binary_class`). -Similarly, {py:func}`~sdm.evaluation.to_quantiles` converts quantile predictions of one or more targets into a single tensor together with their quantile levels. ## Ensembling diff --git a/sdm/evaluation/__init__.py b/sdm/evaluation/__init__.py index 08304db08..7d3b78251 100644 --- a/sdm/evaluation/__init__.py +++ b/sdm/evaluation/__init__.py @@ -4,10 +4,8 @@ """Evaluation utilities.""" from sdm.evaluation.classification import to_class_indices, to_binary_class -from sdm.evaluation.regression import to_quantiles __all__ = [ "to_class_indices", "to_binary_class", - "to_quantiles", ] diff --git a/sdm/evaluation/regression.py b/sdm/evaluation/regression.py deleted file mode 100644 index 8ff24bc10..000000000 --- a/sdm/evaluation/regression.py +++ /dev/null @@ -1,61 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -from torch import Tensor - -from sdm import Stype, TableTensor - - -def to_quantiles(pred: TableTensor) -> tuple[Tensor, Tensor]: - r"""Convert quantile predictions to a per-target quantile tensor. - - Numerical columns must be named ``"q"`` for a single target or - ``"__q"`` for multiple targets, where the digits encode the - quantile level as a decimal fraction, *e.g.*, ``"q500"`` and ``"q50"`` both - denote the median. - Columns are expected to be grouped by target, with the same quantile levels - in the same order for every target. - - Args: - pred: Prediction table whose numerical columns contain quantiles. - - Returns: - The quantile predictions with shape ``[..., V, Q]`` for ``V`` targets - and ``Q`` quantile levels, and the quantile levels with shape ``[Q]``. - - >>> import torch - >>> import sdm - >>> from sdm.evaluation import to_quantiles - >>> pred = sdm.TableTensor.from_tensor( - ... torch.tensor([[1.0, 2.0, 3.0, 4.0]]), - ... columns=["q100", "q400", "q700", "q900"], - ... ) - >>> values, levels = to_quantiles(pred) - >>> values - tensor([[[1., 2., 3., 4.]]]) - >>> levels - tensor([0.1000, 0.4000, 0.7000, 0.9000]) - >>> pred = sdm.TableTensor.from_tensor( - ... torch.tensor([[1.0, 2.0, 3.0, 4.0]]), - ... columns=["a__q10", "a__q90", "b__q10", "b__q90"], - ... ) - >>> values, levels = to_quantiles(pred) - >>> values - tensor([[[1., 2.], - [3., 4.]]]) - >>> levels - tensor([0.1000, 0.9000]) - """ - columns = pred.columns[Stype.numerical] - prefixes, _, digits = zip(*(column.rpartition("q") for column in columns)) - targets = dict.fromkeys(prefixes) - digits = digits[: len(columns) // len(targets)] - if columns != tuple(f"{t}q{d}" for t in targets for d in digits): - raise ValueError( - "Expected quantile columns grouped by target with the same " - f"quantile levels (got {list(columns)})" - ) - - levels = [int(d) / 10 ** len(d) for d in digits] - values = pred.numerical.unflatten(-1, (len(targets), -1)) # [..., V, Q] - return values, values.new_tensor(levels) diff --git a/test/evaluation/test_regression.py b/test/evaluation/test_regression.py deleted file mode 100644 index 9ae0941a8..000000000 --- a/test/evaluation/test_regression.py +++ /dev/null @@ -1,26 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -import pytest -import torch - -import sdm - - -def test_to_quantiles() -> None: - pred = sdm.TableTensor.from_tensor( - torch.randn(2, 5, 4), - columns=["b__q10", "b__q90", "a__q10", "a__q90"], - ) - values, levels = sdm.evaluation.to_quantiles(pred) - torch.testing.assert_close(values, pred.numerical.view(2, 5, 2, 2)) - torch.testing.assert_close(levels, torch.tensor([0.1, 0.9])) - - -def test_to_quantiles_invalid() -> None: - pred = sdm.TableTensor.from_tensor( - torch.randn(5, 4), - columns=["a__q10", "b__q10", "a__q90", "b__q90"], - ) - with pytest.raises(ValueError, match="grouped by target"): - sdm.evaluation.to_quantiles(pred) From 2c54f4eba3ebe6b480791b3983a03773a8bea214 Mon Sep 17 00:00:00 2001 From: rusty1s Date: Tue, 27 Oct 2026 07:30:50 +0000 Subject: [PATCH 09/14] update --- benchmark/timeseries/gift_eval/main.py | 2 +- benchmark/timeseries/{gift_eval => }/predictor.py | 2 -- benchmark/timeseries/time_bench/main.py | 9 +++------ 3 files changed, 4 insertions(+), 9 deletions(-) rename benchmark/timeseries/{gift_eval => }/predictor.py (98%) diff --git a/benchmark/timeseries/gift_eval/main.py b/benchmark/timeseries/gift_eval/main.py index cc028a591..3f2fd8d2f 100644 --- a/benchmark/timeseries/gift_eval/main.py +++ b/benchmark/timeseries/gift_eval/main.py @@ -26,8 +26,8 @@ MED_LONG_DATASETS, PRETTY_NAMES, ) -from benchmark.timeseries.gift_eval.predictor import SDMPredictor from benchmark.timeseries.model import MODEL_CONFIGS +from benchmark.timeseries.predictor import SDMPredictor parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( diff --git a/benchmark/timeseries/gift_eval/predictor.py b/benchmark/timeseries/predictor.py similarity index 98% rename from benchmark/timeseries/gift_eval/predictor.py rename to benchmark/timeseries/predictor.py index 7be3448a4..7c774326b 100644 --- a/benchmark/timeseries/gift_eval/predictor.py +++ b/benchmark/timeseries/predictor.py @@ -12,8 +12,6 @@ import sdm from benchmark.timeseries.model import ModelConfig -QUANTILE_LEVELS = (0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9) - class SDMPredictor(Predictor): def __init__( diff --git a/benchmark/timeseries/time_bench/main.py b/benchmark/timeseries/time_bench/main.py index 7efa1c792..e49e44089 100644 --- a/benchmark/timeseries/time_bench/main.py +++ b/benchmark/timeseries/time_bench/main.py @@ -18,11 +18,8 @@ from timebench.evaluation.saver import save_window_predictions from timebench.evaluation.utils import get_available_terms -from benchmark.forecasting.gift_eval.models import ( - QUANTILE_LEVELS, - SDMPredictor, -) from benchmark.forecasting.model import MODEL_CONFIGS, add_model_args +from benchmark.timeseries.predictor import SDMPredictor def main() -> None: @@ -72,7 +69,7 @@ def main() -> None: prediction_length=dataset.prediction_length, batch_size=args.batch_size, ) - forecasts = predictor.predict(dataset.test_data.input) + forecasts = list(predictor.predict(dataset.test_data.input)) # GluonTS stores quantiles as [Q, H, V] (or [Q, H] if univariate): quantiles = np.stack([f.forecast_array for f in forecasts]) if quantiles.ndim == 4: @@ -85,7 +82,7 @@ def main() -> None: output_base_dir=str(result_dir), seasonality=get_seasonality(dataset.freq), model_hyperparams=vars(args), - quantile_levels=list(QUANTILE_LEVELS), + quantile_levels=[float(q) for q in forecasts[0].forecast_keys], ) From e1ee64c4314d36b867bf981dfe1bb7d51f1eb5a5 Mon Sep 17 00:00:00 2001 From: rusty1s Date: Tue, 27 Oct 2026 07:39:27 +0000 Subject: [PATCH 10/14] update --- benchmark/timeseries/time_bench/README.md | 122 ++++------------------ 1 file changed, 21 insertions(+), 101 deletions(-) diff --git a/benchmark/timeseries/time_bench/README.md b/benchmark/timeseries/time_bench/README.md index c529b7512..31e7de356 100644 --- a/benchmark/timeseries/time_bench/README.md +++ b/benchmark/timeseries/time_bench/README.md @@ -1,123 +1,43 @@ -# Time-Series Forecasting Benchmarks +# Time Series Forecasting Foundation Models on TIME -This directory contains benchmarks of `structured-data-models` on [GIFT-Eval](https://github.com/SalesforceAIResearch/gift-eval), [fev-bench](https://github.com/autogluon/fev), and [TIME](https://github.com/zqiao11/TIME). +This benchmark evaluates `structured-data-models` on the [TIME](https://github.com/zqiao11/TIME) benchmark. -All benchmarks share one model adapter (see `benchmark/forecasting/model.py`) that maps each forecasting task onto the `ICLModel` interface: time steps become rows, past targets become `y_context`, all covariates and timestamps of the context become `x_context`, and the forecast horizon becomes `x_query` with one row of known covariates and timestamps per future time step. Columns keep their semantic types, so each model decides which ones it consumes. Series in a batch are left-padded with missing values, and multivariate series are flattened for models without multi-target support. - -______________________________________________________________________ - -> [!NOTE] -> Weights of `TimesFM3` are distributed under the [TimesFM Non-Commercial License v1.0](https://huggingface.co/google/timesfm-3.0-pytorch/blob/main/LICENSE). -> Review the license before running the `TimesFM3` benchmark, which will download its weights noninteractively. - -## GIFT-Eval - -### Setup +## Setup Run the commands below from the repository root: ```bash -pip install . gluonts datasets python-dotenv -pip install --no-deps "git+https://github.com/SalesforceAIResearch/gift-eval.git@577a65cfa3acb812405c00d989230bf59f7f8a3d" - -hf download Salesforce/GiftEval --repo-type dataset --local-dir /path/to/gift_eval -export GIFT_EVAL=/path/to/gift_eval -``` - -GIFT-Eval is not released on PyPI and is therefore pinned to a commit. It is installed without dependencies since it pins outdated `numpy` and `datasets` versions. - -### Run - -```bash -python -m benchmark.forecasting.gift_eval.main --model timesfm3 -``` - -Pass `--dataset` to run only selected GIFT-Eval datasets, *e.g.*, `--dataset hospital --dataset ett1/H`. -Use `--batch_size` to set the number of series per model call and `--max_context_size` to truncate long contexts. -Results are written to `benchmark/forecasting/gift_eval_out//`, and already completed configurations are skipped. - -### Evaluate - -```bash -python -m benchmark.forecasting.gift_eval.evaluate -``` - -This compares all available SDM results against published GIFT-Eval results on the same configurations, normalized by a seasonal naive baseline and aggregated by geometric mean. - -`TimesFM3` follows the published `TimesFM-3` evaluation with symmetric averaging over two estimators, contexts truncated to 15,360 time steps, and at most 32 variates per forward pass. Its results can still differ from the published ones: - -- Published results batch series of different context lengths together, which shifts their positional encodings on datasets with varying context lengths. -- Published results clip forecasts of non-negative series at zero. - -______________________________________________________________________ - -## fev-bench - -### Setup - -Run the command below from the repository root: - -```bash -pip install . fev==0.10.0 -``` - -Tasks are downloaded from Hugging Face on first use. - -### Run - -```bash -python -m benchmark.forecasting.fev_bench.main --model timesfm3 -``` - -Pass `--task` to run only selected fev-bench tasks, *e.g.*, `--task hospital --task proenfo_gfc12`. -Results are written to `benchmark/forecasting/fev_bench_out//`, and already completed tasks are skipped. - -### Evaluate - -```bash -python -m benchmark.forecasting.fev_bench.evaluate +pip install . gluonts datasets python-dotenv scipy pyyaml +pip install --no-deps "git+https://github.com/zqiao11/TIME.git@3ca5c41d71c76f3c70632c0121142d325350ee32" ``` -This compares all available SDM results against published fev-bench results on the same tasks with `fev.leaderboard`, using a seasonal naive baseline. - -On fev-bench, `TimesFM3` also imputes missing values with the last observed value, converts dynamic covariates to numbers, and ignores static covariates. In addition to the differences listed for GIFT-Eval, published results hash string covariates with a randomly seeded instead of a fixed hash function. +## Run -______________________________________________________________________ +- **`TimesFM3`**: -## TIME + ```bash + python -m benchmark.timeseries.gift_eval.main --model timesfm3 + ``` -### Setup + `TimesFM3` follows official evaluation with symmetric averaging over two estimators, contexts truncated to 15,360 time steps and up to 32 variates per forward pass. + Unlike the official `float32` evaluation, we use `bfloat16` mixed precision for improved efficiency with negligible differences in aggregate benchmark accuracy. -Run the commands below from the repository root: +> [!NOTE] +> Weights of `TimesFM3` are distributed under the [TimesFM Non-Commercial License v1.0](https://huggingface.co/google/timesfm-3.0-pytorch/blob/main/LICENSE). +> Review the license before running the `TimesFM3` benchmark, which will download its weights noninteractively. ```bash -pip install . gluonts datasets python-dotenv scipy pyyaml - -git clone https://github.com/zqiao11/TIME.git -git -C TIME checkout 3ca5c41d71c76f3c70632c0121142d325350ee32 -pip install --no-deps -e TIME - -hf download Real-TSF/TIME --repo-type dataset --local-dir /path/to/time -export TIME_DATASET=/path/to/time +python -m benchmark.timeseries.time_bench.main --model timesfm3 ``` -TIME is not released on PyPI and is therefore pinned to a commit. It is installed without dependencies since it pins outdated `numpy`, `gluonts` and `datasets` versions. - -### Run - -```bash -python -m benchmark.forecasting.time_bench.main --model timesfm3 -``` +Results are written to `benchmark/timeseries/time_bench_out//`, and already completed configurations are skipped. Pass `--dataset` to run only selected TIME datasets, *e.g.*, `--dataset Job_Claims/W --dataset CPHL/15T`. -Results are written to `benchmark/forecasting/time_bench_out//`, and already completed configurations are skipped. -### Evaluate +## Evaluate + +Evaluate all available model results with: ```bash -python -m benchmark.forecasting.time_bench.evaluate --time-path /path/to/TIME +python -m benchmark.timeseries.time_bench.evaluate ``` - -This compares all available SDM results against published TIME results on the same configurations with TIME's official leaderboard script, normalized by a seasonal naive baseline. - -The differences to published `TimesFM-3` results listed for GIFT-Eval apply to TIME as well. From 4392e7b96b5ceec46b6f781b96d4b182f75654ef Mon Sep 17 00:00:00 2001 From: rusty1s Date: Tue, 27 Oct 2026 07:40:11 +0000 Subject: [PATCH 11/14] update --- benchmark/timeseries/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/timeseries/README.md b/benchmark/timeseries/README.md index 1f2955fc8..77073eb84 100644 --- a/benchmark/timeseries/README.md +++ b/benchmark/timeseries/README.md @@ -1,3 +1,3 @@ # Time Series Benchmarks -This directory contains time series benchmarks of `structured-data-models` on [GIFT-Eval](gift_eval/), and [fev-bench](fev_bench/). +This directory contains time series benchmarks of `structured-data-models` on [GIFT-Eval](gift_eval/), [fev-bench](fev_bench/), and [TIME](time_bench/). From 41742fb2903408ba7a1e9e1a3cf12a5c5e5c7cb6 Mon Sep 17 00:00:00 2001 From: rusty1s Date: Tue, 27 Oct 2026 08:05:49 +0000 Subject: [PATCH 12/14] update --- benchmark/timeseries/time_bench/evaluate.py | 2 +- benchmark/timeseries/time_bench/main.py | 115 ++++++++++---------- 2 files changed, 61 insertions(+), 56 deletions(-) diff --git a/benchmark/timeseries/time_bench/evaluate.py b/benchmark/timeseries/time_bench/evaluate.py index 26e8eb638..25b411786 100644 --- a/benchmark/timeseries/time_bench/evaluate.py +++ b/benchmark/timeseries/time_bench/evaluate.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -r"""Evaluate SDM results with TIME's official leaderboard script.""" +r"""Evaluate SDM time series forecasting results on TIME.""" import argparse import subprocess diff --git a/benchmark/timeseries/time_bench/main.py b/benchmark/timeseries/time_bench/main.py index e49e44089..bae3eee39 100644 --- a/benchmark/timeseries/time_bench/main.py +++ b/benchmark/timeseries/time_bench/main.py @@ -1,15 +1,18 @@ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -r"""Run an SDM forecasting model on the TIME benchmark.""" +r"""Run an SDM time series forecasting model on TIME.""" import argparse -from dataclasses import replace +import os +from contextlib import redirect_stdout from pathlib import Path import numpy as np import torch +import tqdm from gluonts.time_feature import get_seasonality +from huggingface_hub import snapshot_download from timebench.evaluation.data import ( Dataset, get_dataset_settings, @@ -18,73 +21,75 @@ from timebench.evaluation.saver import save_window_predictions from timebench.evaluation.utils import get_available_terms -from benchmark.forecasting.model import MODEL_CONFIGS, add_model_args +from benchmark.timeseries.model import MODEL_CONFIGS from benchmark.timeseries.predictor import SDMPredictor +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument( + "--model", + choices=tuple(MODEL_CONFIGS), + default="timesfm3", + help="SDM model to benchmark.", +) +parser.add_argument( + "--dataset", + action="append", + help="Run only the selected TIME dataset; repeat to combine.", +) +args = parser.parse_args() -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - add_model_args(parser) - parser.add_argument( - "--dataset", - action="append", - help="Run only the selected TIME dataset; repeat to combine.", - ) - args = parser.parse_args() +config = MODEL_CONFIGS[args.model] +dataset_config = load_dataset_config() +path = Path(__file__).parent.parent / "time_bench_out" / config.name - model_config = MODEL_CONFIGS[args.model] - if args.max_context_size is not None: - model_config = replace( - model_config, max_context_size=args.max_context_size - ) - result_dir = ( - Path(__file__).parent.parent / "time_bench_out" / model_config.name +names = args.dataset or dataset_config["datasets"] +if not os.getenv("TIME_DATASET"): + os.environ["TIME_DATASET"] = snapshot_download( + repo_id="Real-TSF/TIME", + repo_type="dataset", + allow_patterns=[f"{name}/*" for name in names], ) - device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - model = model_config.factory(device) - # Mirrors TIME's official `experiments/*.py` runners: - config = load_dataset_config() - for name in args.dataset or config["datasets"]: - for term in get_available_terms(name, config): - if (result_dir / name / term / "metrics.npz").exists(): - continue +device = torch.device("cuda" if torch.cuda.is_available() else "cpu") +model = config.factory(device) - settings = get_dataset_settings(name, term, config) - dataset = Dataset( - name=name, - term=term, - prediction_length=settings["prediction_length"], - test_length=settings["test_length"], - val_length=settings.get("val_length", 0), - # Flatten multivariate series for models without joint - # multi-target forecasting: - to_univariate=not model.supports_multi_target, - ) +for name in names: + for term in get_available_terms(name, dataset_config): + if (path / name / term / "metrics.npz").exists(): + continue - predictor = SDMPredictor( - model, - config=model_config, - prediction_length=dataset.prediction_length, - batch_size=args.batch_size, + settings = get_dataset_settings(name, term, dataset_config) + dataset = Dataset( + name=name, + term=term, + prediction_length=settings["prediction_length"], + test_length=settings["test_length"], + ) + + predictor = SDMPredictor(model, config, dataset.prediction_length) + outs = list( + tqdm.tqdm( + predictor.predict(dataset.test_data.input), + total=len(dataset.test_data), + desc=f"{name}/{term}", ) - forecasts = list(predictor.predict(dataset.test_data.input)) - # GluonTS stores quantiles as [Q, H, V] (or [Q, H] if univariate): - quantiles = np.stack([f.forecast_array for f in forecasts]) - if quantiles.ndim == 4: - quantiles = quantiles.swapaxes(-1, -2) # [N, Q, V, H] + ) + + quantiles = np.stack([out.forecast_array for out in outs]) + if quantiles.ndim == 4: + quantiles = quantiles.swapaxes(-1, -2) # [B, Q, C, R] + with redirect_stdout(None): save_window_predictions( dataset=dataset, fc_quantiles=quantiles, ds_config=f"{name}/{term}", - output_base_dir=str(result_dir), + output_base_dir=str(path), seasonality=get_seasonality(dataset.freq), - model_hyperparams=vars(args), - quantile_levels=[float(q) for q in forecasts[0].forecast_keys], + quantile_levels=[float(q) for q in outs[0].forecast_keys], ) - - -if __name__ == "__main__": - main() + with np.load(path / name / term / "metrics.npz") as metrics: + mase = np.nanmean(metrics["MASE"]) + crps = np.nanmean(metrics["CRPS"]) + print(f"{name}/{term}: MASE={mase:.4f}, CRPS={crps:.4f}") From 2025fa81aab5a2a0d8cd2dd7964a7943cbf445d0 Mon Sep 17 00:00:00 2001 From: rusty1s Date: Tue, 27 Oct 2026 08:07:19 +0000 Subject: [PATCH 13/14] update --- benchmark/timeseries/time_bench/main.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/benchmark/timeseries/time_bench/main.py b/benchmark/timeseries/time_bench/main.py index bae3eee39..6a80b776d 100644 --- a/benchmark/timeseries/time_bench/main.py +++ b/benchmark/timeseries/time_bench/main.py @@ -5,6 +5,7 @@ import argparse import os +import warnings from contextlib import redirect_stdout from pathlib import Path @@ -38,6 +39,8 @@ ) args = parser.parse_args() +warnings.filterwarnings("ignore", message=r"'.*' is deprecated.*") + config = MODEL_CONFIGS[args.model] dataset_config = load_dataset_config() path = Path(__file__).parent.parent / "time_bench_out" / config.name From 23b0a00929991841762ee47984238303433fb7bc Mon Sep 17 00:00:00 2001 From: rusty1s Date: Tue, 27 Oct 2026 08:34:54 +0000 Subject: [PATCH 14/14] update --- benchmark/timeseries/time_bench/README.md | 6 +- benchmark/timeseries/time_bench/evaluate.py | 122 ++++++++------------ 2 files changed, 52 insertions(+), 76 deletions(-) diff --git a/benchmark/timeseries/time_bench/README.md b/benchmark/timeseries/time_bench/README.md index 31e7de356..52bf2a125 100644 --- a/benchmark/timeseries/time_bench/README.md +++ b/benchmark/timeseries/time_bench/README.md @@ -16,7 +16,7 @@ pip install --no-deps "git+https://github.com/zqiao11/TIME.git@3ca5c41d71c76f3c7 - **`TimesFM3`**: ```bash - python -m benchmark.timeseries.gift_eval.main --model timesfm3 + python -m benchmark.timeseries.time_bench.main --model timesfm3 ``` `TimesFM3` follows official evaluation with symmetric averaging over two estimators, contexts truncated to 15,360 time steps and up to 32 variates per forward pass. @@ -26,10 +26,6 @@ pip install --no-deps "git+https://github.com/zqiao11/TIME.git@3ca5c41d71c76f3c7 > Weights of `TimesFM3` are distributed under the [TimesFM Non-Commercial License v1.0](https://huggingface.co/google/timesfm-3.0-pytorch/blob/main/LICENSE). > Review the license before running the `TimesFM3` benchmark, which will download its weights noninteractively. -```bash -python -m benchmark.timeseries.time_bench.main --model timesfm3 -``` - Results are written to `benchmark/timeseries/time_bench_out//`, and already completed configurations are skipped. Pass `--dataset` to run only selected TIME datasets, *e.g.*, `--dataset Job_Claims/W --dataset CPHL/15T`. diff --git a/benchmark/timeseries/time_bench/evaluate.py b/benchmark/timeseries/time_bench/evaluate.py index 25b411786..e7943cf59 100644 --- a/benchmark/timeseries/time_bench/evaluate.py +++ b/benchmark/timeseries/time_bench/evaluate.py @@ -3,86 +3,66 @@ r"""Evaluate SDM time series forecasting results on TIME.""" -import argparse -import subprocess -import sys -import tempfile +import gc from pathlib import Path +import numpy as np +import pandas as pd from huggingface_hub import snapshot_download -from benchmark.forecasting.model import MODEL_CONFIGS +from benchmark.timeseries.model import MODEL_CONFIGS -# Pinned revision of the published results. The official script adds the -# seasonal naive baseline itself: -REVISION = "1ebbe655f7fad151f5447ef94b2c371d71ba6d7e" -REFERENCES = ("TimesFM-3", "TimesFM-2.5", "Chronos-2") +BASELINE = "seasonal_naive" +REFERENCES = ("TimesFM-3", "TimesFM-2.5", "Chronos-2", BASELINE) +METRICS = ("MASE", "CRPS") +runs = {} +for config in MODEL_CONFIGS.values(): + path = Path(__file__).parent.parent / "time_bench_out" / config.name + if any(path.glob("**/metrics.npz")): + runs[f"[SDM] {config.name}"] = path -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--time-path", - type=Path, - required=True, - help="Path to the pinned TIME checkout.", +configs = set.intersection( + *( + {path.parent.relative_to(run) for path in run.rglob("metrics.npz")} + for run in runs.values() ) - args = parser.parse_args() +) - script = args.time_path / "scripts" / "compute_local_leaderboard.py" - if not script.is_file(): - raise FileNotFoundError(f"TIME checkout not found at {args.time_path}") +path = snapshot_download( + repo_id="Real-TSF/TIME-Output", + repo_type="dataset", + allow_patterns=[ + f"results/{name}/{config.as_posix()}/metrics.npz" + for name in REFERENCES + for config in sorted(configs) + ], +) +gc.collect() +for name in REFERENCES: + runs[name] = Path(path) / "results" / name - result_root = Path(__file__).parent.parent / "time_bench_out" - runs = {} - for model_config in MODEL_CONFIGS.values(): - if (result_root / model_config.name).is_dir(): - runs[f"[SDM] {model_config.name}"] = ( - result_root / model_config.name +rows = [] +for model, run in runs.items(): + for config in sorted(configs): + with np.load(run / config / "metrics.npz") as metrics: + rows.append( + { + "model": model, + "dataset": config.as_posix(), + **{ + metric: np.nanmean(metrics[metric]) + for metric in METRICS + }, + } ) - if not runs: - raise FileNotFoundError(f"No TIME results found under {result_root}") - - # Compare with published results on configurations run by all models: - configs = set.intersection( - *( - { - path.parent.relative_to(run) - for path in run.glob("**/metrics.npz") - } - for run in runs.values() - ) - ) - reference_root = snapshot_download( - repo_id="Real-TSF/TIME-Output", - repo_type="dataset", - revision=REVISION, - allow_patterns=[ - f"results/{name}/**/{file}" - for name in REFERENCES - for file in ("metrics.npz", "config.json") - ], - ) - for name in REFERENCES: - runs[name] = Path(reference_root) / "results" / name - - # The official script reads all models under `output/results/`: - with tempfile.TemporaryDirectory(prefix="sdm-time-") as directory: - results = Path(directory) / "output" / "results" - for name, run in runs.items(): - for config in configs: - if (run / config / "metrics.npz").is_file(): - (results / name / config).parent.mkdir( - parents=True, exist_ok=True - ) - (results / name / config).symlink_to(run / config) - subprocess.run( - [sys.executable, str(script.resolve())], - cwd=directory, - check=True, - ) - - -if __name__ == "__main__": - main() +df = pd.DataFrame(rows).set_index(["model", "dataset"]) +scores = np.log(df.div(df.loc[BASELINE], level="dataset")) +terms = scores.index.get_level_values("dataset").str.split("/").str[-1] +for i, (group, score) in enumerate([("all", scores), *scores.groupby(terms)]): + if i > 0: + print() + summary = np.exp(score.groupby("model").mean()).sort_values("CRPS") + print(f"TIME leaderboard ({group}):") + print(summary.reset_index().to_string(index=False, float_format="%.4f"))