diff --git a/.gitignore b/.gitignore index 6c21372db..f2c550828 100644 --- a/.gitignore +++ b/.gitignore @@ -223,6 +223,7 @@ datasets.json # Time Series benchmarks gift_eval_out/ +fev_bench_out/ # Checkpoint files *.pt diff --git a/benchmark/README.md b/benchmark/README.md index 66abebf99..1af452860 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -11,3 +11,4 @@ This folder contains benchmark integrations for `structured-data-models`: - [**`relarena/`**](relational/relarena): [RelArena](https://github.com/PriorLabs/relarena) - [**`timeseries/`**](timeseries/): Time series benchmarks - [**`gift_eval/`**](timeseries/gift_eval/): [GIFT-Eval](https://github.com/SalesforceAIResearch/gift-eval) + - [**`fev_bench/`**](timeseries/fev_bench/): [fev-bench](https://github.com/autogluon/fev) diff --git a/benchmark/timeseries/README.md b/benchmark/timeseries/README.md index abd9b02a7..1f2955fc8 100644 --- a/benchmark/timeseries/README.md +++ b/benchmark/timeseries/README.md @@ -1,3 +1,3 @@ # Time Series Benchmarks -This directory contains time series benchmarks of `structured-data-models` on [GIFT-Eval](gift_eval/). +This directory contains time series benchmarks of `structured-data-models` on [GIFT-Eval](gift_eval/), and [fev-bench](fev_bench/). diff --git a/benchmark/timeseries/fev_bench/README.md b/benchmark/timeseries/fev_bench/README.md new file mode 100644 index 000000000..30701cdf4 --- /dev/null +++ b/benchmark/timeseries/fev_bench/README.md @@ -0,0 +1,38 @@ +# Time Series Forecasting Foundation Models on fev-bench + +This benchmark evaluates `structured-data-models` on the [fev-bench](https://github.com/autogluon/fev) benchmark. + +## Setup + +Run the command below from the repository root: + +```bash +pip install . fev==0.10.0 +``` + +## Run + +- **`TimesFM3`**: + + ```bash + python -m benchmark.timeseries.fev_bench.main --model timesfm3 + ``` + + `TimesFM3` follows official evaluation with symmetric averaging over two estimators, contexts truncated to 15,360 time steps and up to 32 variates per forward pass. + +> [!NOTE] +> Weights of `TimesFM3` are distributed under the [TimesFM Non-Commercial License v1.0](https://huggingface.co/google/timesfm-3.0-pytorch/blob/main/LICENSE). +> Review the license before running the `TimesFM3` benchmark, which will download its weights noninteractively. + +Results will be written to `benchmark/timeseries/fev_bench_out//`, and already completed configurations are skipped. + +Pass `--task` to run only selected fev-bench tasks, *e.g.*, `--task hospital --task proenfo_gfc12`. +Pass `--mini` to run the 20-task fev-bench-mini subset: + +## Evaluate + +Evaluate all available model results with: + +```bash +python -m benchmark.timeseries.fev_bench.evaluate +``` diff --git a/benchmark/timeseries/fev_bench/__init__.py b/benchmark/timeseries/fev_bench/__init__.py new file mode 100644 index 000000000..4a93993ce --- /dev/null +++ b/benchmark/timeseries/fev_bench/__init__.py @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +FEV_BENCH_URL = "https://raw.githubusercontent.com/autogluon/fev/v0.10.0" diff --git a/benchmark/timeseries/fev_bench/evaluate.py b/benchmark/timeseries/fev_bench/evaluate.py new file mode 100644 index 000000000..70ed2e6cb --- /dev/null +++ b/benchmark/timeseries/fev_bench/evaluate.py @@ -0,0 +1,60 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +r"""Evaluate SDM time series forecasting results on fev-bench.""" + +from pathlib import Path + +import fev +import pandas as pd + +from benchmark.timeseries.fev_bench import FEV_BENCH_URL +from benchmark.timeseries.model import MODEL_CONFIGS + +BASELINE = "Seasonal Naive" +REFERENCES = ("timesfm-3", "timesfm-2_5", "chronos-2", "seasonal_naive") +METRICS = ("SQL", "MASE") + + +def main() -> None: + benchmark_dir = Path(__file__).parent.parent + result_root = benchmark_dir / "fev_bench_out" + output_root = benchmark_dir / "evals" / "fev_bench" + + summaries = [] + for model_config in MODEL_CONFIGS.values(): + path = result_root / model_config.name / "results.csv" + if path.exists(): + summaries.append(pd.read_csv(path)) + + if not summaries: + raise FileNotFoundError( + f"No fev-bench results found under {result_root}" + ) + + # Compare with published results on tasks run by all models: + tasks = set.intersection(*(set(df["task_name"]) for df in summaries)) + for name in REFERENCES: + path = f"{FEV_BENCH_URL}/benchmarks/fev_bench/results/{name}.csv" + summaries.append(pd.read_csv(path)) + summaries = pd.concat(summaries) + summaries = summaries[summaries["task_name"].isin(tasks)] + + output_root.mkdir(parents=True, exist_ok=True) + for metric in METRICS: + leaderboard = fev.leaderboard( + summaries, + metric_column=metric, + baseline_model=BASELINE, + ) + leaderboard.to_csv(output_root / f"leaderboard_{metric}.csv") + print(f"\nfev-bench leaderboard ({metric}, {len(tasks)} tasks):") + print( + leaderboard[["skill_score", "win_rate"]].to_string( + float_format="%.4f" + ) + ) + + +if __name__ == "__main__": + main() diff --git a/benchmark/timeseries/fev_bench/main.py b/benchmark/timeseries/fev_bench/main.py new file mode 100644 index 000000000..828c016da --- /dev/null +++ b/benchmark/timeseries/fev_bench/main.py @@ -0,0 +1,70 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +r"""Run an SDM time series forecasting model on fev-bench.""" + +import argparse +from pathlib import Path + +import fev +import pandas as pd +import torch + +from benchmark.timeseries.fev_bench import FEV_BENCH_URL +from benchmark.timeseries.fev_bench.predictor import SDMPredictor +from benchmark.timeseries.model import MODEL_CONFIGS + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument( + "--model", + choices=tuple(MODEL_CONFIGS), + default="timesfm3", + help="SDM model to benchmark.", +) +parser.add_argument( + "--mini", + action="store_true", + help="Run the 20-task fev-bench-mini subset.", +) +parser.add_argument( + "--task", + action="append", + help="Run only the selected fev-bench task; repeat to combine.", +) +args = parser.parse_args() + +config = MODEL_CONFIGS[args.model] +path = Path(__file__).parent.parent +path = path / "fev_bench_out" / config.name / "results.csv" +path.parent.mkdir(parents=True, exist_ok=True) + +device = torch.device("cuda" if torch.cuda.is_available() else "cpu") +model = SDMPredictor( + model=config.factory(device), + config=config, +) + +completed = set(pd.read_csv(path)["task_name"]) if path.exists() else set() +filename = "tasks_mini.yaml" if args.mini else "tasks.yaml" +benchmark = fev.Benchmark.from_yaml( + f"{FEV_BENCH_URL}/benchmarks/fev_bench/{filename}" +) +for task in benchmark.tasks: + if args.task is not None and task.task_name not in args.task: + continue + if task.task_name in completed: + continue + + summary = task.evaluation_summary( + model.fit_predict(task), + model_name=f"[SDM] {config.name}", + training_time_s=model.training_time, + inference_time_s=model.inference_time, + ) + print( + f"{task.task_name}: Error={summary['test_error']:6f}, " + f"Time={model.inference_time:.2f}s" + ) + row = pd.DataFrame([summary]) + row.to_csv(path, mode="a", header=not path.exists(), index=False) + completed.add(task.task_name) diff --git a/benchmark/timeseries/fev_bench/predictor.py b/benchmark/timeseries/fev_bench/predictor.py new file mode 100644 index 000000000..bd5a48983 --- /dev/null +++ b/benchmark/timeseries/fev_bench/predictor.py @@ -0,0 +1,190 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from collections.abc import Iterator + +import datasets +import fev +import numpy as np +import torch +import tqdm + +import sdm +from benchmark.timeseries.model import ModelConfig + + +class SDMPredictor(fev.ForecastingModel): + def __init__( + self, + model: sdm.models.ICLModel, + config: ModelConfig, + batch_size: int = 64, + ) -> None: + super().__init__() + self.model = model + self.config = config + self.batch_size = batch_size + self.device = next(model.parameters()).device + + def _fit_predict(self, task: fev.Task) -> list[datasets.DatasetDict]: + predictions = [] + for window in tqdm.tqdm( + task.iter_windows(), + total=task.num_windows, + desc=task.task_name, + ): + past, future = window.get_input_data() + + all_columns = task.known_dynamic_columns.copy() + if self.model.supports_context_only_columns: + all_columns += task.past_dynamic_columns + + values = [] + with self._record_inference_time(): + for ys, xs_context, xs_query in zip( + _from(past, task.target_columns, self.batch_size), + _from(past, all_columns, self.batch_size), + _from(future, task.known_dynamic_columns, self.batch_size), + strict=True, + ): + generator = torch.Generator().manual_seed(42) + + if self.config.max_context_size is not None: + length = self.config.max_context_size + ys = [y[..., -length:] for y in ys] + xs_context = [x[..., -length:] for x in xs_context] + + length = max(y.size(-1) for y in ys) + y = torch.full( + (len(ys), ys[0].size(0), length), + fill_value=torch.nan, + ) + for i, y_i in enumerate(ys): + y[i, :, -y_i.size(-1) :] = y_i + x_context = torch.full( + ( + len(ys), + xs_context[0].size(0) + if len(xs_context) > 0 + else 0, + length, + ), + fill_value=torch.nan, + ) + for i, x_context_i in enumerate(xs_context): + x_context[i, :, -x_context_i.size(-1) :] = x_context_i + x_query = torch.empty( + len(ys), + xs_query[0].size(0) if len(xs_query) > 0 else 0, + task.horizon, + ) + for i, x_query_i in enumerate(xs_query): + x_query[i, :, :] = x_query_i + + if ( # Keep room for at least one target: + self.config.max_variates is not None + and x_query.size(1) >= self.config.max_variates + ): + perm = torch.randperm( + x_query.size(1), generator=generator + )[: self.config.max_variates - 1] + x_context = x_context[:, perm] + x_query = x_query[:, perm] + elif ( + self.config.max_variates is not None + and x_context.size(1) >= self.config.max_variates + ): + perm = torch.randperm( + x_context.size(1) - x_query.size(1), + generator=generator, + )[: self.config.max_variates - 1 - x_query.size(1)] + x_context = torch.cat( + [ + x_context[:, : x_query.size(1)], + x_context[:, perm + x_query.size(1)], + ], + dim=1, + ) + + y = y.transpose(-1, -2).to( + self.device, + memory_format=torch.contiguous_format, + ) + x_context = x_context.transpose(-1, -2).to( + self.device, + memory_format=torch.contiguous_format, + ) + x_query = x_query.transpose(-1, -2).to( + self.device, + memory_format=torch.contiguous_format, + ) + + size = y.size(-1) + if self.config.max_variates is not None: + size = min( + size, + self.config.max_variates - x_context.size(-1), + ) + + preds = [] + for start in range(0, y.size(-1), size): + y_i = y[..., start : start + size] + if y_i.size(-1) < size: + y_i = torch.cat( + [y_i, y[..., : size - y_i.size(-1)]], dim=-1 + ) + + out = self.model( + x_context=sdm.TableTensor.from_tensor(x_context), + y_context=sdm.TableTensor.from_tensor(y_i), + x_query=sdm.TableTensor.from_tensor(x_query), + num_estimators=self.config.num_estimators, + ) + pred = out.numerical + pred = pred.view(y.size(0), task.horizon, size, -1) + preds.append(pred) + + pred = torch.cat(preds, dim=-2)[..., : y.size(-1), :] + pred = pred.cpu().numpy() # [B, R, C, Q] + levels = out.numerical_columns[: pred.shape[1]] + levels = [float(f"0.{q.split('q')[-1]}") for q in levels] + wanted = [0.5, *task.quantile_levels] + index = [levels.index(q) for q in wanted] + values.append(pred[..., index]) + + values = np.concatenate(values) # [B, R, C, 1 + Q] + window_predictions = datasets.DatasetDict() + for i, column in enumerate(task.target_columns): + window_predictions[column] = datasets.Dataset.from_dict( + { + "predictions": values[:, :, i, 0], + **{ + str(level): values[:, :, i, j + 1] + for j, level in enumerate(task.quantile_levels) + }, + } + ) + predictions.append(window_predictions) + + return predictions + + +def _from( + dataset: datasets.Dataset, + columns: list[str], + batch_size: int, +) -> Iterator[list[torch.Tensor]]: + for batch in dataset.with_format("numpy").iter(batch_size): + if not columns: + yield [] + continue + + yield [ + torch.from_numpy( + np.asarray( + [batch[column][i] for column in columns], + dtype=np.float32, + ) + ) + for i in range(len(batch[columns[0]])) + ] diff --git a/benchmark/timeseries/gift_eval/README.md b/benchmark/timeseries/gift_eval/README.md index b0117fad6..ef0711fd9 100644 --- a/benchmark/timeseries/gift_eval/README.md +++ b/benchmark/timeseries/gift_eval/README.md @@ -27,6 +27,8 @@ pip install --no-deps "git+https://github.com/SalesforceAIResearch/gift-eval.git Results will be written to `benchmark/timeseries/gift_eval_out//`, and already completed configurations are skipped. +Pass `--dataset` to run only selected GIFT-Eval datasets, *e.g.*, `--dataset hospital --dataset ett1/H`. + ## Evaluate Evaluate all available model results with: diff --git a/benchmark/timeseries/gift_eval/main.py b/benchmark/timeseries/gift_eval/main.py index 23a14b832..cc028a591 100644 --- a/benchmark/timeseries/gift_eval/main.py +++ b/benchmark/timeseries/gift_eval/main.py @@ -69,9 +69,6 @@ path = path / "gift_eval_out" / config.name / "all_results.csv" path.parent.mkdir(parents=True, exist_ok=True) -completed = set(pd.read_csv(path)["dataset"]) if path.exists() else set() -properties = pd.read_json(f"{GIFT_EVAL_URL}/notebooks/dataset_properties.json") - names = args.dataset or DATASETS if not os.getenv("GIFT_EVAL"): os.environ["GIFT_EVAL"] = snapshot_download( @@ -83,6 +80,8 @@ device = torch.device("cuda" if torch.cuda.is_available() else "cpu") model = config.factory(device) +completed = set(pd.read_csv(path)["dataset"]) if path.exists() else set() +properties = pd.read_json(f"{GIFT_EVAL_URL}/notebooks/dataset_properties.json") for name in names: key, _, freq = name.partition("/") key = PRETTY_NAMES.get(key.lower(), key.lower()) diff --git a/benchmark/timeseries/gift_eval/predictor.py b/benchmark/timeseries/gift_eval/predictor.py index 52d801bc9..3174b5e95 100644 --- a/benchmark/timeseries/gift_eval/predictor.py +++ b/benchmark/timeseries/gift_eval/predictor.py @@ -36,7 +36,7 @@ def predict( ) -> Iterator[QuantileForecast]: for batch in batcher(dataset, batch_size=self.batch_size): - cpu_generator = torch.Generator().manual_seed(42) + generator = torch.Generator().manual_seed(42) ys = [torch.from_numpy(entry["target"]) for entry in batch] ys = [torch.atleast_2d(y) for y in ys] @@ -69,11 +69,11 @@ def predict( for i, x_i in enumerate(xs): x[i, :, -x_i.size(-1) :] = x_i - if ( + if ( # Keep room for at least one target: self.config.max_variates is not None - and x.size(1) > self.config.max_variates - 1 - ): # Keep room for at least one target: - perm = torch.randperm(x.size(1), generator=cpu_generator) + and x.size(1) >= self.config.max_variates + ): + perm = torch.randperm(x.size(1), generator=generator) x = x[:, perm[: self.config.max_variates - 1]] y = y.transpose(-1, -2).to( @@ -96,10 +96,10 @@ def predict( y_i = torch.cat([y_i, y[..., : size - y_i.size(-1)]], -1) out = self.model( - x_context=x, - y_context=y_i, - x_query=torch.empty( - (x.size(0), self.prediction_length, 0), + x_context=sdm.TableTensor.from_tensor(x), + y_context=sdm.TableTensor.from_tensor(y_i), + x_query=sdm.TableTensor( + size=(x.size(0), self.prediction_length), device=self.device, ), num_estimators=self.config.num_estimators,