diff --git a/scripts/benchmarks/compare_with_anomalib.py b/scripts/benchmarks/compare_with_anomalib.py
index ed0795a..d303d64 100644
--- a/scripts/benchmarks/compare_with_anomalib.py
+++ b/scripts/benchmarks/compare_with_anomalib.py
@@ -555,32 +555,12 @@ def _build_anomavision_datasets(self):
)
return train, test
- # def _build_anomalib_datamodule(self, algorithm: str):
- # # from anomalib.data import MVTecAD
-
- # from anomalib.data import Visa as MVTecAD
- # kwargs = {
- # "root": str(self.dataset_path),
- # "category": self.class_name,
- # "train_batch_size": 1 if algorithm == "efficientad" else BATCH_SIZE,
- # "eval_batch_size": BATCH_SIZE,
- # "num_workers": 0,
- # "seed": self.seed,
- # }
- # params = inspect.signature(MVTecAD).parameters
- # datamodule = MVTecAD(**{k: v for k, v in kwargs.items() if k in params})
- # datamodule.setup()
- # return datamodule
-
def _build_anomalib_datamodule(self, algorithm: str):
from anomalib.data import MVTecAD, Visa
dataset_name = self.dataset_path.name.lower()
- if "visa" in dataset_name:
- datamodule_cls = Visa
- else:
- datamodule_cls = MVTecAD
+ datamodule_cls = Visa if "visa" in dataset_name else MVTecAD
kwargs = {
"root": str(self.dataset_path),
@@ -593,7 +573,6 @@ def _build_anomalib_datamodule(self, algorithm: str):
params = inspect.signature(datamodule_cls).parameters
- # Apply the benchmark image size only if supported by this constructor.
if "image_size" in params:
kwargs["image_size"] = IMAGE_SIZE
@@ -660,6 +639,44 @@ def _benchmark_latency(
anomaly_flag,
)
+ def _benchmark_end_to_end_latency(self, dataset, call_batch) -> float:
+ """Measure per-image latency including dataset retrieval and preprocessing.
+
+ Samples are fetched directly with ``dataset[index]`` rather than through a
+ DataLoader. This avoids PyTorch's default collation, which cannot handle
+ Anomalib's ImageItem dataclass. Timing includes dataset transforms, device
+ transfer, model preprocessing/inference, and prediction generation; metric
+ computation and report writing are excluded.
+ """
+ dataset_size = len(dataset)
+ if dataset_size == 0:
+ raise RuntimeError("Cannot time end-to-end latency on an empty dataset")
+
+ sample_index = 0
+
+ def next_sample():
+ nonlocal sample_index
+ sample = dataset[sample_index]
+ sample_index = (sample_index + 1) % dataset_size
+ return sample
+
+ self._reset_memory()
+ with torch.inference_mode():
+ for _ in range(WARMUP_ITERS):
+ call_batch(next_sample())
+ sync(self.device)
+
+ times = []
+ for _ in range(TIMING_ITERS):
+ sync(self.device)
+ start = time.perf_counter()
+ sample = next_sample()
+ call_batch(sample)
+ sync(self.device)
+ times.append(time.perf_counter() - start)
+
+ return float(np.mean(times) * 1000.0)
+
def _build_anomavision_model(self, algorithm: str):
if algorithm == "padim":
from anomavision import Padim
@@ -902,9 +919,18 @@ def benchmark_anomavision(self, algorithm: str) -> ModelMetrics:
metrics.timing_cv,
metrics.timing_anomaly_flag,
) = self._last_timing_checks
- metrics.end_to_end_latency_ms = float(
- "nan"
- ) # preprocessing is performed by the dataset, outside model.predict timing
+
+ def anomavision_end_to_end(sample):
+ images = sample[0]
+ if not isinstance(images, torch.Tensor):
+ images = torch.as_tensor(images)
+ if images.ndim == 3:
+ images = images.unsqueeze(0)
+ model.predict(images.to(self.device))
+
+ metrics.end_to_end_latency_ms = self._benchmark_end_to_end_latency(
+ test_dataset, anomavision_end_to_end
+ )
metrics.__dict__.update(self._evaluate_anomavision(model, test_loader))
print(
f" image={metrics.image_auroc:.4f} pixel={metrics.pixel_auroc:.4f} latency={metrics.latency_ms:.2f}ms"
@@ -981,9 +1007,20 @@ def benchmark_anomalib(self, algorithm: str) -> ModelMetrics:
metrics.timing_cv,
metrics.timing_anomaly_flag,
) = self._last_timing_checks
- metrics.end_to_end_latency_ms = float(
- "nan"
- ) # full data pipeline is intentionally not included in this inference-only benchmark
+
+ def anomalib_end_to_end(batch):
+ images = _get_anomalib_batch_value(batch, "image")
+ if not isinstance(images, torch.Tensor):
+ images = torch.as_tensor(images)
+ if images.ndim == 3:
+ images = images.unsqueeze(0)
+ images = images.to(self.device)
+ processed = self._anomalib_preprocess(model, images)
+ self._anomalib_raw(model, processed)
+
+ metrics.end_to_end_latency_ms = self._benchmark_end_to_end_latency(
+ test_loader.dataset, anomalib_end_to_end
+ )
metrics.__dict__.update(self._evaluate_anomalib(model, test_loader))
print(
f" image={metrics.image_auroc:.4f} pixel={metrics.pixel_auroc:.4f} latency={metrics.latency_ms:.2f}ms"
@@ -1094,13 +1131,14 @@ def report(self, results: List[ModelMetrics]) -> None:
"state_dict_size_mb",
"training_memory_mb",
"inference_memory_mb",
+ "end_to_end_latency_ms",
"timing_anomaly_flag",
]
)
+ ""
)
config = rows[0]
- html = f"""
AnomaVision Benchmark - {self.class_name}🔬 AnomaVision Benchmark
{self.class_name} · {config['image_size']} · {self.device}
PaDiMPatchCoreEfficientADAnomaVision vs Anomalib
Benchmark Configuration
Backbone{config['backbone']}
Image size{config['image_size']}
Layer indices{config['layer_indices']}
Layers{config['layers']}
N features{config['n_features']}
| Algorithm | Implementation | Image AP | Image F1 | Image AUROC | Image Specificity | Pixel AP | Pixel F1 | Pixel AUROC | AUPRO | Dice | IoU | Fit Time (s) | Latency (ms) | P95 (ms) | FPS | Model (MB) | Train RAM/VRAM (MB) | Inference RAM/VRAM (MB) | Timing anomaly |
{''.join(html_rows)}
"""
+ html = f"""AnomaVision Benchmark - {self.class_name}🔬 AnomaVision Benchmark
{self.class_name} · {config['image_size']} · {self.device}
PaDiMPatchCoreEfficientADAnomaVision vs Anomalib
Benchmark Configuration
Backbone{config['backbone']}
Image size{config['image_size']}
Layer indices{config['layer_indices']}
Layers{config['layers']}
N features{config['n_features']}
| Algorithm | Implementation | Image AP | Image F1 | Image AUROC | Image Specificity | Pixel AP | Pixel F1 | Pixel AUROC | AUPRO | Dice | IoU | Fit Time (s) | Latency (ms) | P95 (ms) | FPS | Model (MB) | Train RAM/VRAM (MB) | Inference RAM/VRAM (MB) | End-to-end (ms/image) | Timing anomaly |
{''.join(html_rows)}
"""
with open(f"{stem}.html", "w", encoding="utf-8") as handle:
handle.write(html)
print("\nReports:")