From c895fdec02b03b789c73112429510a3d1ef000d5 Mon Sep 17 00:00:00 2001 From: DeepKnowledge1 Date: Tue, 27 Oct 2026 12:36:09 +0100 Subject: [PATCH] Improve AnomaVision vs Anomalib benchmark timing - Add end-to-end latency measurement - Handle Anomalib ImageItem samples without default DataLoader collation - Separate model inference latency from full-pipeline latency - Include end-to-end latency in benchmark reports - Preserve timing anomaly detection based on outlier fraction and CV - Keep timing stability and pipeline overhead as separate metrics --- scripts/benchmarks/compare_with_anomalib.py | 96 ++++++++++++++------- 1 file changed, 67 insertions(+), 29 deletions(-) diff --git a/scripts/benchmarks/compare_with_anomalib.py b/scripts/benchmarks/compare_with_anomalib.py index ed0795a..d303d64 100644 --- a/scripts/benchmarks/compare_with_anomalib.py +++ b/scripts/benchmarks/compare_with_anomalib.py @@ -555,32 +555,12 @@ def _build_anomavision_datasets(self): ) return train, test - # def _build_anomalib_datamodule(self, algorithm: str): - # # from anomalib.data import MVTecAD - - # from anomalib.data import Visa as MVTecAD - # kwargs = { - # "root": str(self.dataset_path), - # "category": self.class_name, - # "train_batch_size": 1 if algorithm == "efficientad" else BATCH_SIZE, - # "eval_batch_size": BATCH_SIZE, - # "num_workers": 0, - # "seed": self.seed, - # } - # params = inspect.signature(MVTecAD).parameters - # datamodule = MVTecAD(**{k: v for k, v in kwargs.items() if k in params}) - # datamodule.setup() - # return datamodule - def _build_anomalib_datamodule(self, algorithm: str): from anomalib.data import MVTecAD, Visa dataset_name = self.dataset_path.name.lower() - if "visa" in dataset_name: - datamodule_cls = Visa - else: - datamodule_cls = MVTecAD + datamodule_cls = Visa if "visa" in dataset_name else MVTecAD kwargs = { "root": str(self.dataset_path), @@ -593,7 +573,6 @@ def _build_anomalib_datamodule(self, algorithm: str): params = inspect.signature(datamodule_cls).parameters - # Apply the benchmark image size only if supported by this constructor. if "image_size" in params: kwargs["image_size"] = IMAGE_SIZE @@ -660,6 +639,44 @@ def _benchmark_latency( anomaly_flag, ) + def _benchmark_end_to_end_latency(self, dataset, call_batch) -> float: + """Measure per-image latency including dataset retrieval and preprocessing. + + Samples are fetched directly with ``dataset[index]`` rather than through a + DataLoader. This avoids PyTorch's default collation, which cannot handle + Anomalib's ImageItem dataclass. Timing includes dataset transforms, device + transfer, model preprocessing/inference, and prediction generation; metric + computation and report writing are excluded. + """ + dataset_size = len(dataset) + if dataset_size == 0: + raise RuntimeError("Cannot time end-to-end latency on an empty dataset") + + sample_index = 0 + + def next_sample(): + nonlocal sample_index + sample = dataset[sample_index] + sample_index = (sample_index + 1) % dataset_size + return sample + + self._reset_memory() + with torch.inference_mode(): + for _ in range(WARMUP_ITERS): + call_batch(next_sample()) + sync(self.device) + + times = [] + for _ in range(TIMING_ITERS): + sync(self.device) + start = time.perf_counter() + sample = next_sample() + call_batch(sample) + sync(self.device) + times.append(time.perf_counter() - start) + + return float(np.mean(times) * 1000.0) + def _build_anomavision_model(self, algorithm: str): if algorithm == "padim": from anomavision import Padim @@ -902,9 +919,18 @@ def benchmark_anomavision(self, algorithm: str) -> ModelMetrics: metrics.timing_cv, metrics.timing_anomaly_flag, ) = self._last_timing_checks - metrics.end_to_end_latency_ms = float( - "nan" - ) # preprocessing is performed by the dataset, outside model.predict timing + + def anomavision_end_to_end(sample): + images = sample[0] + if not isinstance(images, torch.Tensor): + images = torch.as_tensor(images) + if images.ndim == 3: + images = images.unsqueeze(0) + model.predict(images.to(self.device)) + + metrics.end_to_end_latency_ms = self._benchmark_end_to_end_latency( + test_dataset, anomavision_end_to_end + ) metrics.__dict__.update(self._evaluate_anomavision(model, test_loader)) print( f" image={metrics.image_auroc:.4f} pixel={metrics.pixel_auroc:.4f} latency={metrics.latency_ms:.2f}ms" @@ -981,9 +1007,20 @@ def benchmark_anomalib(self, algorithm: str) -> ModelMetrics: metrics.timing_cv, metrics.timing_anomaly_flag, ) = self._last_timing_checks - metrics.end_to_end_latency_ms = float( - "nan" - ) # full data pipeline is intentionally not included in this inference-only benchmark + + def anomalib_end_to_end(batch): + images = _get_anomalib_batch_value(batch, "image") + if not isinstance(images, torch.Tensor): + images = torch.as_tensor(images) + if images.ndim == 3: + images = images.unsqueeze(0) + images = images.to(self.device) + processed = self._anomalib_preprocess(model, images) + self._anomalib_raw(model, processed) + + metrics.end_to_end_latency_ms = self._benchmark_end_to_end_latency( + test_loader.dataset, anomalib_end_to_end + ) metrics.__dict__.update(self._evaluate_anomalib(model, test_loader)) print( f" image={metrics.image_auroc:.4f} pixel={metrics.pixel_auroc:.4f} latency={metrics.latency_ms:.2f}ms" @@ -1094,13 +1131,14 @@ def report(self, results: List[ModelMetrics]) -> None: "state_dict_size_mb", "training_memory_mb", "inference_memory_mb", + "end_to_end_latency_ms", "timing_anomaly_flag", ] ) + "" ) config = rows[0] - html = f"""AnomaVision Benchmark - {self.class_name}

🔬 AnomaVision Benchmark

{self.class_name} · {config['image_size']} · {self.device}
PaDiMPatchCoreEfficientADAnomaVision vs Anomalib

Benchmark Configuration

Backbone{config['backbone']}
Image size{config['image_size']}
Layer indices{config['layer_indices']}
Layers{config['layers']}
N features{config['n_features']}
{''.join(html_rows)}
AlgorithmImplementationImage APImage F1Image AUROCImage SpecificityPixel APPixel F1Pixel AUROCAUPRODiceIoUFit Time (s)Latency (ms)P95 (ms)FPSModel (MB)Train RAM/VRAM (MB)Inference RAM/VRAM (MB)Timing anomaly
""" + html = f"""AnomaVision Benchmark - {self.class_name}

🔬 AnomaVision Benchmark

{self.class_name} · {config['image_size']} · {self.device}
PaDiMPatchCoreEfficientADAnomaVision vs Anomalib

Benchmark Configuration

Backbone{config['backbone']}
Image size{config['image_size']}
Layer indices{config['layer_indices']}
Layers{config['layers']}
N features{config['n_features']}
{''.join(html_rows)}
AlgorithmImplementationImage APImage F1Image AUROCImage SpecificityPixel APPixel F1Pixel AUROCAUPRODiceIoUFit Time (s)Latency (ms)P95 (ms)FPSModel (MB)Train RAM/VRAM (MB)Inference RAM/VRAM (MB)End-to-end (ms/image)Timing anomaly
""" with open(f"{stem}.html", "w", encoding="utf-8") as handle: handle.write(html) print("\nReports:")