diff --git a/changelog.d/154.public-torch-backend.added b/changelog.d/154.public-torch-backend.added new file mode 100644 index 00000000..d842a834 --- /dev/null +++ b/changelog.d/154.public-torch-backend.added @@ -0,0 +1 @@ +Added the public `ExportBackend.Torch` backend for fast torch inference, e.g. running evals. `finalize(backend=ExportBackend.Torch)` folds weight compression (palettization, weight quantization, pruning) into dense weights and keeps activation fake quantization. The private `ExportBackend._TORCH` name is deprecated in favor of `ExportBackend.Torch`. diff --git a/src/coreai_opt/_utils/torch_utils.py b/src/coreai_opt/_utils/torch_utils.py index 3343d450..0b86f051 100644 --- a/src/coreai_opt/_utils/torch_utils.py +++ b/src/coreai_opt/_utils/torch_utils.py @@ -362,6 +362,54 @@ def remove_compression_parametrizations( return num_removed +def fold_compression_parametrizations( + model: torch.nn.Module, + parametrization_types: type[torch.nn.Module] | tuple[type[torch.nn.Module], ...], +) -> int: + """Fold parametrized tensors carrying a compression parametrization into plain tensors. + + Every tensor whose parametrization list contains an instance of + ``parametrization_types`` is unparametrized, keeping its current + parametrized value, computed in eval mode. The whole parametrization list on + that tensor is folded, since composed parametrizations cannot be folded + individually. + + The value is written into the ``original`` tensor in place (see + ``torch.nn.utils.parametrize.remove_parametrizations``), so tensor identity + is preserved and tied tensors stay tied. + + Args: + model (torch.nn.Module): The model to modify in place. + parametrization_types (type[torch.nn.Module] | tuple[type[torch.nn.Module], ...]): + Parametrization types that mark a tensor for folding. + + Returns: + int: Number of tensors folded. + """ + to_fold = [ + (module, tensor_name) + for module in model.modules() + if P.is_parametrized(module) + for tensor_name, parametrizations in module.parametrizations.items() + if any(isinstance(p, parametrization_types) for p in parametrizations) + ] + + folded_originals: set[int] = set() + for module, tensor_name in to_fold: + parametrizations = module.parametrizations[tensor_name] + # Only the parametrizations are moved to eval: they are discarded by the fold, so + # their mode needs no restoring and the rest of the model is left untouched. + parametrizations.eval() + original = parametrizations.original + # A tensor tied to one folded earlier shares its ``original``, which already + # holds the folded value; re-applying the parametrizations would compress it twice. + leave_parametrized = id(original) not in folded_originals + folded_originals.add(id(original)) + P.remove_parametrizations(module, tensor_name, leave_parametrized=leave_parametrized) + + return len(to_fold) + + def find_parametrization_matching_cls( module: torch.nn.Module, param_name: str, diff --git a/src/coreai_opt/base_model_compressor.py b/src/coreai_opt/base_model_compressor.py index 84362570..2de3f126 100644 --- a/src/coreai_opt/base_model_compressor.py +++ b/src/coreai_opt/base_model_compressor.py @@ -79,6 +79,21 @@ def _mark_model_as_prepared(model: torch.nn.Module) -> None: """ model.register_buffer(_COREAI_OPT_PREPARED_ATTR, torch.tensor(True), persistent=False) + @staticmethod + def _unmark_model_as_prepared(model: torch.nn.Module) -> None: + """ + Clear the prepared marker from a model. + + Called at the end of ``finalize`` so the finalized model can be further + processed (e.g., by Quantizer for activation quantization after weight + palettization). + + Args: + model: The model to unmark. + """ + if hasattr(model, _COREAI_OPT_PREPARED_ATTR): + delattr(model, _COREAI_OPT_PREPARED_ATTR) + @classmethod def supported_modules(cls) -> tuple[type[torch.nn.Module]]: """ @@ -129,10 +144,12 @@ def finalize( For example, for quantization: - For CoreAI backend (default), custom ops will be inserted. - For CoreML, compression metadata will be inserted as buffers in the model. + - For Torch, weight compression is folded into dense weights for fast torch + inference. Args: model: Optional model to finalize. If None, uses self._model - backend: Target export backend (CoreAI, CoreML) + backend: Target backend (CoreAI, CoreML, Torch) mmap_dir (str | None): If provided, serialize finalized compressed weights to safetensors files under this directory and re-load them via mmap, so large models can be finalized without holding diff --git a/src/coreai_opt/common.py b/src/coreai_opt/common.py index 8e180955..a2dda78b 100644 --- a/src/coreai_opt/common.py +++ b/src/coreai_opt/common.py @@ -140,19 +140,29 @@ class ExportBackend(_StrEnum, metaclass=_DeprecatedMemberEnumMeta): Each member is a string value representing the backend format. Attributes: + Torch: Plain PyTorch model for fast inference (e.g. running evals). Weight + compression is folded into dense weights; activation fake quantization + is kept. CoreML: Core ML format with compression metadata buffers. CoreAI: Core AI format with custom ops. """ - _TORCH = auto() + Torch = auto() CoreML = auto() CoreAI = auto() - __deprecated_aliases__: ClassVar[dict[str, str]] = {"MIL": "CoreML", "MLIR": "CoreAI"} + __deprecated_aliases__: ClassVar[dict[str, str]] = { + "_TORCH": "Torch", + "MIL": "CoreML", + "MLIR": "CoreAI", + } if TYPE_CHECKING: # Surface the deprecated aliases above for static type checkers. + _TORCH: ExportBackend + """Deprecated. Use `ExportBackend.Torch` instead.""" + MIL: ExportBackend """Deprecated. Use `ExportBackend.CoreML` instead.""" diff --git a/src/coreai_opt/palettization/kmeans/palettizer.py b/src/coreai_opt/palettization/kmeans/palettizer.py index 2be31405..affcc585 100644 --- a/src/coreai_opt/palettization/kmeans/palettizer.py +++ b/src/coreai_opt/palettization/kmeans/palettizer.py @@ -29,6 +29,7 @@ ) from coreai_opt._utils.spec_utils import PartialConstructor as _PartialConstructor from coreai_opt._utils.torch_utils import ( + fold_compression_parametrizations as _fold_compression_parametrizations, move_model_to_train as _move_model_to_train, remove_compression_parametrizations as _remove_compression_parametrizations, ) @@ -45,6 +46,7 @@ from coreai_opt.palettization.spec.fake_palettize import ( _disable_fake_palett, _enable_fake_palett, + _FakePalettizeImplBase, ) from ._prepare_for_export import ( @@ -477,15 +479,15 @@ def finalize( ) -> torch.nn.Module: """Convert palettized model to backend-specific representations. - Only call ``finalize`` when exporting to a target backend. For torch-based - evaluation, use the model returned by ``prepare()`` directly rather than - calling ``finalize``. + Use ``backend=ExportBackend.Torch`` for fast torch inference (e.g. running + evals): palettized weights are folded into dense weights, so the model no + longer reconstructs them from the LUT on every forward pass. Args: model (nn.Module | None): Model to finalize. If None, uses the internal prepared model. - backend (ExportBackend): Target export backend for the palettized - model. Supports CoreAI (default) and CoreML backends. + backend (ExportBackend): Target backend for the palettized model. + Supports CoreAI (default), CoreML and Torch backends. mmap_dir (str | None): If provided, finalized palettized weights are written under this directory and re-loaded as mmap-backed tensors so they don't have to be held in RAM. Only supported @@ -518,8 +520,8 @@ def finalize( # Backend-specific processing match backend: - case ExportBackend._TORCH: - pass + case ExportBackend.Torch: + _fold_compression_parametrizations(finalized_model, _FakePalettizeImplBase) case ExportBackend.CoreAI: finalized_model = _prepare_for_mlir_export(finalized_model, mmap_dir=mmap_dir) @@ -531,10 +533,7 @@ def finalize( msg = f"Unsupported backend: {backend}" raise NotImplementedError(msg) - # Clear the prepared flag so the finalized model can be further processed - # (e.g., by Quantizer for activation quantization after weight palettization) - if hasattr(finalized_model, "_coreai_opt_prepared"): - delattr(finalized_model, "_coreai_opt_prepared") + self._unmark_model_as_prepared(finalized_model) return finalized_model diff --git a/src/coreai_opt/pruning/magnitude_pruner.py b/src/coreai_opt/pruning/magnitude_pruner.py index 035b27c8..2daff311 100644 --- a/src/coreai_opt/pruning/magnitude_pruner.py +++ b/src/coreai_opt/pruning/magnitude_pruner.py @@ -20,6 +20,7 @@ from coreai_opt._utils.spec_utils import PartialConstructor as _PartialConstructor from coreai_opt._utils.torch_utils import ( find_parametrization_matching_cls as _find_parametrization_matching_cls, + fold_compression_parametrizations as _fold_compression_parametrizations, move_model_to_eval as _move_model_to_eval, ) from coreai_opt.common import ExportBackend @@ -142,10 +143,14 @@ def finalize( ) -> torch.nn.Module: """Finalize the model to be lowered to the target backend. + Use ``backend=ExportBackend.Torch`` for fast torch inference (e.g. running + evals): pruned weights are folded into dense weights. + Args: model (torch.nn.Module | None): Model to finalize. Uses the model passed at construction time when ``None``. - backend (ExportBackend): Target export backend. + backend (ExportBackend): Target backend. Supports CoreAI (default), + CoreML and Torch backends. Returns: torch.nn.Module: The finalized model ready for the target backend. @@ -161,8 +166,8 @@ def finalize( raise RuntimeError("Model must be prepared before finalization. Call prepare() first.") match backend: - case ExportBackend._TORCH: - pass + case ExportBackend.Torch: + _fold_compression_parametrizations(model, PruneImplBase) case ExportBackend.CoreAI: model = _prepare_for_mlir_export(model) @@ -173,6 +178,7 @@ def finalize( case _: raise ValueError(f"Unsupported backend: {backend}") + self._unmark_model_as_prepared(model) if update_internal: self._model = model return model diff --git a/src/coreai_opt/quantization/_eager/quantizer.py b/src/coreai_opt/quantization/_eager/quantizer.py index 15e89384..596b901d 100644 --- a/src/coreai_opt/quantization/_eager/quantizer.py +++ b/src/coreai_opt/quantization/_eager/quantizer.py @@ -27,7 +27,11 @@ TorchFunctionEagerHandler, ) from coreai_opt._utils.spec_utils import PartialConstructor -from coreai_opt._utils.torch_utils import move_model_to_eval, move_model_to_train +from coreai_opt._utils.torch_utils import ( + fold_compression_parametrizations, + move_model_to_eval, + move_model_to_train, +) from coreai_opt.common import ExportBackend from coreai_opt.config.compression_config import ModuleCompressionConfig from coreai_opt.config.spec import CompressionTargetTensor @@ -237,13 +241,13 @@ def finalize( """Convert quantized model to backend-specific representations. Converts fake quantization modules into backend-specific quantization ops. - Only call ``finalize`` when exporting to a target backend. For torch-based evaluation, use - the model returned by ``prepare()`` directly rather than calling ``finalize``. + For ``backend=ExportBackend.Torch``, weight fake quantization is folded into + dense weights and activation fake quantization is kept. Args: model: Model to finalize (uses internal model if None). - backend: Target export backend for the quantized model. Supports - CoreAI (default) and CoreML. + backend: Target backend for the quantized model. Supports + CoreAI (default), CoreML and Torch. mmap_dir (str | None): If provided, serialize finalized quantized weights to safetensors files under this directory and re-load them via mmap. Only supported with the CoreAI backend. The @@ -272,7 +276,10 @@ def finalize( # Backend specific logic match backend: - case ExportBackend._TORCH: + case ExportBackend.Torch: + # Activation fake quantization lives in standalone modules, not + # parametrizations, so only weight fake quantization is folded. + fold_compression_parametrizations(model, FakeQuantizeImplBase) finalized_model = model case ExportBackend.CoreAI: finalized_model = prepare_for_mlir_export(model, mmap_dir=mmap_dir) @@ -282,6 +289,7 @@ def finalize( msg = f"Unsupported backend {backend}" raise ValueError(msg) + self._unmark_model_as_prepared(finalized_model) return finalized_model @contextmanager diff --git a/src/coreai_opt/quantization/_export_utils.py b/src/coreai_opt/quantization/_export_utils.py index 0d97a1e1..e57cddcc 100644 --- a/src/coreai_opt/quantization/_export_utils.py +++ b/src/coreai_opt/quantization/_export_utils.py @@ -418,7 +418,7 @@ def validate_activation_export_supported(fake_quant_mod: FakeQuantizeImplBase) - "Core AI export does not support PerBlockGranularity on activations: a " "per-block scale cannot be canonicalized to 0-D or 1-D. Use " "PerTensorGranularity or PerChannelGranularity, export with " - "backend=ExportBackend._TORCH, or install and activate an extension " + "backend=ExportBackend.Torch, or install and activate an extension " "package that registers an activation export handler for this granularity." ) diff --git a/src/coreai_opt/quantization/_graph/_prepare_for_export.py b/src/coreai_opt/quantization/_graph/_prepare_for_export.py index cb8bc7d5..e06625ff 100644 --- a/src/coreai_opt/quantization/_graph/_prepare_for_export.py +++ b/src/coreai_opt/quantization/_graph/_prepare_for_export.py @@ -6,7 +6,7 @@ """Export preparation utilities for quantized models. This module provides functions to prepare PT2E-quantized models for different -export backends (CoreML, CoreAI) by converting fake quantization operations to +backends (CoreML, CoreAI, Torch) by converting fake quantization operations to backend-specific representations. """ @@ -653,6 +653,52 @@ def prepare_for_mil_export(model: torch.fx.GraphModule) -> torch.fx.GraphModule: return model +def prepare_for_torch_inference(model: torch.fx.GraphModule) -> torch.fx.GraphModule: + """Fold weight fake quantization into dense weights for fast torch inference. + + Each weight read through a fake quantization node is replaced by its + fake-quantized value, and the node and its module are removed. Activation + fake quantization is kept. + + Args: + model (torch.fx.GraphModule): The quantized GraphModule. Modified in place. + + Returns: + torch.fx.GraphModule: The same GraphModule with weight fake quantization folded. + + Raises: + TypeError: If model is not a torch.fx.GraphModule. + """ + if not isinstance(model, torch.fx.GraphModule): + raise TypeError("Model must be a torch.fx.GraphModule") + + for node, fake_quant_mod in _collect_fake_quantize_nodes(model): + if not _is_weight_fake_quant(node, fake_quant_mod): + continue + + weight_node = _get_fake_quant_input(node) + if len(weight_node.users) > 1: + logger.warning( + "Weight %r is also read directly by the graph; " + "keeping its fake quantization instead of folding it.", + weight_node.target, + ) + continue + + dense_weight = resolve_attr(model, str(weight_node.target)) + with torch.no_grad(): + dense_weight.data = fake_quant_mod(dense_weight.data) + + node.replace_all_uses_with(weight_node) + model.graph.erase_node(node) + remove_fake_quant_module(model, node) + + model.graph.eliminate_dead_code() + model.recompile() + + return model + + def _release_unused_dense_weights( model: torch.fx.GraphModule, dense_weight_targets: Iterable[str], diff --git a/src/coreai_opt/quantization/_graph/quantizer.py b/src/coreai_opt/quantization/_graph/quantizer.py index 03e9dec0..5d9cbb15 100644 --- a/src/coreai_opt/quantization/_graph/quantizer.py +++ b/src/coreai_opt/quantization/_graph/quantizer.py @@ -89,6 +89,7 @@ _move_cache_dequant_to_output, prepare_for_mil_export, prepare_for_mlir_export, + prepare_for_torch_inference, ) from ._qspec_reconcile import ( _AnnotationContext, @@ -1118,16 +1119,14 @@ def finalize( This method converts the prepared model with fake quantization modules to a final quantized model with actual quantized operations, ready for deployment - to the target export backend. - - Only call ``finalize`` when exporting to a target backend. For torch-based - evaluation, use the model returned by ``prepare()`` directly rather than - calling ``finalize``. + to the target export backend. For ``backend=ExportBackend.Torch``, weight fake + quantization is folded into dense weights and activation fake quantization + is kept. Args: model: Optional model to finalize. If None, uses the internal prepared model. - backend: Target export backend for the quantized model. - Supports CoreAI (default) and CoreML. + backend: Target backend for the quantized model. + Supports CoreAI (default), CoreML and Torch. mmap_dir (str | None): If provided, each quantized weight is serialized to its own safetensors file under this directory and re-read mmap-backed. Only supported with the CoreAI backend. @@ -1181,11 +1180,11 @@ def finalize( # Backend-specific processing match backend: - case ExportBackend._TORCH: + case ExportBackend.Torch: # KV-cache quantization is a no-op for the torch backend: the # fake-quant observers on the cache op input stay in the graph. # Cache-buffer retyping only happens on the CoreAI backend. - pass + finalized_model = prepare_for_torch_inference(finalized_model) case ExportBackend.CoreML: if self._config.kv_cache_quant_configs: @@ -1216,6 +1215,7 @@ def finalize( # allow_exported_model_train_eval again on the converted model. allow_exported_model_train_eval(finalized_model) + self._unmark_model_as_prepared(finalized_model) return finalized_model @contextmanager diff --git a/src/coreai_opt/quantization/quantizer.py b/src/coreai_opt/quantization/quantizer.py index 581edc0a..8bd7aa8d 100644 --- a/src/coreai_opt/quantization/quantizer.py +++ b/src/coreai_opt/quantization/quantizer.py @@ -417,7 +417,7 @@ def _validate_no_persistent_observer_calculators( ``EagerActivationExportHandler`` in eager mode. """ - if backend == ExportBackend._TORCH: + if backend == ExportBackend.Torch: return model_to_check = model if model is not None else self._model stateless_fq_names = [ @@ -432,7 +432,7 @@ def _validate_no_persistent_observer_calculators( f"backend={backend} does not yet support qparams calculators that " f"recompute every forward (e.g. dynamic quantization). " f"Affected FakeQuantize modules: {stateless_fq_names}. Use " - f"backend=ExportBackend._TORCH for torch-only inference." + f"backend=ExportBackend.Torch for torch-only inference." ) def finalize( @@ -445,8 +445,6 @@ def finalize( """Convert quantized model to backend-specific representations. Converts fake quantization modules into backend-specific quantization ops. - Only call ``finalize`` when exporting to a target backend. For torch-based evaluation, use - the model returned by ``prepare()`` directly rather than calling ``finalize``. Backend-specific processing: @@ -454,12 +452,15 @@ def finalize( with Core AI specific PyTorch custom ops. - CoreML: Prepares for CoreML export by registering compression metadata as buffers and removes fake quantization modules. + - Torch: Prepares for fast torch inference (e.g. running evals) by folding + weight fake quantization into dense weights. Activation fake quantization + is kept. Args: model: Optional model to finalize. If None, uses the internal prepared model. backend: Target export backend for the quantized model. Supports - CoreAI (default), CoreML, and _TORCH backends. + CoreAI (default), CoreML, and Torch backends. mmap_dir (str | None): If provided, serialize finalized quantized weights to safetensors files under this directory and re-load them via mmap, one file per weight. Supported in both execution diff --git a/tests/deprecation/test_export_backend.py b/tests/deprecation/test_export_backend.py index 053de2c7..946f07c4 100644 --- a/tests/deprecation/test_export_backend.py +++ b/tests/deprecation/test_export_backend.py @@ -20,7 +20,7 @@ class TestExportBackend: @pytest.mark.parametrize( ("value", "expected"), [ - ("_torch", ExportBackend._TORCH), + ("torch", ExportBackend.Torch), ("coreml", ExportBackend.CoreML), ("coreai", ExportBackend.CoreAI), ], @@ -38,6 +38,7 @@ def test_construct_from_current_value_does_not_warn( @pytest.mark.parametrize( ("old_name", "expected"), [ + ("_TORCH", ExportBackend.Torch), ("MIL", ExportBackend.CoreML), ("MLIR", ExportBackend.CoreAI), ], diff --git a/tests/palettization/test_kmeans_palettizer.py b/tests/palettization/test_kmeans_palettizer.py index 54ef2b74..ad1b3b6a 100644 --- a/tests/palettization/test_kmeans_palettizer.py +++ b/tests/palettization/test_kmeans_palettizer.py @@ -33,6 +33,7 @@ from coreai_opt.palettization.spec.errors import _IncompatibleGranularityError from coreai_opt.palettization.spec.fake_palettize import _FakePalettizeImplBase from coreai_opt.quantization.spec import QuantizationScheme, QuantizationSpec +from tests.models.simple import SimpleModel from tests.utils import spy_cluster_weights @@ -417,20 +418,26 @@ def test_prepared_model_supports_torch_inference( assert output.shape == (1, 10) def test_finalize_torch_backend(self, simple_conv_linear_model, simple_model_input): + """Torch finalize folds palettized weights, matches the prepared output, and + strict-loads into a fresh model.""" palettizer = KMeansPalettizer(simple_conv_linear_model) prepared_model = palettizer.prepare(example_inputs=(simple_model_input,)) prepared_output = prepared_model(simple_model_input) - finalized_model = palettizer.finalize(backend=ExportBackend._TORCH) + finalized_model = palettizer.finalize(backend=ExportBackend.Torch) - assert finalized_model is not None + assert not any(is_parametrized(m) for m in finalized_model.modules()) finalized_output = finalized_model(simple_model_input) torch.testing.assert_close(finalized_output, prepared_output) + fresh_model = SimpleModel() + fresh_model.load_state_dict(finalized_model.state_dict(), strict=True) + torch.testing.assert_close(fresh_model(simple_model_input), prepared_output) + def test_finalize_mil_backend(self, simple_conv_linear_model, simple_model_input): palettizer = KMeansPalettizer(simple_conv_linear_model) @@ -462,7 +469,7 @@ def _palettize_model(model, palettization_config, example_input, mmap_path, back @pytest.mark.parametrize( "backend", - [ExportBackend._TORCH, ExportBackend.CoreML], + [ExportBackend.Torch, ExportBackend.CoreML], ) def test_finalize_mmap_value_error_unsupported_backend( self, simple_conv_linear_model, simple_model_input, basic_config, tmp_path, backend @@ -483,14 +490,14 @@ def test_finalize_mmap_value_error_unsupported_backend( @pytest.mark.parametrize( "backend", - [ExportBackend._TORCH, ExportBackend.CoreAI, ExportBackend.CoreML], + [ExportBackend.Torch, ExportBackend.CoreAI, ExportBackend.CoreML], ) def test_finalize_weight_parametrization_state_per_backend( self, backend, simple_conv_linear_model, simple_model_input, basic_config ): """Post-finalize parametrization state is backend-specific: - - _TORCH: parametrization preserved with dense ``.original`` intact + - Torch: parametrization folded into a dense weight - CoreAI: parametrization replaced with palettize module; ``.original`` cleared to a zero-size placeholder - CoreML: parametrization removed entirely @@ -499,13 +506,14 @@ def test_finalize_weight_parametrization_state_per_backend( simple_conv_linear_model, basic_config, simple_model_input, None, backend ) + assert not KMeansPalettizer._is_model_prepared(finalized) weight_modules = [m for m in finalized.modules() if isinstance(m, (nn.Conv2d, nn.Linear))] assert weight_modules, "fixture regressed: no Conv2d/Linear to check" - if backend is ExportBackend._TORCH: + if backend is ExportBackend.Torch: for m in weight_modules: - assert P.is_parametrized(m, "weight") - assert m.parametrizations["weight"].original.numel() > 0 + assert not P.is_parametrized(m, "weight") + assert m.weight.numel() > 0 elif backend is ExportBackend.CoreAI: for m in weight_modules: assert P.is_parametrized(m, "weight") diff --git a/tests/pruning/test_magnitude_pruner.py b/tests/pruning/test_magnitude_pruner.py index b3b60e3b..96e75914 100644 --- a/tests/pruning/test_magnitude_pruner.py +++ b/tests/pruning/test_magnitude_pruner.py @@ -12,6 +12,7 @@ import torch.nn as nn import torch.nn.functional as F +from coreai_opt import ExportBackend from coreai_opt.pruning import ( MagnitudePruner, MagnitudePrunerConfig, @@ -84,6 +85,7 @@ def test_torch_finalize(self) -> None: prepared_weight = model.weight.detach().clone() finalized = pruner.finalize() + assert not pruner._is_model_prepared(finalized) assert hasattr(finalized, "parametrizations") assert "weight" in finalized.parametrizations @@ -91,6 +93,30 @@ def test_torch_finalize(self) -> None: sparsity = (finalized.weight == 0).float().mean().item() assert sparsity == 0.5 + def test_finalize_torch_folds_mask(self) -> None: + """Torch finalize folds the pruning mask into a dense weight that strict-loads + into a fresh model.""" + model = nn.Linear(100, 50, bias=False) + with torch.no_grad(): + model.weight.copy_(torch.arange(1, 5001, dtype=torch.float32).reshape(50, 100)) + example_input = torch.randn(1, 100) + + pruner = MagnitudePruner(model) + prepared_model = pruner.prepare((example_input,)) + prepared_weight = prepared_model.weight.detach().clone() + expected_output = prepared_model(example_input) + + finalized = pruner.finalize(backend=ExportBackend.Torch) + + assert not pruner._is_model_prepared(finalized) + assert not hasattr(finalized, "parametrizations") + assert torch.equal(finalized.weight, prepared_weight) + torch.testing.assert_close(finalized(example_input), expected_output) + fresh_model = nn.Linear(100, 50, bias=False) + fresh_model.load_state_dict(finalized.state_dict(), strict=True) + assert torch.equal(fresh_model.weight, prepared_weight) + torch.testing.assert_close(fresh_model(example_input), expected_output) + def test_invalid_state(self) -> None: """Double prepare raises; finalize without prepare raises.""" model = nn.Linear(10, 10) diff --git a/tests/quantization/test_activation_export_handlers.py b/tests/quantization/test_activation_export_handlers.py index 22f7c3ea..8faa7ece 100644 --- a/tests/quantization/test_activation_export_handlers.py +++ b/tests/quantization/test_activation_export_handlers.py @@ -90,7 +90,7 @@ def test_per_block_export_error_names_the_caller_alternatives() -> None: validate_activation_export_supported(fake_quant_mod) msg = str(e.value) - for alternative in ("PerChannelGranularity", "ExportBackend._TORCH", "extension"): + for alternative in ("PerChannelGranularity", "ExportBackend.Torch", "extension"): assert alternative in msg, f"the error should offer {alternative}: {msg}" diff --git a/tests/quantization/test_conv_bn_folding.py b/tests/quantization/test_conv_bn_folding.py index 438e84d4..2dd9f760 100644 --- a/tests/quantization/test_conv_bn_folding.py +++ b/tests/quantization/test_conv_bn_folding.py @@ -431,7 +431,7 @@ def test_conv_bn_folding_numerical_correctness( Export backend logic is verified separately in test_conv_bn_folding_export_verification. """ - backend = ExportBackend._TORCH + backend = ExportBackend.Torch # Create model and re-initialize BNs with non-trivial params so the fold # actually bakes a non-identity scale into conv weights. model = model_config.model_cls(**model_config.model_kwargs) diff --git a/tests/quantization/test_eager_quant.py b/tests/quantization/test_eager_quant.py index cd932eec..550e22d2 100644 --- a/tests/quantization/test_eager_quant.py +++ b/tests/quantization/test_eager_quant.py @@ -339,7 +339,7 @@ def test_finalize_after_deepcopy(self, simple_model, basic_config, example_input assert quantizer._is_model_prepared(copied_prepared_model) - finalized_model = quantizer.finalize(copied_prepared_model, ExportBackend._TORCH) + finalized_model = quantizer.finalize(copied_prepared_model, ExportBackend.Torch) output = finalized_model(example_input) assert output.shape == (1, 10) @@ -365,7 +365,7 @@ def test_finalize_after_state_dict_roundtrip(self, simple_model, basic_config, e prepared_b = quantizer_b.prepare((example_input,)) prepared_b.load_state_dict(saved_state_dict) - finalized_b = quantizer_b.finalize(prepared_b, ExportBackend._TORCH) + finalized_b = quantizer_b.finalize(prepared_b, ExportBackend.Torch) actual_output = finalized_b(example_input) assert torch.allclose(expected_output, actual_output) @@ -383,27 +383,28 @@ def _quantize_model(model, quantization_config, example_input, mmap_path, backen @pytest.mark.parametrize( "backend", - [ExportBackend._TORCH, ExportBackend.CoreAI, ExportBackend.CoreML], + [ExportBackend.Torch, ExportBackend.CoreAI, ExportBackend.CoreML], ) def test_finalize_weight_parametrization_state_per_backend( self, backend, simple_model, basic_config, example_input ): """Post-finalize parametrization state is backend-specific: - - _TORCH: parametrization preserved with dense ``.original`` intact + - Torch: parametrization folded into a dense weight - CoreAI: parametrization replaced with dequantize module; ``.original`` cleared to a zero-size placeholder - CoreML: parametrization removed entirely """ finalized = self._quantize_model(simple_model, basic_config, example_input, None, backend) + assert not Quantizer._is_model_prepared(finalized) weight_modules = [m for m in finalized.modules() if isinstance(m, (nn.Conv2d, nn.Linear))] assert weight_modules, "fixture regressed: no Conv2d/Linear to check" - if backend is ExportBackend._TORCH: + if backend is ExportBackend.Torch: for m in weight_modules: - assert is_parametrized(m, "weight") - assert m.parametrizations["weight"].original.numel() > 0 + assert not is_parametrized(m, "weight") + assert m.weight.numel() > 0 elif backend is ExportBackend.CoreAI: for m in weight_modules: assert is_parametrized(m, "weight") @@ -413,6 +414,47 @@ def test_finalize_weight_parametrization_state_per_backend( assert not is_parametrized(m, "weight") assert m.weight.numel() > 0 + def test_finalize_torch_matches_prepared_and_keeps_activation_fq( + self, simple_model, basic_config, example_input + ): + """Torch finalize folds weight fake quantization, keeps activation fake + quantization, and produces the prepared model's output.""" + quantizer = Quantizer(simple_model, basic_config) + prepared_model = quantizer.prepare((example_input,)) + expected_output = prepared_model(example_input) + + finalized = quantizer.finalize(backend=ExportBackend.Torch) + + assert not any(is_parametrized(m) for m in finalized.modules()) + assert any(isinstance(m, FakeQuantizeImplBase) for m in finalized.modules()), ( + "activation fake quantization must be kept" + ) + torch.testing.assert_close(finalized(example_input), expected_output) + + def test_finalize_torch_state_dict_loads_into_original_model( + self, simple_model, weight_only_config, example_input + ): + """A weight-only Torch-finalized state dict strict-loads into a fresh original model.""" + quantizer = Quantizer(simple_model, weight_only_config) + quantizer.prepare((example_input,)) + finalized = quantizer.finalize(backend=ExportBackend.Torch) + + fresh_model = SimpleModel() + fresh_model.load_state_dict(finalized.state_dict(), strict=True) + torch.testing.assert_close(fresh_model(example_input), finalized(example_input)) + + def test_finalize_torch_keeps_weight_tie(self, weight_only_config): + """Tied weights sharing a fake quantizer fold into one shared tensor.""" + example_input = torch.randn(2, 4) + quantizer = Quantizer(SharedWeightModel(), weight_only_config) + prepared_model = quantizer.prepare((example_input,)) + expected_output = prepared_model(example_input) + + finalized = quantizer.finalize(backend=ExportBackend.Torch) + + assert finalized.linear1.weight is finalized.linear2.weight + torch.testing.assert_close(finalized(example_input), expected_output) + def test_calibration_mode(self, simple_model, basic_config, example_input): """ Test that calibration mode works as expected, and scales are getting updated @@ -2987,7 +3029,7 @@ class TestEagerHalfPrecisionSupport: @pytest.mark.parametrize( "backend", [ - pytest.param(ExportBackend._TORCH, id="torch"), + pytest.param(ExportBackend.Torch, id="torch"), pytest.param(ExportBackend.CoreAI, id="coreai"), pytest.param(ExportBackend.CoreML, id="coreml"), ], @@ -3014,7 +3056,7 @@ def test_prepare_calibrate_finalize_with_half_precision( finalized_model = quantizer.finalize(backend=backend) if model_dtype == torch.float32 or backend in [ - ExportBackend._TORCH, + ExportBackend.Torch, ExportBackend.CoreAI, ]: output = finalized_model(input_tensor) diff --git a/tests/quantization/test_eager_quant_e2e.py b/tests/quantization/test_eager_quant_e2e.py index 7141f4ce..4619a22e 100644 --- a/tests/quantization/test_eager_quant_e2e.py +++ b/tests/quantization/test_eager_quant_e2e.py @@ -147,7 +147,7 @@ def test_weight_ptq_mnist( f"Accuracy drop too high: before={accuracy:.4f}, after={post_prepare_accuracy:.4f}" ) - finalized_model = quantizer.finalize(backend=ExportBackend._TORCH) + finalized_model = quantizer.finalize(backend=ExportBackend.Torch) finalized_accuracy = utils.eval_model(finalized_model, test_loader) # Accuracy before and after finalize should match @@ -296,7 +296,7 @@ def test_weight_and_activation_ptq_mnist( post_calibrate_accuracy = utils.eval_model(prepared_model, test_loader) assert post_calibrate_accuracy > 90.0, "Expect accuracy to climb above 90% after calibration" - finalized_model = quantizer.finalize(backend=ExportBackend._TORCH) + finalized_model = quantizer.finalize(backend=ExportBackend.Torch) finalized_accuracy = utils.eval_model(finalized_model, test_loader) # Accuracy before and after finalize should match @@ -366,7 +366,7 @@ def test_weight_and_activation_qat_mnist(mnist_pretrained_model, mnist_dataset, print("\n accuracy of post qat model: ", post_qat_accuracy) assert post_qat_accuracy > 96.0, "Expect accuracy to climb above 96% after QAT" - finalized_model = quantizer.finalize(backend=ExportBackend._TORCH) + finalized_model = quantizer.finalize(backend=ExportBackend.Torch) finalized_accuracy = utils.eval_model(finalized_model, test_loader) # Accuracy before and after finalize should match diff --git a/tests/quantization/test_graph_mode_quantizer.py b/tests/quantization/test_graph_mode_quantizer.py index bf47b422..7d761a5b 100644 --- a/tests/quantization/test_graph_mode_quantizer.py +++ b/tests/quantization/test_graph_mode_quantizer.py @@ -227,28 +227,48 @@ def test_prepared_model_supports_torch_inference( def test_finalize_torch_backend( self, simple_conv_linear_model, basic_config, simple_model_input ): - """Test finalize with _TORCH backend.""" + """Torch finalize folds weight fake quantization and keeps activation fake quantization.""" quantizer = Quantizer(simple_conv_linear_model, basic_config) prepared_model = quantizer.prepare((simple_model_input,)) + expected_output = prepared_model(simple_model_input) - finalized_model = quantizer.finalize(prepared_model, ExportBackend._TORCH) + finalized_model = quantizer.finalize(prepared_model, ExportBackend.Torch) assert isinstance(finalized_model, torch.fx.GraphModule) + assert not quantizer._is_model_prepared(finalized_model) + fake_quant_mods = [ + m for m in finalized_model.modules() if isinstance(m, FakeQuantizeImplBase) + ] + assert fake_quant_mods, "activation fake quantization must be kept" + assert all(m.quantization_target != CompressionTargetTensor.WEIGHT for m in fake_quant_mods) + torch.testing.assert_close(finalized_model(simple_model_input), expected_output) - # Fake quantize layers should remain the same post finalize - prepared_fake_quant_nodes = get_fake_quant_nodes(prepared_model) - finalized_fake_quant_nodes = get_fake_quant_nodes(finalized_model) + def test_finalize_torch_backend_keeps_weight_read_directly(self, basic_config, caplog): + """A weight the graph also reads directly keeps its fake quantization unfolded.""" - for prepared_node in prepared_fake_quant_nodes: - finalized_node = [ - n for n in finalized_fake_quant_nodes if prepared_node.target == n.target - ][0] + class LinearPlusWeightSum(nn.Module): + def __init__(self) -> None: + super().__init__() + self.linear = nn.Linear(8, 8) - assert prepared_node == finalized_node + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.linear(x) + self.linear.weight.sum() - # Test inference on finalized model - output = finalized_model(simple_model_input) - assert output.shape == (1, 10) + example_input = torch.randn(2, 8) + quantizer = Quantizer(LinearPlusWeightSum(), basic_config) + prepared_model = quantizer.prepare((example_input,)) + expected_output = prepared_model(example_input) + + with caplog.at_level(logging.WARNING): + finalized_model = quantizer.finalize(prepared_model, ExportBackend.Torch) + + assert any( + isinstance(m, FakeQuantizeImplBase) + and m.quantization_target == CompressionTargetTensor.WEIGHT + for m in finalized_model.modules() + ) + assert "keeping its fake quantization" in caplog.text + torch.testing.assert_close(finalized_model(example_input), expected_output) def test_finalize_after_deepcopy( self, simple_conv_linear_model, basic_config, simple_model_input @@ -261,7 +281,7 @@ def test_finalize_after_deepcopy( assert quantizer._is_model_prepared(copied_prepared_model) - finalized_model = quantizer.finalize(copied_prepared_model, ExportBackend._TORCH) + finalized_model = quantizer.finalize(copied_prepared_model, ExportBackend.Torch) assert isinstance(finalized_model, torch.fx.GraphModule) output = finalized_model(simple_model_input) @@ -290,7 +310,7 @@ def test_finalize_after_state_dict_roundtrip( prepared_b = quantizer_b.prepare((simple_model_input,)) prepared_b.load_state_dict(saved_state_dict) - finalized_b = quantizer_b.finalize(prepared_b, ExportBackend._TORCH) + finalized_b = quantizer_b.finalize(prepared_b, ExportBackend.Torch) actual_output = finalized_b(simple_model_input) assert torch.allclose(expected_output, actual_output) @@ -308,6 +328,7 @@ def test_finalize_mil_backend( fake_quant_nodes = get_fake_quant_nodes(prepared_model) finalized_model = quantizer.finalize(backend=ExportBackend.CoreML) + assert not quantizer._is_model_prepared(finalized_model) # Count activation quantization modules (Sequential containing # quantize + dequantize) @@ -345,6 +366,7 @@ def test_finalize_mlir_backend( assert len(fake_quant_nodes) == 4 finalized_model = quantizer.finalize(backend=ExportBackend.CoreAI) + assert not quantizer._is_model_prepared(finalized_model) blockwise_shift_scale_nodes = finalized_model.graph.find_nodes( op="call_function", target=coreai.constexpr_blockwise_shift_scale @@ -484,7 +506,7 @@ def test_end_to_end_workflow(self, simple_conv_linear_model, basic_config, simpl assert f"Deepcopy failed for prepared model with error: {e}" # Step 3: Finalize - finalized_model = quantizer.finalize(backend=ExportBackend._TORCH) + finalized_model = quantizer.finalize(backend=ExportBackend.Torch) assert isinstance(finalized_model, torch.fx.GraphModule) # prepare model output before finalize should match output after finalize final_output = finalized_model(simple_model_input_3) @@ -938,7 +960,7 @@ def test_finalize_before_prepare(self, simple_conv_linear_model, basic_config): # Should raise RuntimeError when trying to finalize unprepared model with pytest.raises(RuntimeError, match="Model must be prepared before finalization"): - quantizer.finalize(None, ExportBackend._TORCH) + quantizer.finalize(None, ExportBackend.Torch) def test_calibration_mode_before_prepare(self, simple_conv_linear_model, basic_config): """Test calling calibration_mode before prepare should raise error.""" @@ -1402,7 +1424,7 @@ class TestGraphModeHalfPrecisionSupport: @pytest.mark.parametrize( "backend", [ - pytest.param(ExportBackend._TORCH, id="torch"), + pytest.param(ExportBackend.Torch, id="torch"), pytest.param(ExportBackend.CoreAI, id="coreai"), pytest.param(ExportBackend.CoreML, id="coreml"), ], @@ -1429,7 +1451,7 @@ def test_prepare_calibrate_finalize_with_half_precision( finalized_model = quantizer.finalize(backend=backend) if model_dtype == torch.float32 or backend in [ - ExportBackend._TORCH, + ExportBackend.Torch, ExportBackend.CoreAI, ]: output = finalized_model(example_input) @@ -1542,7 +1564,7 @@ def test_preserved_attributes_missing_attribute_warning( @pytest.mark.parametrize( "backend", [ - pytest.param(ExportBackend._TORCH, id="torch"), + pytest.param(ExportBackend.Torch, id="torch"), pytest.param(ExportBackend.CoreAI, id="coreai"), pytest.param(ExportBackend.CoreML, id="coreml"), ], diff --git a/tests/quantization/test_graph_mode_quantizer_mnist.py b/tests/quantization/test_graph_mode_quantizer_mnist.py index a23b5fce..511bcc75 100644 --- a/tests/quantization/test_graph_mode_quantizer_mnist.py +++ b/tests/quantization/test_graph_mode_quantizer_mnist.py @@ -110,7 +110,7 @@ def test_weight_only_ptq_mnist( f"Accuracy drop too high: before={accuracy:.4f}, after={post_prepare_accuracy:.4f}" ) - finalized_model = quantizer.finalize(backend=ExportBackend._TORCH) + finalized_model = quantizer.finalize(backend=ExportBackend.Torch) finalized_accuracy = utils.eval_model(finalized_model, test_loader) # Accuracy before and after finalize should match @@ -251,7 +251,7 @@ def test_weight_and_activation_ptq_mnist( post_calibrate_accuracy = utils.eval_model(prepared_model, test_loader) assert post_calibrate_accuracy > 90.0, "Expect accuracy to climb above 90% after calibration" - finalized_model = quantizer.finalize(backend=ExportBackend._TORCH) + finalized_model = quantizer.finalize(backend=ExportBackend.Torch) finalized_accuracy = utils.eval_model(finalized_model, test_loader) # Accuracy before and after finalize should match closely. Allow a small @@ -326,7 +326,7 @@ def test_weight_and_activation_qat_mnist(mnist_pretrained_model, mnist_dataset, post_qat_accuracy = utils.eval_model(prepared_model, test_loader) assert post_qat_accuracy > 96.0, "Expect accuracy to climb above 96% after QAT" - finalized_model = quantizer.finalize(backend=ExportBackend._TORCH) + finalized_model = quantizer.finalize(backend=ExportBackend.Torch) finalized_accuracy = utils.eval_model(finalized_model, test_loader) # Accuracy before and after finalize should match diff --git a/tests/quantization/test_mmap_finalize.py b/tests/quantization/test_mmap_finalize.py index 175892e9..e1eb4741 100644 --- a/tests/quantization/test_mmap_finalize.py +++ b/tests/quantization/test_mmap_finalize.py @@ -150,7 +150,7 @@ def test_finalize_mmap_preserves_weight_sharing( @pytest.mark.parametrize("execution_mode", MODES) -@pytest.mark.parametrize("backend", [ExportBackend.CoreML, ExportBackend._TORCH]) +@pytest.mark.parametrize("backend", [ExportBackend.CoreML, ExportBackend.Torch]) def test_finalize_mmap_rejects_non_coreai_backend( execution_mode, backend, simple_linear_model, simple_linear_model_input, tmp_path ): diff --git a/tests/quantization/test_quantization.py b/tests/quantization/test_quantization.py index f369401f..d87a8596 100644 --- a/tests/quantization/test_quantization.py +++ b/tests/quantization/test_quantization.py @@ -492,14 +492,15 @@ def test_dynamic_qparams_lifecycle(self, execution_mode): [ (ExportBackend.CoreAI, False), (ExportBackend.CoreML, False), - (ExportBackend._TORCH, True), + (ExportBackend.Torch, True), ], ) def test_finalize_rejects_dynamic_for_non_torch_backends( self, execution_mode, backend, is_supported ): """``finalize`` must reject CoreAI/CoreML for dynamic FakeQuantize - modules. ``_TORCH`` is allowed since it returns the prepared model as-is.""" + modules. ``Torch`` is allowed since it keeps activation fake quantization, + so dynamic qparams are still recomputed every forward.""" config = self._make_mixed_dynamic_static_config(execution_mode) quantizer = Quantizer(SimpleLinearModel(), config) prepared_model = quantizer.prepare((torch.randn(4, 64),)) diff --git a/tests/test_joint_compression.py b/tests/test_joint_compression.py index 6bf73e53..14dc2470 100644 --- a/tests/test_joint_compression.py +++ b/tests/test_joint_compression.py @@ -79,7 +79,7 @@ def test_p4a8_compression_mnist_accuracy( ) # Finalize and verify accuracy is preserved - finalized = quantizer.finalize(backend=ExportBackend._TORCH) + finalized = quantizer.finalize(backend=ExportBackend.Torch) finalized_acc = utils.eval_model(finalized, test_loader) assert post_calib_acc == finalized_acc, ( f"Post-calibration accuracy ({post_calib_acc:.2f}%) != " diff --git a/tests/test_utils/test_torch_utils.py b/tests/test_utils/test_torch_utils.py index f13d4d2e..65ad861a 100644 --- a/tests/test_utils/test_torch_utils.py +++ b/tests/test_utils/test_torch_utils.py @@ -10,10 +10,12 @@ from coreai_torch._compression._floatx import Float4Tensor from safetensors.torch import load_file from torch import nn +from torch.nn.utils import parametrize as P from torchao.quantization.pt2e import allow_exported_model_train_eval from coreai_opt._utils.fx_utils import normalize_module_fqn from coreai_opt._utils.torch_utils import ( + fold_compression_parametrizations, mmap_module_state_dict, mmap_named_tensors, move_model_to_eval, @@ -70,6 +72,63 @@ def test_works_with_explicit_original_state(): pass +class _RoundInEval(nn.Module): + """Round to a 0.25 grid in eval mode; identity in training mode.""" + + def forward(self, tensor: torch.Tensor) -> torch.Tensor: + return tensor if self.training else torch.round(tensor * 4) / 4 + + +class _Scale(nn.Module): + def forward(self, tensor: torch.Tensor) -> torch.Tensor: + return tensor * 2 + + +class TestFoldCompressionParametrizations: + """Test fold_compression_parametrizations.""" + + @staticmethod + def test_skips_tensors_without_matching_parametrization(): + """Tensors whose parametrizations are all of other types stay parametrized.""" + model = nn.Sequential(nn.Linear(4, 4), nn.Linear(4, 4)) + P.register_parametrization(model[0], "weight", _RoundInEval()) + P.register_parametrization(model[1], "weight", _Scale()) + + assert fold_compression_parametrizations(model, _RoundInEval) == 1 + + assert not P.is_parametrized(model[0]) + assert P.is_parametrized(model[1], "weight") + + @staticmethod + def test_keeps_tie_and_folds_shared_original_once(): + """Tied tensors stay tied, and their parametrizations are applied only once.""" + first, second = nn.Linear(4, 4), nn.Linear(4, 4) + second.weight = first.weight + dense = first.weight.detach().clone() + shared = _Scale() + P.register_parametrization(first, "weight", shared) + P.register_parametrization(second, "weight", shared) + model = nn.ModuleList([first, second]) + + assert fold_compression_parametrizations(model, _Scale) == 2 + + assert first.weight is second.weight + assert torch.equal(first.weight, dense * 2) + + @staticmethod + def test_keeps_parameter_identity(): + """The folded value is written into the original parameter in place.""" + model = nn.Sequential(nn.Linear(4, 4)).eval() + P.register_parametrization(model[0], "weight", _RoundInEval()) + original = model[0].parametrizations["weight"].original + parametrized_value = model[0].weight.detach().clone() + + fold_compression_parametrizations(model, _RoundInEval) + + assert model[0].weight is original + assert torch.equal(model[0].weight, parametrized_value) + + class TestNormalizeModuleFqn: """Test normalize_module_fqn path normalization."""