From e0b44ffdeabbc8a8d5a4efcf2366e2e4a0644cbb Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Fri, 23 Oct 2026 02:15:43 -0700 Subject: [PATCH 01/20] [None][feat] modeling_v2: build target C++/CUDA as per-family libraries Native code used only by modeling_v2 targets moves out of the stock tree into cpp/tensorrt_llm/modeling_v2//, built as one MODULE library per model family (libtrtllm_mv2_.so) that is loaded only when routing picks one of the family's targets. - CMake module (cpp/cmake/modules/modelingV2.cmake): trtllm_mv2_family, trtllm_mv2_shared and trtllm_mv2_target. Each target is compiled for its declared architectures intersected with the build's, with no placeholder kernels; each gets its own torch and C++ namespace; the family library exports no symbols. The build writes modeling_v2_manifest.json. - Contract fingerprint: a hash of the stock headers in contractSurface.txt, the global definitions and the torch/CUDA versions, reported by th_common (trtllm::mv2_contract_fingerprint) and by every family library, and compared by the loader. - Loader (_native.py): load_target_ops(__name__) from a target package. A target this build or device cannot run is NoTarget; a broken installation is NativeLoadError, never a fallback. - modeling_v2_resolve_on_all_ranks: every rank reports its routing outcome and all ranks fail unless they agree; the model loader uses it. - Fixture family: one target with one op exercising build, packaging, load, fingerprint, fake op and GPU op; the template for new families. - Packaging: build_wheel.py installs the libraries and manifest and fails if a listed library is missing; setup.py ships them. - CBTS: ModelingV2Rule claims the native tree and narrows a change confined to one family to that family's test entries; the modeling_v2 unit-test entry is split per family (waivers carried over unchanged). - Claims tests for the stock/target boundary, loader and rank agreement; CODEOWNERS and the author guide (cpp/tensorrt_llm/modeling_v2/README.md). Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- .cmake-format.json | 21 + .github/CODEOWNERS | 5 + cpp/CMakeLists.txt | 8 + cpp/cmake/modules/modelingV2.cmake | 833 ++++++++++++++++++ cpp/cmake/modules/modelingV2Fingerprint.cmake | 66 ++ cpp/tensorrt_llm/CMakeLists.txt | 6 + cpp/tensorrt_llm/modeling_v2/CMakeLists.txt | 41 + cpp/tensorrt_llm/modeling_v2/README.md | 267 ++++++ .../modeling_v2/common/familyLibrary.version | 7 + .../modeling_v2/common/namespace.h | 41 + .../modeling_v2/common/torchLibrary.h | 39 + .../modeling_v2/contractSurface.txt | 22 + .../modeling_v2/fixture/CMakeLists.txt | 28 + .../scaleAddSm80Default/kernels/scaleAdd.cu | 97 ++ .../scaleAddSm80Default/kernels/scaleAdd.h | 44 + .../scaleAddSm80Default/ops/scaleAddOp.cpp | 61 ++ cpp/tensorrt_llm/thop/CMakeLists.txt | 10 +- .../thop/modelingV2ContractOp.cpp | 46 + jenkins/scripts/cbts/README.md | 4 +- jenkins/scripts/cbts/cbts/rules/README.md | 24 +- .../scripts/cbts/cbts/rules/modeling_v2.py | 321 ++++--- scripts/build_wheel.py | 36 +- setup.py | 2 + .../_experimental/modeling_v2/README.md | 15 + .../_experimental/modeling_v2/__init__.py | 2 + .../_experimental/modeling_v2/_native.py | 304 +++++++ .../modeling_v2/_router_index.py | 109 ++- .../modeling_v2/models/fixture/__init__.py | 13 + .../scale_add__sm_80__default/__init__.py | 15 + .../scale_add__sm_80__default/fake_ops.py | 20 + .../scale_add__sm_80__default/ops/__init__.py | 3 + .../ops/elementwise/__init__.py | 2 + .../ops/elementwise/scale_add.py | 56 ++ .../_torch/pyexecutor/model_loader.py | 4 +- .../test_lists/test-db/l0_b300.yml | 38 +- .../integration/test_lists/test-db/l0_cpu.yml | 3 + tests/integration/test_lists/waives.txt | 4 + .../elementwise/test_modeling_v2_scale_add.py | 67 ++ .../modeling_v2/test_modeling_v2_claims.py | 12 +- .../test_modeling_v2_native_claims.py | 413 +++++++++ .../test_modeling_v2_native_loader.py | 234 +++++ .../test_modeling_v2_rank_agreement.py | 113 +++ .../scripts/cbts/test_modeling_v2_rule.py | 168 ++++ 43 files changed, 3392 insertions(+), 232 deletions(-) create mode 100644 cpp/cmake/modules/modelingV2.cmake create mode 100644 cpp/cmake/modules/modelingV2Fingerprint.cmake create mode 100644 cpp/tensorrt_llm/modeling_v2/CMakeLists.txt create mode 100644 cpp/tensorrt_llm/modeling_v2/README.md create mode 100644 cpp/tensorrt_llm/modeling_v2/common/familyLibrary.version create mode 100644 cpp/tensorrt_llm/modeling_v2/common/namespace.h create mode 100644 cpp/tensorrt_llm/modeling_v2/common/torchLibrary.h create mode 100644 cpp/tensorrt_llm/modeling_v2/contractSurface.txt create mode 100644 cpp/tensorrt_llm/modeling_v2/fixture/CMakeLists.txt create mode 100644 cpp/tensorrt_llm/modeling_v2/fixture/scaleAddSm80Default/kernels/scaleAdd.cu create mode 100644 cpp/tensorrt_llm/modeling_v2/fixture/scaleAddSm80Default/kernels/scaleAdd.h create mode 100644 cpp/tensorrt_llm/modeling_v2/fixture/scaleAddSm80Default/ops/scaleAddOp.cpp create mode 100644 cpp/tensorrt_llm/thop/modelingV2ContractOp.cpp create mode 100644 tensorrt_llm/_torch/_experimental/modeling_v2/_native.py create mode 100644 tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/__init__.py create mode 100644 tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_80__default/__init__.py create mode 100644 tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_80__default/fake_ops.py create mode 100644 tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_80__default/ops/__init__.py create mode 100644 tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_80__default/ops/elementwise/__init__.py create mode 100644 tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_80__default/ops/elementwise/scale_add.py create mode 100644 tests/unittest/_torch/modeling_v2/models/fixture/scale_add__sm_80__default/elementwise/test_modeling_v2_scale_add.py create mode 100644 tests/unittest/_torch/modeling_v2/test_modeling_v2_native_claims.py create mode 100644 tests/unittest/_torch/modeling_v2/test_modeling_v2_native_loader.py create mode 100644 tests/unittest/_torch/modeling_v2/test_modeling_v2_rank_agreement.py create mode 100644 tests/unittest/scripts/cbts/test_modeling_v2_rule.py diff --git a/.cmake-format.json b/.cmake-format.json index e7eb73ae25fb..194be4f77f9c 100644 --- a/.cmake-format.json +++ b/.cmake-format.json @@ -25,6 +25,27 @@ "NAMESPACE": "+", "ARCHS": "+" } + }, + "trtllm_mv2_family": { + "pargs": 1 + }, + "trtllm_mv2_shared": { + "pargs": 1, + "kwargs": { + "ARCHS": "+", + "SOURCES": "+", + "DEPS": "+", + "REQUIRES": "+" + } + }, + "trtllm_mv2_target": { + "pargs": 2, + "kwargs": { + "ARCHS": "+", + "SOURCES": "+", + "DEPS": "+", + "REQUIRES": "+" + } } }, "_help_vartags": [ diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 2d06dffa3268..6bf54f3e0e2a 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -387,6 +387,11 @@ # Individual handles rather than a team, like SCAFFOLDING: this is one bounded # experiment with named owners, not a standing domain. /tensorrt_llm/_torch/_experimental/modeling_v2 @tianyuxbear @Wanli-Jiang @WeiHaocheng @litaotju +# modeling_v2 native code: the modeling_v2 owners for the targets, the kernels +# owners because it is CUDA and links against the stock libraries. +/cpp/tensorrt_llm/modeling_v2 @tianyuxbear @Wanli-Jiang @WeiHaocheng @litaotju @NVIDIA/trt-llm-kernels-devs +/cpp/cmake/modules/modelingV2.cmake @tianyuxbear @Wanli-Jiang @WeiHaocheng @litaotju @NVIDIA/trt-llm-kernels-devs +/cpp/cmake/modules/modelingV2Fingerprint.cmake @tianyuxbear @Wanli-Jiang @WeiHaocheng @litaotju @NVIDIA/trt-llm-kernels-devs /tests/unittest/_torch/modeling_v2 @tianyuxbear @Wanli-Jiang @WeiHaocheng @litaotju /tests/integration/defs/accuracy/modeling_v2 @tianyuxbear @Wanli-Jiang @WeiHaocheng @litaotju # The two catalog categories whose contracts state kernel behaviour the diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 31c58f03a236..2074f7122614 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -49,6 +49,14 @@ option(BUILD_DEEP_EP "Build the Deep EP module" ON) option(BUILD_NCCL_EXTENSIONS "Build the nccl-extensions Python wheel" ON) option(BUILD_DEEP_GEMM "Build the DeepGEMM module" ON) option(BUILD_FLASH_MLA "Build the FlashMLA module" ON) +option(BUILD_MODELING_V2 + "Build the modeling_v2 per-family native libraries (libtrtllm_mv2_*)" ON) +set(TRTLLM_MODELING_V2_TARGETS + "all" + CACHE + STRING + "modeling_v2 native targets to build: all, none, or a list of / globs" +) option(BUILD_MICRO_BENCHMARKS "Build C++ micro benchmarks" OFF) option(NVTX_DISABLE "Disable all NVTX features" ON) option(WARNING_IS_ERROR "Treat all warnings as errors" OFF) diff --git a/cpp/cmake/modules/modelingV2.cmake b/cpp/cmake/modules/modelingV2.cmake new file mode 100644 index 000000000000..9904cc64728e --- /dev/null +++ b/cpp/cmake/modules/modelingV2.cmake @@ -0,0 +1,833 @@ +# +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# + +#[[ +Native (C++/CUDA) code of modeling_v2 targets. + +Each model family under cpp/tensorrt_llm/modeling_v2// builds into +one MODULE library, libtrtllm_mv2_.so. Nothing in it is linked into +libtensorrt_llm.so or libth_common.so; the Python loader +(tensorrt_llm/_torch/_experimental/modeling_v2/_native.py) opens it only when +routing picks one of the family's targets. cpp/tensorrt_llm/modeling_v2/README.md +is the guide for authors. + +A family's CMakeLists.txt uses three functions. Names are the Python package +names (snake_case); directories are their camelCase form: + + trtllm_mv2_family() + trtllm_mv2_shared( ARCHS ... SOURCES ... [DEPS ...] [REQUIRES ...]) + trtllm_mv2_target( ARCHS ... SOURCES ... [DEPS ...] [REQUIRES ...]) + +ARCHS NN, NNa or NNf. Each is intersected with the build's requested + architectures; a target with an empty intersection is not built and + the manifest says why. No placeholder kernels are generated. +SOURCES Paths or globs relative to the target directory (shared/ for + trtllm_mv2_shared). .cu files are compiled for ARCHS without torch; + .cpp files are the torch bindings. +DEPS From a closed list: cublas, cublaslt, cuda_driver, internal_cutlass, + nccl, trtllm_gen_fmha_interface. +REQUIRES Build features: MULTI_DEVICE, NVSHMEM. Unmet requirements skip the + target. + +The stock side of the build calls trtllm_mv2_define_contract_fingerprint() +before thop, then adds cpp/tensorrt_llm/modeling_v2, which calls +trtllm_mv2_finalize() after the families. +#]] + +include_guard(GLOBAL) + +set(_TRTLLM_MV2_MODULE_DIR "${CMAKE_CURRENT_LIST_DIR}") +set(_TRTLLM_MV2_KNOWN_DEPS cublas cublaslt cuda_driver internal_cutlass nccl + trtllm_gen_fmha_interface) +set(_TRTLLM_MV2_KNOWN_REQUIRES MULTI_DEVICE NVSHMEM) +set(_TRTLLM_MV2_MANIFEST_SCHEMA 1) + +# --- Naming ------------------------------------------------------------------ + +# camelCase form of a snake_case name: minimax_m3_nvfp4__sm_107__tp2 -> +# minimaxM3Nvfp4Sm107Tp2. The same mapping as camel() in the Python checks. +function(_trtllm_mv2_camel out name) + string(REPLACE "_" ";" parts "${name}") + set(result "") + foreach(part IN LISTS parts) + if(part STREQUAL "") + continue() + endif() + if(result STREQUAL "") + set(result "${part}") + else() + string(SUBSTRING "${part}" 0 1 head) + string(SUBSTRING "${part}" 1 -1 tail) + string(TOUPPER "${head}" head) + string(APPEND result "${head}${tail}") + endif() + endforeach() + set(${out} + "${result}" + PARENT_SCOPE) +endfunction() + +# A name usable as a C++ or torch identifier: "__" collapsed to "_", since C++ +# reserves identifiers that contain a double underscore. +function(_trtllm_mv2_ident out name) + set(result "${name}") + while(result MATCHES "__") + string(REPLACE "__" "_" result "${result}") + endwhile() + set(${out} + "${result}" + PARENT_SCOPE) +endfunction() + +function(_trtllm_mv2_check_family_name family) + if(NOT family MATCHES "^[a-z][a-z0-9_]*[a-z0-9]$" OR family MATCHES "__") + message( + FATAL_ERROR + "modeling_v2: family '${family}' is not a snake_case package name") + endif() +endfunction() + +# ____, the identity the Python target directory +# carries (see tensorrt_llm/_torch/_experimental/modeling_v2/README.md). +function(_trtllm_mv2_check_target_name target) + string(REPLACE "__" ";" segments "${target}") + list(LENGTH segments count) + if(NOT count EQUAL 3 OR NOT target MATCHES "^[a-z][a-z0-9_]*$") + message( + FATAL_ERROR + "modeling_v2: target '${target}' is not a ____ name" + ) + endif() + list(GET segments 1 sm) + if(NOT sm MATCHES "^sm_[0-9]+[a-z0-9]*$") + message( + FATAL_ERROR + "modeling_v2: target '${target}' has no sm_ segment (got '${sm}')" + ) + endif() +endfunction() + +# --- Selection, requirements, architectures ----------------------------------- + +# Whether TRTLLM_MODELING_V2_TARGETS selects /. +function(_trtllm_mv2_selected out family target) + set(filter "${TRTLLM_MODELING_V2_TARGETS}") + if(filter STREQUAL "" OR filter STREQUAL "all") + set(${out} + TRUE + PARENT_SCOPE) + return() + endif() + if(filter STREQUAL "none") + set(${out} + FALSE + PARENT_SCOPE) + return() + endif() + set(selected FALSE) + foreach(pattern IN LISTS filter) + # Glob to regex: escape the dots, then * -> [^/]*, ? -> [^/]. + string(REPLACE "." "\\." regex "${pattern}") + string(REPLACE "*" "[^/]*" regex "${regex}") + string(REPLACE "?" "[^/]" regex "${regex}") + if("${family}/${target}" MATCHES "^${regex}$") + set(selected TRUE) + endif() + endforeach() + set(${out} + ${selected} + PARENT_SCOPE) +endfunction() + +# Empty when every requirement holds, otherwise the reason. +function(_trtllm_mv2_unmet_requirements out) + set(reasons "") + foreach(requirement IN LISTS ARGN) + if(NOT requirement IN_LIST _TRTLLM_MV2_KNOWN_REQUIRES) + message( + FATAL_ERROR + "modeling_v2: unknown REQUIRES '${requirement}' (known: ${_TRTLLM_MV2_KNOWN_REQUIRES})" + ) + endif() + if(requirement STREQUAL "MULTI_DEVICE" AND NOT ENABLE_MULTI_DEVICE) + list(APPEND reasons "built with ENABLE_MULTI_DEVICE=OFF") + elseif(requirement STREQUAL "NVSHMEM" AND NOT ENABLE_NVSHMEM) + list(APPEND reasons "built with ENABLE_NVSHMEM=OFF") + endif() + endforeach() + list(JOIN reasons ", " reasons) + set(${out} + "${reasons}" + PARENT_SCOPE) +endfunction() + +#[[ +Intersect declared architectures with the requested ones. + + NN -> NN-real if NN is requested + NNa -> NNa-real if NN is requested + NNf -> NNf-real if a requested architecture is in family NN: the same + major version and a minor version >= NN's + +`requested` defaults to CMAKE_CUDA_ARCHITECTURES_ORIG, the plain numbers the +build was asked for. This deliberately does not reuse set_cuda_architectures(), +which drops NNa silently and falls back to placeholder kernels. +#]] +function(_trtllm_mv2_resolve_archs out) + cmake_parse_arguments(PARSE_ARGV 1 arg "" "" "DECLARED;REQUESTED") + if(NOT DEFINED arg_REQUESTED) + set(arg_REQUESTED ${CMAKE_CUDA_ARCHITECTURES_ORIG}) + endif() + set(resolved "") + foreach(arch IN LISTS arg_DECLARED) + if(NOT arch MATCHES "^([0-9]+)([af]?)$") + message( + FATAL_ERROR "modeling_v2: ARCHS entry '${arch}' is not NN, NNa or NNf") + endif() + set(number "${CMAKE_MATCH_1}") + set(suffix "${CMAKE_MATCH_2}") + if(suffix STREQUAL "f") + math(EXPR family_major "${number} / 10") + set(in_family FALSE) + foreach(requested IN LISTS arg_REQUESTED) + math(EXPR requested_major "${requested} / 10") + if(requested_major EQUAL family_major AND requested GREATER_EQUAL + number) + set(in_family TRUE) + endif() + endforeach() + if(in_family) + list(APPEND resolved "${number}f-real") + endif() + elseif(number IN_LIST arg_REQUESTED) + list(APPEND resolved "${number}${suffix}-real") + endif() + endforeach() + list(REMOVE_DUPLICATES resolved) + set(${out} + "${resolved}" + PARENT_SCOPE) +endfunction() + +# --- Contract fingerprint +# ------------------------------------------------------ + +#[[ +Define the target `trtllm_mv2_contract_fingerprint`, which generates +${TRTLLM_MV2_GENERATED_DIR}/mv2ContractFingerprint.h. + +The fingerprint hashes what a family library is compiled against on the stock +side: the stock headers listed in cpp/tensorrt_llm/modeling_v2/contractSurface.txt, +the global compile definitions, and the torch and CUDA versions. th_common and +every family library report it, and the loader refuses a family library whose +fingerprint differs from the loaded th_common's. + +Call from cpp/tensorrt_llm/CMakeLists.txt before thop; it sets +TRTLLM_MV2_GENERATED_DIR in the caller's scope. +#]] +function(trtllm_mv2_define_contract_fingerprint) + set(generated_dir "${CMAKE_BINARY_DIR}/tensorrt_llm/modeling_v2/generated") + set(surface_file + "${PROJECT_SOURCE_DIR}/tensorrt_llm/modeling_v2/contractSurface.txt") + set_property( + DIRECTORY + APPEND + PROPERTY CMAKE_CONFIGURE_DEPENDS "${surface_file}") + + file(STRINGS "${surface_file}" patterns REGEX "^[^#]") + set(headers "") + foreach(pattern IN LISTS patterns) + string(STRIP "${pattern}" pattern) + if(pattern STREQUAL "") + continue() + endif() + set(matched_any FALSE) + foreach(root "${PROJECT_SOURCE_DIR}/include" "${PROJECT_SOURCE_DIR}") + file(GLOB matched CONFIGURE_DEPENDS "${root}/${pattern}") + if(matched) + set(matched_any TRUE) + list(APPEND headers ${matched}) + endif() + endforeach() + if(NOT matched_any) + message( + FATAL_ERROR + "modeling_v2: contract surface pattern '${pattern}' matches no header" + ) + endif() + endforeach() + list(REMOVE_DUPLICATES headers) + list(SORT headers) + + get_directory_property(definitions COMPILE_DEFINITIONS) + set(inputs "torch:${TORCH_VERSION}\ncuda:${CUDAToolkit_VERSION_MAJOR}\n") + foreach(definition IN LISTS definitions) + string(APPEND inputs "definition:${definition}\n") + endforeach() + foreach(header IN LISTS headers) + string(APPEND inputs "file:${header}\n") + endforeach() + file(WRITE "${generated_dir}/contractInputs.txt.tmp" "${inputs}") + configure_file("${generated_dir}/contractInputs.txt.tmp" + "${generated_dir}/contractInputs.txt" COPYONLY) + + set(header "${generated_dir}/mv2ContractFingerprint.h") + set(stamp "${generated_dir}/mv2ContractFingerprint.stamp") + add_custom_command( + OUTPUT "${stamp}" + BYPRODUCTS "${header}" + COMMAND + ${CMAKE_COMMAND} "-DINPUTS=${generated_dir}/contractInputs.txt" + "-DROOT=${PROJECT_SOURCE_DIR}" "-DOUTPUT=${header}" -P + "${_TRTLLM_MV2_MODULE_DIR}/modelingV2Fingerprint.cmake" + COMMAND ${CMAKE_COMMAND} -E touch "${stamp}" + DEPENDS ${headers} "${generated_dir}/contractInputs.txt" + "${_TRTLLM_MV2_MODULE_DIR}/modelingV2Fingerprint.cmake" + COMMENT "Computing the modeling_v2 contract fingerprint" + VERBATIM) + add_custom_target(trtllm_mv2_contract_fingerprint DEPENDS "${stamp}") + + set(TRTLLM_MV2_GENERATED_DIR + "${generated_dir}" + PARENT_SCOPE) +endfunction() + +# --- Families, shared code and targets ---------------------------------------- + +function(trtllm_mv2_family family) + _trtllm_mv2_check_family_name("${family}") + _trtllm_mv2_camel(camel "${family}") + get_filename_component(dir_name "${CMAKE_CURRENT_SOURCE_DIR}" NAME) + if(NOT dir_name STREQUAL camel) + message( + FATAL_ERROR + "modeling_v2: family '${family}' must live in directory '${camel}', not '${dir_name}'" + ) + endif() + get_property(families GLOBAL PROPERTY TRTLLM_MV2_FAMILIES) + if(family IN_LIST families) + message(FATAL_ERROR "modeling_v2: family '${family}' is declared twice") + endif() + set_property(GLOBAL APPEND PROPERTY TRTLLM_MV2_FAMILIES "${family}") + set_property(GLOBAL PROPERTY TRTLLM_MV2_${family}_DIR + "${CMAKE_CURRENT_SOURCE_DIR}") +endfunction() + +function(_trtllm_mv2_require_family family) + get_property(families GLOBAL PROPERTY TRTLLM_MV2_FAMILIES) + if(NOT family IN_LIST families) + message( + FATAL_ERROR + "modeling_v2: call trtllm_mv2_family(${family}) before declaring its code" + ) + endif() + get_property(family_dir GLOBAL PROPERTY TRTLLM_MV2_${family}_DIR) + if(NOT family_dir STREQUAL CMAKE_CURRENT_SOURCE_DIR) + message( + FATAL_ERROR + "modeling_v2: declare family '${family}' code in ${family_dir}/CMakeLists.txt" + ) + endif() +endfunction() + +# Expand SOURCES (paths or globs) relative to `base`. +function(_trtllm_mv2_expand_sources out base) + set(sources "") + foreach(item IN LISTS ARGN) + if(IS_ABSOLUTE "${item}") + message( + FATAL_ERROR + "modeling_v2: SOURCES must be relative to ${base}, got '${item}'") + endif() + if(item MATCHES "[*?]") + file(GLOB matched CONFIGURE_DEPENDS "${base}/${item}") + if(NOT matched) + message(FATAL_ERROR "modeling_v2: '${base}/${item}' matches no file") + endif() + list(SORT matched) + list(APPEND sources ${matched}) + else() + if(NOT EXISTS "${base}/${item}") + message(FATAL_ERROR "modeling_v2: '${base}/${item}' does not exist") + endif() + list(APPEND sources "${base}/${item}") + endif() + endforeach() + set(${out} + "${sources}" + PARENT_SCOPE) +endfunction() + +function(_trtllm_mv2_check_deps) + foreach(dep IN LISTS ARGN) + if(NOT dep IN_LIST _TRTLLM_MV2_KNOWN_DEPS) + message( + FATAL_ERROR + "modeling_v2: unknown DEPS '${dep}' (known: ${_TRTLLM_MV2_KNOWN_DEPS})" + ) + endif() + endforeach() +endfunction() + +# Compile-time usage requirements of DEPS on one OBJECT library. +function(_trtllm_mv2_apply_compile_deps object_lib) + foreach(dep IN LISTS ARGN) + if(dep STREQUAL "internal_cutlass") + target_include_directories( + ${object_lib} + PRIVATE + $ + ) + elseif(dep STREQUAL "trtllm_gen_fmha_interface") + target_link_libraries(${object_lib} PRIVATE trtllm_gen_fmha_interface) + elseif(dep STREQUAL "nccl" AND ENABLE_MULTI_DEVICE) + target_include_directories(${object_lib} PRIVATE ${MPI_C_INCLUDE_DIRS}) + endif() + endforeach() +endfunction() + +# Link-time libraries of DEPS, for the family library. +function(_trtllm_mv2_link_libraries out) + set(libraries "") + foreach(dep IN LISTS ARGN) + if(dep STREQUAL "cublas") + list(APPEND libraries ${CUBLAS_LIB}) + elseif(dep STREQUAL "cublaslt") + list(APPEND libraries ${CUBLASLT_LIB}) + elseif(dep STREQUAL "cuda_driver") + list(APPEND libraries ${CUDA_DRV_LIB}) + elseif(dep STREQUAL "nccl" AND ENABLE_MULTI_DEVICE) + list(APPEND libraries ${NCCL_LIB} ${MPI_C_LIBRARIES}) + endif() + endforeach() + set(${out} + "${libraries}" + PARENT_SCOPE) +endfunction() + +#[[ +Create the OBJECT libraries of one unit (a target, or a family's shared code): +`_kernels` for the .cu sources, compiled for `archs` without torch, and +`_ops` for the .cpp torch bindings. Both get the unit's namespaces as +compile definitions (see common/torchLibrary.h and common/namespace.h). +#]] +function( + _trtllm_mv2_add_objects + name + family + torch_ns + scope + archs + sources + deps) + set(cu_sources "") + set(cpp_sources "") + foreach(source IN LISTS sources) + if(source MATCHES "\\.cu$") + list(APPEND cu_sources "${source}") + elseif(source MATCHES "\\.cpp$") + list(APPEND cpp_sources "${source}") + elseif(NOT source MATCHES "\\.(h|cuh|hpp|inl)$") + message( + FATAL_ERROR + "modeling_v2: '${source}' is neither a .cu kernel source nor a .cpp binding" + ) + endif() + endforeach() + + set(definitions + TRTLLM_MV2_TORCH_NS=${torch_ns} TRTLLM_MV2_FAMILY_TORCH_NS=mv2_${family} + TRTLLM_MV2_SCOPE=${scope}) + set(objects "") + if(cu_sources) + add_library(${name}_kernels OBJECT ${cu_sources}) + set_target_properties( + ${name}_kernels PROPERTIES POSITION_INDEPENDENT_CODE ON CUDA_ARCHITECTURES + "${archs}") + target_compile_definitions(${name}_kernels PRIVATE ${definitions}) + _trtllm_mv2_apply_compile_deps(${name}_kernels ${deps}) + list(APPEND objects ${name}_kernels) + endif() + if(cpp_sources) + add_library(${name}_ops OBJECT ${cpp_sources}) + set_target_properties(${name}_ops PROPERTIES POSITION_INDEPENDENT_CODE ON) + target_compile_definitions(${name}_ops PRIVATE ${definitions}) + target_link_libraries(${name}_ops PRIVATE ${TORCH_LIBRARIES}) + _trtllm_mv2_apply_compile_deps(${name}_ops ${deps}) + list(APPEND objects ${name}_ops) + endif() + set_property(GLOBAL APPEND PROPERTY TRTLLM_MV2_${family}_OBJECTS ${objects}) +endfunction() + +# Record a unit's manifest entry and, when it is built, its link libraries. +function( + _trtllm_mv2_record + family + key + status + declared + archs + reason + torch_ns + deps) + set_property(GLOBAL PROPERTY TRTLLM_MV2_${family}_${key}_STATUS "${status}") + set_property(GLOBAL PROPERTY TRTLLM_MV2_${family}_${key}_DECLARED + "${declared}") + set_property(GLOBAL PROPERTY TRTLLM_MV2_${family}_${key}_ARCHS "${archs}") + set_property(GLOBAL PROPERTY TRTLLM_MV2_${family}_${key}_REASON "${reason}") + set_property(GLOBAL PROPERTY TRTLLM_MV2_${family}_${key}_NAMESPACE + "${torch_ns}") + if(status STREQUAL "built") + set_property(GLOBAL APPEND PROPERTY TRTLLM_MV2_${family}_DEPS ${deps}) + endif() +endfunction() + +function(trtllm_mv2_shared family) + cmake_parse_arguments(PARSE_ARGV 1 arg "" "" "ARCHS;SOURCES;DEPS;REQUIRES") + _trtllm_mv2_require_family("${family}") + get_property(already GLOBAL PROPERTY TRTLLM_MV2_${family}_HAS_SHARED) + if(already) + message( + FATAL_ERROR + "modeling_v2: family '${family}' declares trtllm_mv2_shared twice") + endif() + set_property(GLOBAL PROPERTY TRTLLM_MV2_${family}_HAS_SHARED TRUE) + if(NOT arg_ARCHS OR NOT arg_SOURCES) + message( + FATAL_ERROR + "modeling_v2: trtllm_mv2_shared(${family}) needs ARCHS and SOURCES") + endif() + _trtllm_mv2_check_deps(${arg_DEPS}) + if(NOT IS_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/shared") + message( + FATAL_ERROR + "modeling_v2: family '${family}' shared code lives in ${CMAKE_CURRENT_SOURCE_DIR}/shared" + ) + endif() + + # Shared code is built when a target of the family is; trtllm_mv2_finalize + # drops it otherwise. Selection and requirements still apply to it here. + _trtllm_mv2_unmet_requirements(unmet ${arg_REQUIRES}) + _trtllm_mv2_resolve_archs(archs DECLARED ${arg_ARCHS}) + if(NOT unmet STREQUAL "") + set(status "skipped") + set(reason "${unmet}") + set(archs "") + elseif(NOT archs) + set(status "skipped") + set(reason + "declares ${arg_ARCHS}; this build requests ${CMAKE_CUDA_ARCHITECTURES_ORIG}" + ) + else() + set(status "built") + set(reason "") + _trtllm_mv2_expand_sources(sources "${CMAKE_CURRENT_SOURCE_DIR}/shared" + ${arg_SOURCES}) + _trtllm_mv2_add_objects( + trtllm_mv2_${family}_shared + "${family}" + "mv2_${family}" + "${family}::shared" + "${archs}" + "${sources}" + "${arg_DEPS}") + endif() + _trtllm_mv2_record( + "${family}" + "SHARED" + "${status}" + "${arg_ARCHS}" + "${archs}" + "${reason}" + "mv2_${family}" + "${arg_DEPS}") +endfunction() + +function(trtllm_mv2_target family target) + cmake_parse_arguments(PARSE_ARGV 2 arg "" "" "ARCHS;SOURCES;DEPS;REQUIRES") + _trtllm_mv2_require_family("${family}") + _trtllm_mv2_check_target_name("${target}") + if(NOT arg_ARCHS OR NOT arg_SOURCES) + message( + FATAL_ERROR + "modeling_v2: trtllm_mv2_target(${family} ${target}) needs ARCHS and SOURCES" + ) + endif() + _trtllm_mv2_check_deps(${arg_DEPS}) + _trtllm_mv2_camel(target_dir_name "${target}") + set(target_dir "${CMAKE_CURRENT_SOURCE_DIR}/${target_dir_name}") + if(NOT IS_DIRECTORY "${target_dir}") + message( + FATAL_ERROR + "modeling_v2: target '${target}' must live in directory ${target_dir}") + endif() + + get_property(targets GLOBAL PROPERTY TRTLLM_MV2_${family}_TARGETS) + if(target IN_LIST targets) + message( + FATAL_ERROR "modeling_v2: target '${family}/${target}' is declared twice") + endif() + set_property(GLOBAL APPEND PROPERTY TRTLLM_MV2_${family}_TARGETS "${target}") + + _trtllm_mv2_ident(target_ident "${target}") + set(torch_ns "mv2_${family}_${target_ident}") + get_property(namespaces GLOBAL PROPERTY TRTLLM_MV2_NAMESPACES) + if(torch_ns IN_LIST namespaces) + message( + FATAL_ERROR + "modeling_v2: torch namespace '${torch_ns}' of ${family}/${target} is already taken" + ) + endif() + set_property(GLOBAL APPEND PROPERTY TRTLLM_MV2_NAMESPACES "${torch_ns}") + + _trtllm_mv2_selected(selected "${family}" "${target}") + _trtllm_mv2_unmet_requirements(unmet ${arg_REQUIRES}) + _trtllm_mv2_resolve_archs(archs DECLARED ${arg_ARCHS}) + if(NOT selected) + set(status "excluded") + set(reason + "not selected by TRTLLM_MODELING_V2_TARGETS=${TRTLLM_MODELING_V2_TARGETS}" + ) + set(archs "") + elseif(NOT unmet STREQUAL "") + set(status "skipped") + set(reason "${unmet}") + set(archs "") + elseif(NOT archs) + set(status "skipped") + set(reason + "declares ${arg_ARCHS}; this build requests ${CMAKE_CUDA_ARCHITECTURES_ORIG}" + ) + else() + set(status "built") + set(reason "") + _trtllm_mv2_expand_sources(sources "${target_dir}" ${arg_SOURCES}) + _trtllm_mv2_add_objects( + trtllm_mv2_${family}_${target_ident} + "${family}" + "${torch_ns}" + "${family}::${target_ident}" + "${archs}" + "${sources}" + "${arg_DEPS}") + endif() + _trtllm_mv2_record( + "${family}" + "TARGET_${target}" + "${status}" + "${arg_ARCHS}" + "${archs}" + "${reason}" + "${torch_ns}" + "${arg_DEPS}") +endfunction() + +# --- Family libraries and the manifest --------------------------------------- + +function(_trtllm_mv2_json_list out) + set(json "[]") + set(index 0) + foreach(item IN LISTS ARGN) + string(JSON json SET "${json}" ${index} "\"${item}\"") + math(EXPR index "${index} + 1") + endforeach() + set(${out} + "${json}" + PARENT_SCOPE) +endfunction() + +function(_trtllm_mv2_json_unit out family key) + get_property(status GLOBAL PROPERTY TRTLLM_MV2_${family}_${key}_STATUS) + get_property(declared GLOBAL PROPERTY TRTLLM_MV2_${family}_${key}_DECLARED) + get_property(archs GLOBAL PROPERTY TRTLLM_MV2_${family}_${key}_ARCHS) + get_property(reason GLOBAL PROPERTY TRTLLM_MV2_${family}_${key}_REASON) + get_property(torch_ns GLOBAL PROPERTY TRTLLM_MV2_${family}_${key}_NAMESPACE) + _trtllm_mv2_json_list(declared_json ${declared}) + _trtllm_mv2_json_list(archs_json ${archs}) + set(json "{}") + string(JSON json SET "${json}" status "\"${status}\"") + string(JSON json SET "${json}" namespace "\"${torch_ns}\"") + string(JSON json SET "${json}" declared_archs "${declared_json}") + string(JSON json SET "${json}" archs "${archs_json}") + string(JSON json SET "${json}" reason "\"${reason}\"") + set(${out} + "${json}" + PARENT_SCOPE) +endfunction() + +function(_trtllm_mv2_write_manifest json) + file( + GENERATE + OUTPUT + "${CMAKE_BINARY_DIR}/tensorrt_llm/modeling_v2/modeling_v2_manifest.json" + CONTENT "${json}\n") +endfunction() + +# The manifest of a build without modeling_v2 libraries. +function(trtllm_mv2_write_disabled_manifest reason) + set(json "{}") + string(JSON json SET "${json}" schema ${_TRTLLM_MV2_MANIFEST_SCHEMA}) + string(JSON json SET "${json}" enabled false) + string(JSON json SET "${json}" reason "\"${reason}\"") + string(JSON json SET "${json}" families "{}") + _trtllm_mv2_write_manifest("${json}") +endfunction() + +# The registration unit of a family library: it claims the family's torch +# namespace (with contract_fingerprint) and every built target's namespace, so a +# second library claiming any of them fails when it is loaded. +function(_trtllm_mv2_registration_source out family) + set(source + "// Generated by cpp/cmake/modules/modelingV2.cmake for family ${family}. Do not edit.\n" + ) + string(APPEND source "#include \"mv2ContractFingerprint.h\"\n\n") + string(APPEND source "#include \n#include \n\n") + string(APPEND source "namespace\n{\nstd::string contractFingerprint()\n{\n") + string(APPEND source + " return TRTLLM_MV2_CONTRACT_FINGERPRINT;\n}\n} // namespace\n\n") + string(APPEND source "TORCH_LIBRARY(mv2_${family}, m)\n{\n") + string(APPEND source " m.def(\"contract_fingerprint() -> str\");\n}\n\n") + string(APPEND source + "TORCH_LIBRARY_IMPL(mv2_${family}, CompositeExplicitAutograd, m)\n{\n") + string(APPEND source + " m.impl(\"contract_fingerprint\", &contractFingerprint);\n}\n") + foreach(torch_ns IN LISTS ARGN) + string(APPEND source + "\nTORCH_LIBRARY(${torch_ns}, m)\n{\n (void) m;\n}\n") + endforeach() + set(path "${CMAKE_CURRENT_BINARY_DIR}/${family}/registration.cpp") + file( + GENERATE + OUTPUT "${path}" + CONTENT "${source}") + set_source_files_properties("${path}" PROPERTIES GENERATED TRUE) + set(${out} + "${path}" + PARENT_SCOPE) +endfunction() + +function(_trtllm_mv2_add_family_library family) + get_property(objects GLOBAL PROPERTY TRTLLM_MV2_${family}_OBJECTS) + get_property(targets GLOBAL PROPERTY TRTLLM_MV2_${family}_TARGETS) + get_property(deps GLOBAL PROPERTY TRTLLM_MV2_${family}_DEPS) + + set(built_namespaces "") + foreach(target IN LISTS targets) + get_property(status GLOBAL + PROPERTY TRTLLM_MV2_${family}_TARGET_${target}_STATUS) + if(status STREQUAL "built") + get_property(torch_ns GLOBAL + PROPERTY TRTLLM_MV2_${family}_TARGET_${target}_NAMESPACE) + list(APPEND built_namespaces "${torch_ns}") + endif() + endforeach() + + # Shared code without a built target has nothing to serve. + get_property(shared_status GLOBAL PROPERTY TRTLLM_MV2_${family}_SHARED_STATUS) + if(NOT built_namespaces) + if(shared_status STREQUAL "built") + set_property(GLOBAL PROPERTY TRTLLM_MV2_${family}_SHARED_STATUS "skipped") + set_property(GLOBAL PROPERTY TRTLLM_MV2_${family}_SHARED_REASON + "no target of the family is built") + set_property(GLOBAL PROPERTY TRTLLM_MV2_${family}_SHARED_ARCHS "") + endif() + set_property(GLOBAL PROPERTY TRTLLM_MV2_${family}_LIBRARY "") + return() + endif() + + set(library trtllm_mv2_${family}) + _trtllm_mv2_registration_source(registration "${family}" ${built_namespaces}) + set(object_sources "") + foreach(object IN LISTS objects) + list(APPEND object_sources $) + endforeach() + add_library(${library} MODULE "${registration}" ${object_sources}) + add_dependencies(${library} trtllm_mv2_contract_fingerprint) + target_include_directories(${library} PRIVATE "${TRTLLM_MV2_GENERATED_DIR}") + list(REMOVE_DUPLICATES deps) + _trtllm_mv2_link_libraries(dep_libraries ${deps}) + target_link_libraries( + ${library} PRIVATE ${TORCH_LIBRARIES} th_utils ${SHARED_TARGET} + ${CUDA_RT_LIB} ${CUDA_DRV_LIB} ${dep_libraries}) + set(version_script + "${PROJECT_SOURCE_DIR}/tensorrt_llm/modeling_v2/common/familyLibrary.version" + ) + set_target_properties( + ${library} + PROPERTIES OUTPUT_NAME ${library} + PREFIX "lib" + LIBRARY_OUTPUT_DIRECTORY + "${CMAKE_BINARY_DIR}/tensorrt_llm/modeling_v2" + POSITION_INDEPENDENT_CODE ON + LINKER_LANGUAGE CXX + LINK_DEPENDS "${version_script}" + BUILD_RPATH "$ORIGIN;$ORIGIN/../../nvidia/nccl/lib") + target_link_options( + ${library} PRIVATE "-Wl,--version-script=${version_script}" + "-Wl,--no-undefined") + add_dependencies(modeling_v2_libs ${library}) + set_property(GLOBAL PROPERTY TRTLLM_MV2_${family}_LIBRARY "lib${library}.so") +endfunction() + +# Build every declared family's library and write the manifest. Called by +# cpp/tensorrt_llm/modeling_v2/CMakeLists.txt after it added the families. +function(trtllm_mv2_finalize) + get_property(families GLOBAL PROPERTY TRTLLM_MV2_FAMILIES) + _trtllm_mv2_json_list(requested_json ${CMAKE_CUDA_ARCHITECTURES_ORIG}) + set(json "{}") + string(JSON json SET "${json}" schema ${_TRTLLM_MV2_MANIFEST_SCHEMA}) + string(JSON json SET "${json}" enabled true) + string(JSON json SET "${json}" requested_archs "${requested_json}") + string(JSON json SET "${json}" families "{}") + + foreach(family IN LISTS families) + _trtllm_mv2_add_family_library("${family}") + get_property(library GLOBAL PROPERTY TRTLLM_MV2_${family}_LIBRARY) + get_property(targets GLOBAL PROPERTY TRTLLM_MV2_${family}_TARGETS) + get_property(has_shared GLOBAL PROPERTY TRTLLM_MV2_${family}_HAS_SHARED) + + set(family_json "{}") + if(library STREQUAL "") + string(JSON family_json SET "${family_json}" library null) + else() + string(JSON family_json SET "${family_json}" library "\"${library}\"") + endif() + string(JSON family_json SET "${family_json}" namespace "\"mv2_${family}\"") + if(has_shared) + _trtllm_mv2_json_unit(shared_json "${family}" "SHARED") + string(JSON family_json SET "${family_json}" shared "${shared_json}") + else() + string(JSON family_json SET "${family_json}" shared null) + endif() + set(targets_json "{}") + foreach(target IN LISTS targets) + _trtllm_mv2_json_unit(target_json "${family}" "TARGET_${target}") + string(JSON targets_json SET "${targets_json}" "${target}" + "${target_json}") + endforeach() + string(JSON family_json SET "${family_json}" targets "${targets_json}") + string( + JSON + json + SET + "${json}" + families + "${family}" + "${family_json}") + endforeach() + _trtllm_mv2_write_manifest("${json}") +endfunction() diff --git a/cpp/cmake/modules/modelingV2Fingerprint.cmake b/cpp/cmake/modules/modelingV2Fingerprint.cmake new file mode 100644 index 000000000000..e7860cf1c897 --- /dev/null +++ b/cpp/cmake/modules/modelingV2Fingerprint.cmake @@ -0,0 +1,66 @@ +# +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# + +#[[ +Write the modeling_v2 contract fingerprint header (build step of +trtllm_mv2_define_contract_fingerprint in modelingV2.cmake). + + cmake -DINPUTS= -DROOT= -DOUTPUT=
+ -P modelingV2Fingerprint.cmake + +INPUTS has one entry per line: `file:` lines are hashed by +content under their path relative to ROOT, every other line is hashed as is. +The header is rewritten only when the fingerprint changes, so an unchanged +contract recompiles nothing. +#]] + +foreach(required INPUTS ROOT OUTPUT) + if(NOT DEFINED ${required}) + message( + FATAL_ERROR "modelingV2Fingerprint.cmake: -D${required} is required") + endif() +endforeach() + +file(STRINGS "${INPUTS}" lines) +set(digest_input "") +foreach(line IN LISTS lines) + if(line MATCHES "^file:(.*)$") + set(path "${CMAKE_MATCH_1}") + file(SHA256 "${path}" content_digest) + file(RELATIVE_PATH relative "${ROOT}" "${path}") + string(APPEND digest_input "${relative} ${content_digest}\n") + else() + string(APPEND digest_input "${line}\n") + endif() +endforeach() +string(SHA256 fingerprint "${digest_input}") +string(SUBSTRING "${fingerprint}" 0 16 fingerprint) + +set(content + "// Generated by cpp/cmake/modules/modelingV2Fingerprint.cmake. Do not edit.\n" +) +string(APPEND content "#pragma once\n\n") +string(APPEND content + "#define TRTLLM_MV2_CONTRACT_FINGERPRINT \"${fingerprint}\"\n") + +if(EXISTS "${OUTPUT}") + file(READ "${OUTPUT}" previous) + if(previous STREQUAL content) + return() + endif() +endif() +file(WRITE "${OUTPUT}" "${content}") diff --git a/cpp/tensorrt_llm/CMakeLists.txt b/cpp/tensorrt_llm/CMakeLists.txt index 8fa6c33c8eee..5b06fc841510 100644 --- a/cpp/tensorrt_llm/CMakeLists.txt +++ b/cpp/tensorrt_llm/CMakeLists.txt @@ -340,7 +340,13 @@ if(NOT WIN32) endif() if(BUILD_PYT) + # modeling_v2 family libraries (cpp/tensorrt_llm/modeling_v2) are separate + # modules loaded on demand; th_common reports the contract fingerprint they + # are checked against, so it is defined before thop. + include(modelingV2) + trtllm_mv2_define_contract_fingerprint() add_subdirectory(thop) + add_subdirectory(modeling_v2) endif() add_subdirectory(nanobind) diff --git a/cpp/tensorrt_llm/modeling_v2/CMakeLists.txt b/cpp/tensorrt_llm/modeling_v2/CMakeLists.txt new file mode 100644 index 000000000000..a38827529a57 --- /dev/null +++ b/cpp/tensorrt_llm/modeling_v2/CMakeLists.txt @@ -0,0 +1,41 @@ +# +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# + +# Native code of modeling_v2 targets: one library per model family, loaded only +# when routing picks one of its targets. Every subdirectory with a +# CMakeLists.txt is a family; see README.md and +# cpp/cmake/modules/modelingV2.cmake. + +add_custom_target(modeling_v2_libs) + +if(WIN32) + trtllm_mv2_write_disabled_manifest( + "modeling_v2 libraries are not built on Windows") +elseif(NOT BUILD_MODELING_V2) + trtllm_mv2_write_disabled_manifest("built with BUILD_MODELING_V2=OFF") +else() + file( + GLOB family_lists + LIST_DIRECTORIES false + CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/*/CMakeLists.txt") + list(SORT family_lists) + foreach(family_list IN LISTS family_lists) + get_filename_component(family_dir "${family_list}" DIRECTORY) + add_subdirectory("${family_dir}") + endforeach() + trtllm_mv2_finalize() +endif() diff --git a/cpp/tensorrt_llm/modeling_v2/README.md b/cpp/tensorrt_llm/modeling_v2/README.md new file mode 100644 index 000000000000..c8e37cb6b836 --- /dev/null +++ b/cpp/tensorrt_llm/modeling_v2/README.md @@ -0,0 +1,267 @@ + + + +# modeling_v2 native code + +C++/CUDA used only by modeling_v2 targets +(`tensorrt_llm/_torch/_experimental/modeling_v2/`) lives here, not in +`cpp/tensorrt_llm/kernels/` or `thop/`. Each model family builds into one +library, `libtrtllm_mv2_.so`. + +- **Not linked into stock.** Nothing in the family library is linked into + `libtensorrt_llm.so` or `libth_common.so`. +- **Loaded on demand.** Python loads the family library only when routing + picks one of the family's targets. With `modeling_v2=off` (the default), + nothing from here is loaded. + +This document is the guide for adding native code to a target. The family +`fixture/` is the smallest complete example: one target with one op. Copy it. + +## Layout + +``` +cpp/tensorrt_llm/modeling_v2/ + CMakeLists.txt adds every family directory (the only stock hook) + contractSurface.txt the stock headers native code may include + common/ namespace and torch-registration macros + / one model family -> libtrtllm_mv2_.so + CMakeLists.txt trtllm_mv2_family / trtllm_mv2_shared / trtllm_mv2_target + shared/ code more than one target of the family calls + / one target + kernels/ *.cu *.cuh *.h -- compiled for the target's ARCHS, no torch + ops/ *.cpp torch bindings +``` + +### Naming + +**Names are the Python package names.** A family is `minimax_m3`; a target is +`minimax_m3_nvfp4__sm_107__tp2`. + +**Directories are their camelCase form**, following `CODING_GUIDELINES.md`: +`minimaxM3/minimaxM3Nvfp4Sm107Tp2/`. + +The build, the loader and CI compute the same mapping, and +`test_modeling_v2_native_claims.py` checks that it holds in both directions. + +## Adding native code to a target + +The example below adds an op to target `foo_nvfp4__sm_103__tp4` of family +`foo`. + +1. **Declare it.** The declaration goes in `cpp/tensorrt_llm/modeling_v2/foo/CMakeLists.txt`: + + ```cmake + trtllm_mv2_family(foo) + + trtllm_mv2_target( + foo foo_nvfp4__sm_103__tp4 + ARCHS 103a + SOURCES kernels/*.cu ops/*.cpp + REQUIRES MULTI_DEVICE) + ``` + +2. **Write the kernel.** It goes in `fooNvfp4Sm103Tp4/kernels/`, wrapped in the + target's namespace: + + ```cpp + #include "tensorrt_llm/modeling_v2/common/namespace.h" + + TRTLLM_MV2_NAMESPACE_BEGIN // tensorrt_llm::modeling_v2::foo::foo_nvfp4_sm_103_tp4 + namespace kernels + { + cudaError_t invokeMyOp(/* ... */, cudaStream_t stream); + } + TRTLLM_MV2_NAMESPACE_END + ``` + +3. **Bind it to torch** in `fooNvfp4Sm103Tp4/ops/`, through the target's own torch namespace: + + ```cpp + #include "tensorrt_llm/modeling_v2/common/torchLibrary.h" + + TRTLLM_MV2_LIBRARY_FRAGMENT(m) { m.def("my_op(Tensor x) -> Tensor"); } + TRTLLM_MV2_LIBRARY_IMPL(CUDA, m) { m.impl("my_op", &TRTLLM_MV2_NS::myOp); } + ``` + +4. **Load it** from the Python target package, + `models/foo/foo_nvfp4__sm_103__tp4/__init__.py`: + + ```python + from tensorrt_llm._torch._experimental.modeling_v2._native import load_target_ops + + native_ops = load_target_ops(__name__) # torch.ops.mv2_foo_foo_nvfp4_sm_103_tp4 + ``` + + The name is not `ops`: that is the target's `ops/` package, which would + replace the attribute when it is imported. + +5. **Fake every op** in the target's `fake_ops.py`. The loader imports it right + after opening the library: + + ```python + from tensorrt_llm._torch._experimental.modeling_v2._native import namespace_of + + _NS = namespace_of(__package__) + + @torch.library.register_fake(f"{_NS}::my_op") + def _(x): + return torch.empty_like(x) + ``` + +6. **Wrap it in an entry**, `ops//my_op.py`, which calls + `native_ops.my_op(...)`. Test the entry like any other catalog entry, under + `tests/unittest/_torch/modeling_v2/models/foo//`. + +7. **List the tests.** + - The family's unit tests go in a test-db entry of its own: + `unittest/_torch/modeling_v2/models/foo`. + - Its accuracy gates go in `accuracy/modeling_v2/test_foo.py`. + + CBTS selects a family's tests by these two names (see [CI](#ci)). + +No stock file changes along the way. + +## CMake API + +The functions are defined in `cpp/cmake/modules/modelingV2.cmake`. + +| Function | What it declares | +|---|---| +| `trtllm_mv2_family()` | The family; call it first in the family's `CMakeLists.txt`. | +| `trtllm_mv2_target( ARCHS ... SOURCES ... [DEPS ...] [REQUIRES ...])` | One target's code, in `/`. | +| `trtllm_mv2_shared( ARCHS ... SOURCES ... [DEPS ...] [REQUIRES ...])` | Code several targets of the family call, in `shared/`. Its ops are in `torch.ops.mv2_`. | + +| Keyword | Meaning | +|---|---| +| `ARCHS` | `NN`, `NNa` or `NNf`, intersected with the build's requested architectures: `NN`/`NNa` build when `NN` is requested; `NNf` builds when any member of family `NN` is. A target whose intersection is empty is **not built**: no placeholder kernels, and the manifest records why. The declared architectures must cover the SM in the target's name; when moving existing code, keep the architectures it was gated with. | +| `SOURCES` | Paths or globs relative to the unit's directory. `.cu` files are compiled for `ARCHS` without torch; `.cpp` files are the torch bindings. | +| `DEPS` | `cublas`, `cublaslt`, `cuda_driver`, `internal_cutlass`, `nccl`, `trtllm_gen_fmha_interface`. Ask for anything else by extending the module. | +| `REQUIRES` | `MULTI_DEVICE`, `NVSHMEM`. A target whose requirement the build lacks (e.g. a single-device build) is skipped and recorded. | + +### Build options + +- **`-DBUILD_MODELING_V2=OFF`** builds no family library. The manifest then + says so, and every target with native code is a `NoTarget`. +- **`-DTRTLLM_MODELING_V2_TARGETS=/;...`** builds only the + matching targets. Globs are allowed, e.g. `fixture/*`; the default is + `all`. Pass it through + `scripts/build_wheel.py --extra-cmake-vars TRTLLM_MODELING_V2_TARGETS=...`. +- **Incremental builds.** A family library is its own build target, so an + edit relinks only that library: + + ```bash + cmake --build cpp/build --target trtllm_mv2_ + ``` + +## Namespaces + +| Kind | Name | +|---|---| +| Target torch ops | `torch.ops.mv2__`, with `__` collapsed to `_` | +| Family shared torch ops | `torch.ops.mv2_`; also holds `contract_fingerprint()` | +| Target C++ | `tensorrt_llm::modeling_v2::::` (`TRTLLM_MV2_NAMESPACE_BEGIN`) | +| Shared C++ | `tensorrt_llm::modeling_v2::::shared` | + +**Each torch namespace is claimed exactly once.** The family library's +generated registration unit claims it with `TORCH_LIBRARY`, so a second +library claiming the same namespace fails at load. Op files only add fragments, +through the `TRTLLM_MV2_*` macros; raw `TORCH_LIBRARY*` is rejected by the +claims test. + +**Sibling targets can share file and function names.** Every target has its +own C++ namespace, and the family library exports no symbol +(`common/familyLibrary.version`). + +## Sharing code + +**Sharing moves code up, never sideways:** + +1. target +2. family `shared/` +3. stock `cpp/tensorrt_llm/kernels` + `thop`, once a built-in model calls it + (normal stock review) + +A target never includes or calls a sibling target's code. When two targets +need the same kernel, move it to `shared/`. A cross-family `catalog/` is +reserved and not built yet. + +## What native code may include + +- Its own target's files. +- Its family's `shared/`. +- `common/`. +- The stock headers listed in `contractSurface.txt`, such as + `tensorrt_llm/common/*` and selected kernel device helpers like + `kernels/quantization.cuh`. +- System, CUDA, torch and cutlass headers. + +It links against `libtensorrt_llm.so` (logger, `TllmException`, mcast +registry and other singletons stay there), `th_utils` and torch. + +**The contract fingerprint.** `contractSurface.txt` is also the stock side of +the contract. The build hashes those headers, the global compile definitions, +and the torch and CUDA versions into a *contract fingerprint*: + +- `libth_common.so` reports it as `torch.ops.trtllm.mv2_contract_fingerprint()`. +- Each family library reports it as `torch.ops.mv2_.contract_fingerprint()`. +- The loader refuses a family library whose fingerprint differs from the + loaded `libth_common.so`'s. + +Adding a header to the list widens what stock changes can affect targets; the +header's stock owners review it. + +**When a target needs stock to change**, for example a new option on a stock +kernel, that change is a separate stock PR. It must leave stock behavior +unchanged by default, and it is reviewed by the stock owners. + +## How it loads + +`_native.load_target_ops(__name__)` runs these steps: + +1. Read `tensorrt_llm/libs/modeling_v2_manifest.json`. +2. Check that the target was built, and built for this GPU. +3. Open the family library. +4. Compare contract fingerprints. +5. Import the target's `fake_ops.py`. + +| Outcome | Raised as | Meaning | +|---|---|---| +| The target was skipped or excluded by this build, or does not run on this GPU | `NoTarget` | `auto` keeps the built-in model; `require` reports why | +| The manifest or library is missing, `dlopen` fails, the fingerprint differs, or an op is missing | `NativeLoadError` | A broken installation; never a fallback | + +`route` imports a target only on the branch that returns it, so this runs as +part of routing. Every rank then agrees on the result +(`modeling_v2_resolve_on_all_ranks`), or every rank fails. + +`_native.available_targets()` lists what the installed wheel supports. + +## CI + +**CBTS rule.** `jenkins/scripts/cbts/cbts/rules/modeling_v2.py` claims both +trees. What it selects depends on what the PR changes: + +| PR changes only… | CI runs… | +|---|---| +| One family (`models//` and/or `cpp/tensorrt_llm/modeling_v2//`) | That family's test-db entries plus the modeling_v2 CPU checks | +| Shared modeling_v2 code (catalog, loader, resolver, `common/`, the CMake module, `contractSurface.txt`) | Every modeling_v2 entry | +| Stock code | The full suite | + +**Why the narrowing is safe.** It relies on these CPU checks, which run in +every one of those scopes: + +- `test_modeling_v2_native_claims.py`: + - stock never references a target; + - targets reach only themselves, their family's `shared/`, `common/` and + the contract surface; + - every op has a fake; + - `modeling_v2` defaults to `off`; + - families keep their own test-db entries. +- `test_modeling_v2_native_loader.py`: the loader's decisions, and a load + smoke test of every family library the wheel ships. +- `test_modeling_v2_rank_agreement.py`: all ranks resolve to the same target, + or all of them fail. + +**Pre-merge entries.** Every pre-merge modeling_v2 entry also runs on every +stock PR that falls back to the full suite. Keep a family's pre-merge entry +to one short smoke test on hardware matching its SM, and put the accuracy +matrix post-merge. diff --git a/cpp/tensorrt_llm/modeling_v2/common/familyLibrary.version b/cpp/tensorrt_llm/modeling_v2/common/familyLibrary.version new file mode 100644 index 000000000000..9feababdfb1a --- /dev/null +++ b/cpp/tensorrt_llm/modeling_v2/common/familyLibrary.version @@ -0,0 +1,7 @@ +/* A modeling_v2 family library exports nothing: its torch ops register from + * static initializers when the library is loaded, and keeping every symbol + * local lets two family libraries define the same C++ names in one process. + * See cpp/cmake/modules/modelingV2.cmake. */ +{ + local: *; +}; diff --git a/cpp/tensorrt_llm/modeling_v2/common/namespace.h b/cpp/tensorrt_llm/modeling_v2/common/namespace.h new file mode 100644 index 000000000000..397bccb42317 --- /dev/null +++ b/cpp/tensorrt_llm/modeling_v2/common/namespace.h @@ -0,0 +1,41 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "tensorrt_llm/common/config.h" + +// trtllm_mv2_target() and trtllm_mv2_shared() define TRTLLM_MV2_SCOPE as +// `::` or `::shared` (cpp/cmake/modules/modelingV2.cmake). +#ifndef TRTLLM_MV2_SCOPE +#error "modeling_v2 native code is compiled through trtllm_mv2_target() or trtllm_mv2_shared()" +#endif + +//! Opens tensorrt_llm::modeling_v2:::: (or ::shared) for the unit +//! being compiled. Every target has its own C++ namespace, so sibling targets can +//! carry identically named code in one family library. +#define TRTLLM_MV2_NAMESPACE_BEGIN \ + TRTLLM_NAMESPACE_BEGIN \ + namespace modeling_v2::TRTLLM_MV2_SCOPE \ + { + +#define TRTLLM_MV2_NAMESPACE_END \ + } \ + TRTLLM_NAMESPACE_END + +//! The fully qualified namespace TRTLLM_MV2_NAMESPACE_BEGIN opens. +#define TRTLLM_MV2_NS ::tensorrt_llm::modeling_v2::TRTLLM_MV2_SCOPE diff --git a/cpp/tensorrt_llm/modeling_v2/common/torchLibrary.h b/cpp/tensorrt_llm/modeling_v2/common/torchLibrary.h new file mode 100644 index 000000000000..64e455da6268 --- /dev/null +++ b/cpp/tensorrt_llm/modeling_v2/common/torchLibrary.h @@ -0,0 +1,39 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +// trtllm_mv2_target() defines TRTLLM_MV2_TORCH_NS as mv2__ and +// trtllm_mv2_shared() as mv2_ (cpp/cmake/modules/modelingV2.cmake). The +// family library claims each namespace once with TORCH_LIBRARY, so op files only +// ever add fragments to it. +#ifndef TRTLLM_MV2_TORCH_NS +#error "modeling_v2 torch bindings are compiled through trtllm_mv2_target() or trtllm_mv2_shared()" +#endif + +// One level of indirection so that TORCH_LIBRARY_*'s own # and ## operators see +// the expanded namespace rather than the macro name. +#define TRTLLM_MV2_DETAIL_LIBRARY_FRAGMENT(ns, m) TORCH_LIBRARY_FRAGMENT(ns, m) +#define TRTLLM_MV2_DETAIL_LIBRARY_IMPL(ns, k, m) TORCH_LIBRARY_IMPL(ns, k, m) + +//! TORCH_LIBRARY_FRAGMENT on this unit's own namespace: declare op schemas. +#define TRTLLM_MV2_LIBRARY_FRAGMENT(m) TRTLLM_MV2_DETAIL_LIBRARY_FRAGMENT(TRTLLM_MV2_TORCH_NS, m) + +//! TORCH_LIBRARY_IMPL on this unit's own namespace: register kernels for dispatch key `k`. +#define TRTLLM_MV2_LIBRARY_IMPL(k, m) TRTLLM_MV2_DETAIL_LIBRARY_IMPL(TRTLLM_MV2_TORCH_NS, k, m) diff --git a/cpp/tensorrt_llm/modeling_v2/contractSurface.txt b/cpp/tensorrt_llm/modeling_v2/contractSurface.txt new file mode 100644 index 000000000000..c55f822396ef --- /dev/null +++ b/cpp/tensorrt_llm/modeling_v2/contractSurface.txt @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# The stock headers modeling_v2 native code may include, as written in #include +# (resolved against cpp/include and cpp/). This list is the contract between +# stock and the family libraries: +# +# * cpp/cmake/modules/modelingV2.cmake hashes these headers into the contract +# fingerprint that th_common and every family library report, and the +# loader refuses a family library whose fingerprint differs; +# * tests/unittest/_torch/modeling_v2/test_modeling_v2_native_claims.py +# rejects any other stock include under cpp/tensorrt_llm/modeling_v2. +# +# Adding a header widens what stock changes can affect modeling_v2 targets; +# it is reviewed by the stock owners of that header. +tensorrt_llm/common/*.h +tensorrt_llm/common/*.cuh +tensorrt_llm/kernels/kvCacheUtils.h +tensorrt_llm/kernels/quantization.cuh +tensorrt_llm/kernels/quantization.h +tensorrt_llm/runtime/mcastDeviceMemory.h +tensorrt_llm/thop/thUtils.h diff --git a/cpp/tensorrt_llm/modeling_v2/fixture/CMakeLists.txt b/cpp/tensorrt_llm/modeling_v2/fixture/CMakeLists.txt new file mode 100644 index 000000000000..9e48b10dafc7 --- /dev/null +++ b/cpp/tensorrt_llm/modeling_v2/fixture/CMakeLists.txt @@ -0,0 +1,28 @@ +# +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# + +# The infrastructure fixture: one target with one op, so that CI drives the +# whole native path -- build, packaging, load, contract fingerprint, fake op, +# GPU op -- on every GPU it has. It is also the smallest complete family to copy +# (see ../README.md). Its architectures cover every GPU CI runs on; a real +# target declares the architectures it was written and gated for. +trtllm_mv2_family(fixture) + +trtllm_mv2_target( + fixture scale_add__sm_80__default + ARCHS 80 90 100f 120f + SOURCES kernels/scaleAdd.cu ops/scaleAddOp.cpp) diff --git a/cpp/tensorrt_llm/modeling_v2/fixture/scaleAddSm80Default/kernels/scaleAdd.cu b/cpp/tensorrt_llm/modeling_v2/fixture/scaleAddSm80Default/kernels/scaleAdd.cu new file mode 100644 index 000000000000..6c5404426ad8 --- /dev/null +++ b/cpp/tensorrt_llm/modeling_v2/fixture/scaleAddSm80Default/kernels/scaleAdd.cu @@ -0,0 +1,97 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "tensorrt_llm/modeling_v2/fixture/scaleAddSm80Default/kernels/scaleAdd.h" + +#include +#include + +TRTLLM_MV2_NAMESPACE_BEGIN + +namespace kernels +{ + +namespace +{ + +constexpr int kThreads = 256; +constexpr int64_t kMaxBlocks = 65535; + +__device__ __forceinline__ float toFloat(float value) +{ + return value; +} + +__device__ __forceinline__ float toFloat(__nv_bfloat16 value) +{ + return __bfloat162float(value); +} + +template +__device__ __forceinline__ T fromFloat(float value); + +template <> +__device__ __forceinline__ float fromFloat(float value) +{ + return value; +} + +template <> +__device__ __forceinline__ __nv_bfloat16 fromFloat<__nv_bfloat16>(float value) +{ + return __float2bfloat16_rn(value); +} + +template +__global__ void scaleAddKernel(T const* x, T* out, int64_t numel, float alpha, float beta) +{ + int64_t const stride = static_cast(gridDim.x) * blockDim.x; + for (int64_t i = static_cast(blockIdx.x) * blockDim.x + threadIdx.x; i < numel; i += stride) + { + out[i] = fromFloat(__fadd_rn(__fmul_rn(toFloat(x[i]), alpha), beta)); + } +} + +template +cudaError_t launchScaleAdd(void const* x, void* out, int64_t numel, float alpha, float beta, cudaStream_t stream) +{ + int64_t const blocks = std::min((numel + kThreads - 1) / kThreads, kMaxBlocks); + scaleAddKernel<<(blocks), kThreads, 0, stream>>>( + static_cast(x), static_cast(out), numel, alpha, beta); + return cudaGetLastError(); +} + +} // namespace + +cudaError_t invokeScaleAdd( + void const* x, void* out, int64_t numel, ScaleAddDtype dtype, float alpha, float beta, cudaStream_t stream) +{ + if (numel == 0) + { + return cudaSuccess; + } + switch (dtype) + { + case ScaleAddDtype::kFloat32: return launchScaleAdd(x, out, numel, alpha, beta, stream); + case ScaleAddDtype::kBfloat16: return launchScaleAdd<__nv_bfloat16>(x, out, numel, alpha, beta, stream); + } + return cudaErrorInvalidValue; +} + +} // namespace kernels + +TRTLLM_MV2_NAMESPACE_END diff --git a/cpp/tensorrt_llm/modeling_v2/fixture/scaleAddSm80Default/kernels/scaleAdd.h b/cpp/tensorrt_llm/modeling_v2/fixture/scaleAddSm80Default/kernels/scaleAdd.h new file mode 100644 index 000000000000..9a093e7b544e --- /dev/null +++ b/cpp/tensorrt_llm/modeling_v2/fixture/scaleAddSm80Default/kernels/scaleAdd.h @@ -0,0 +1,44 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "tensorrt_llm/modeling_v2/common/namespace.h" + +#include +#include + +TRTLLM_MV2_NAMESPACE_BEGIN + +namespace kernels +{ + +enum class ScaleAddDtype +{ + kFloat32, + kBfloat16, +}; + +//! out[i] = x[i] * alpha + beta over `numel` contiguous elements of `dtype`. The product and the sum are each +//! rounded to fp32 (no fused multiply-add), then the result is rounded once to `dtype`, which is what +//! `(x.float() * alpha + beta).to(dtype)` computes in torch. +cudaError_t invokeScaleAdd( + void const* x, void* out, int64_t numel, ScaleAddDtype dtype, float alpha, float beta, cudaStream_t stream); + +} // namespace kernels + +TRTLLM_MV2_NAMESPACE_END diff --git a/cpp/tensorrt_llm/modeling_v2/fixture/scaleAddSm80Default/ops/scaleAddOp.cpp b/cpp/tensorrt_llm/modeling_v2/fixture/scaleAddSm80Default/ops/scaleAddOp.cpp new file mode 100644 index 000000000000..1abf21f025da --- /dev/null +++ b/cpp/tensorrt_llm/modeling_v2/fixture/scaleAddSm80Default/ops/scaleAddOp.cpp @@ -0,0 +1,61 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "tensorrt_llm/common/assert.h" +#include "tensorrt_llm/modeling_v2/common/namespace.h" +#include "tensorrt_llm/modeling_v2/common/torchLibrary.h" +#include "tensorrt_llm/modeling_v2/fixture/scaleAddSm80Default/kernels/scaleAdd.h" + +#include +#include +#include + +TRTLLM_MV2_NAMESPACE_BEGIN + +namespace +{ + +at::Tensor scaleAdd(at::Tensor const& x, double alpha, double beta) +{ + TORCH_CHECK(x.is_cuda() && x.is_contiguous(), "scale_add: x must be a contiguous CUDA tensor"); + TORCH_CHECK(x.scalar_type() == at::kFloat || x.scalar_type() == at::kBFloat16, + "scale_add: x must be float32 or bfloat16, got ", x.scalar_type()); + c10::cuda::CUDAGuard const deviceGuard{x.device()}; + at::Tensor out = at::empty_like(x); + auto const dtype + = x.scalar_type() == at::kFloat ? kernels::ScaleAddDtype::kFloat32 : kernels::ScaleAddDtype::kBfloat16; + cudaError_t const status = kernels::invokeScaleAdd(x.data_ptr(), out.data_ptr(), x.numel(), dtype, + static_cast(alpha), static_cast(beta), at::cuda::getCurrentCUDAStream(x.get_device())); + // TLLM_CHECK_WITH_INFO throws a TllmException, whose code lives in libtensorrt_llm.so: the fixture keeps the + // dynamic link to the stock library exercised. + TLLM_CHECK_WITH_INFO(status == cudaSuccess, "scale_add launch failed: %s", cudaGetErrorString(status)); + return out; +} + +} // namespace + +TRTLLM_MV2_NAMESPACE_END + +TRTLLM_MV2_LIBRARY_FRAGMENT(m) +{ + m.def("scale_add(Tensor x, float alpha, float beta) -> Tensor"); +} + +TRTLLM_MV2_LIBRARY_IMPL(CUDA, m) +{ + m.impl("scale_add", &TRTLLM_MV2_NS::scaleAdd); +} diff --git a/cpp/tensorrt_llm/thop/CMakeLists.txt b/cpp/tensorrt_llm/thop/CMakeLists.txt index fa7c0a458fa2..4108cc8f76d5 100644 --- a/cpp/tensorrt_llm/thop/CMakeLists.txt +++ b/cpp/tensorrt_llm/thop/CMakeLists.txt @@ -171,6 +171,13 @@ add_th_common_family( # model. add_subdirectory(glmMoeDsa) +# trtllm::mv2_contract_fingerprint: the stock side of the check the modeling_v2 +# loader makes before it uses a family library (cpp/tensorrt_llm/modeling_v2). +add_th_common_family(th_common_modeling_v2 modelingV2ContractOp.cpp) +add_dependencies(th_common_modeling_v2 trtllm_mv2_contract_fingerprint) +target_include_directories(th_common_modeling_v2 + PRIVATE ${TRTLLM_MV2_GENERATED_DIR}) + add_library( th_common SHARED $ @@ -179,7 +186,8 @@ add_library( $ $ $ - $) + $ + $) set_property(TARGET th_common PROPERTY POSITION_INDEPENDENT_CODE ON) set_target_properties(th_common PROPERTIES CXX_STANDARD 20 CXX_STANDARD_REQUIRED ON) diff --git a/cpp/tensorrt_llm/thop/modelingV2ContractOp.cpp b/cpp/tensorrt_llm/thop/modelingV2ContractOp.cpp new file mode 100644 index 000000000000..c8467f34a09c --- /dev/null +++ b/cpp/tensorrt_llm/thop/modelingV2ContractOp.cpp @@ -0,0 +1,46 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Generated by trtllm_mv2_define_contract_fingerprint (cpp/cmake/modules/modelingV2.cmake). +#include "mv2ContractFingerprint.h" + +#include +#include + +namespace +{ + +//! The contract fingerprint this library was built with: a hash of the stock headers listed in +//! cpp/tensorrt_llm/modeling_v2/contractSurface.txt, the global compile definitions, and the torch and CUDA +//! versions. Every modeling_v2 family library reports its own, and the loader +//! (tensorrt_llm/_torch/_experimental/modeling_v2/_native.py) refuses one that differs from this. +std::string mv2ContractFingerprint() +{ + return TRTLLM_MV2_CONTRACT_FINGERPRINT; +} + +} // namespace + +TORCH_LIBRARY_FRAGMENT(trtllm, m) +{ + m.def("mv2_contract_fingerprint() -> str"); +} + +TORCH_LIBRARY_IMPL(trtllm, CompositeExplicitAutograd, m) +{ + m.impl("mv2_contract_fingerprint", &mv2ContractFingerprint); +} diff --git a/jenkins/scripts/cbts/README.md b/jenkins/scripts/cbts/README.md index 047fa7692588..c1b5ae6a3886 100644 --- a/jenkins/scripts/cbts/README.md +++ b/jenkins/scripts/cbts/README.md @@ -50,7 +50,7 @@ Nine rules, registered in `cbts/command/main.py::RULE_CLASSES`: | `TestListRule` | `testlistonly` | `tests/integration/test_lists/test-db/*.yml` | | `VisualGenRule` | `visualgenonly` | `examples/visual_gen/**`, `scripts/visualgen_eval/**`, `tensorrt_llm/_torch/visual_gen/**`, `tensorrt_llm/media/**`, `tensorrt_llm/visual_gen/**` (excl. docs; reference images such as `cat_piano.png` ARE test fixtures and stay claimed; outward-facing files force fallback) | | `SpecDecRule` | `specdeconly` | `tensorrt_llm/_torch/speculative/**`, `tensorrt_llm/models/{eagle,medusa,redrafter}/**`, `examples/{eagle,medusa,redrafter,draft_target_model,ngram}/**`, `examples/llm-api/llm_speculative_decoding.py` (excl. docs; other suffixes incl. images kept as potential test fixtures) | -| `ModelingV2Rule` | `modelingv2only` | `tensorrt_llm/_torch/_experimental/modeling_v2/**` (excl. docs) | +| `ModelingV2Rule` | `modelingv2only` | `tensorrt_llm/_torch/_experimental/modeling_v2/**`, `cpp/tensorrt_llm/modeling_v2/**`, `cpp/cmake/modules/modelingV2*.cmake` (excl. docs); per family when only one family changed | | `AgentFlowRule` | `agentflowonly` | `agent-flow/**` (excl. docs and out-of-scope metadata) | | `OpenEngineRule` | `openengineonly` | `tensorrt_llm/grpc/openengine/**` (excl. docs) | | `DocsRule` | `docsonly` | Outside noop paths: `docs/**` and verified Python docstring edits → `CPU-Build_Docs` plus complete `l0_cpu`; other `**/*.md` / `**/*.rst` → `CPU-Build_Docs` only | @@ -67,7 +67,7 @@ See `cbts/rules/README.md` for per-rule logic. | `testlistonly` | `TestListRule` fired solo: PR only adds entries under `tests/integration/test_lists/test-db/*.yml`. | | `visualgenonly` | `VisualGenRule` fired solo: PR only touches VisualGen internal source paths (`examples/visual_gen/**`, `scripts/visualgen_eval/**`, `tensorrt_llm/_torch/visual_gen/**`; excl. docs; image fixtures like `cat_piano.png` are claimed). Narrows to blocks containing VG test entries. Outward-facing files under `tensorrt_llm/visual_gen/**` and `tensorrt_llm/media/**` (eagerly imported by `trtllm-serve`) force `null` fallback. | | `specdeconly` | `SpecDecRule` fired solo: PR only touches speculative-decoding source paths (`tensorrt_llm/_torch/speculative/**`, `tensorrt_llm/models/{eagle,medusa,redrafter}/**`, `examples/{eagle,medusa,redrafter,draft_target_model,ngram}/**`, `examples/llm-api/llm_speculative_decoding.py`; excl. docs). Narrows to blocks containing spec-dec test entries (eagle / medusa / redrafter / ngram / draft-target-model / MTP). | -| `modelingv2only` | `ModelingV2Rule` fired solo: PR only touches the modeling_v2 subtree (`tensorrt_llm/_torch/_experimental/modeling_v2/**`; excl. docs, which are a fifth of the subtree — every catalog entry carries a contract document). Narrows to blocks containing modeling_v2 test entries (`unittest/_torch/modeling_v2/`, `test_modeling_v2_*`). No outward-facing fallback is needed: nothing beyond the resolver module is imported unless `LLM(modeling_v2=...)` is on, and its callers outside the subtree (`_torch/pyexecutor/model_loader.py`, `_torch/models/modeling_auto.py`) are left unclaimed, so touching the shared resolver falls back to baseline. | +| `modelingv2only` | `ModelingV2Rule` fired solo: PR only touches modeling_v2 code -- the Python subtree (`tensorrt_llm/_torch/_experimental/modeling_v2/**`), its native code (`cpp/tensorrt_llm/modeling_v2/**`, one library per model family) or the native build module (`cpp/cmake/modules/modelingV2*.cmake`); excl. docs, which are a fifth of the Python subtree — every catalog entry carries a contract document. A change confined to one family's directories narrows to that family's entries (`accuracy/modeling_v2/test_.py`, `unittest/_torch/modeling_v2/models/`) plus the modeling_v2 CPU checks; any other modeling_v2 change narrows to every modeling_v2 entry. No outward-facing fallback is needed: nothing beyond the resolver module is imported, and no family library is loaded, unless `LLM(modeling_v2=...)` is on; the CPU checks that keep this true run in every scope. The callers outside the subtree (`_torch/pyexecutor/model_loader.py`, `_torch/models/modeling_auto.py`, the `trtllm::mv2_contract_fingerprint` op in `cpp/tensorrt_llm/thop/`) are left unclaimed, so touching them falls back to baseline. | | `agentflowonly` | `AgentFlowRule` fired solo: PR only touches `agent-flow/**` source or test files (excl. docs). Runs `CPU-AgentFlow-UnitTest`. | | `openengineonly` | `OpenEngineRule` fired solo: PR only touches `tensorrt_llm/grpc/openengine/**` source files (excl. docs). Narrows to the registered OpenEngine unit tests: the stub-based ones on the always-run `CPU-Generic-*` stages, plus `test_capability_conformance.py` on `A10-PyTorch-*`, which needs a GPU. | | `docsonly` | `DocsRule` fired solo: all documentation changes run `CPU-Build_Docs`; changes under `docs/**` also run the complete `l0_cpu` suite on matching `CPU-Generic-*` stages. Docs-only orchestration skips the SBSA track, so its resolved ARM stage is not launched. | diff --git a/jenkins/scripts/cbts/cbts/rules/README.md b/jenkins/scripts/cbts/cbts/rules/README.md index f8f722363eb5..aba2aa56619c 100644 --- a/jenkins/scripts/cbts/cbts/rules/README.md +++ b/jenkins/scripts/cbts/cbts/rules/README.md @@ -16,7 +16,7 @@ for the overall CBTS architecture. | `test_list.py` | `TestListRule` | `testlistonly` | `tests/integration/test_lists/test-db/*.yml` | | `visual_gen.py` | `VisualGenRule` | `visualgenonly` | `examples/visual_gen/**`, `scripts/visualgen_eval/**`, `tensorrt_llm/_torch/visual_gen/**`, `tensorrt_llm/media/**`, `tensorrt_llm/visual_gen/**` (each excl. docs) | | `spec_dec.py` | `SpecDecRule` | `specdeconly` | `tensorrt_llm/_torch/speculative/**`, `tensorrt_llm/models/{eagle,medusa,redrafter}/**`, `examples/{eagle,medusa,redrafter,draft_target_model,ngram}/**`, `examples/llm-api/llm_speculative_decoding.py` (each excl. docs) | -| `modeling_v2.py` | `ModelingV2Rule` | `modelingv2only` | `tensorrt_llm/_torch/_experimental/modeling_v2/**` (excl. docs) | +| `modeling_v2.py` | `ModelingV2Rule` | `modelingv2only` | `tensorrt_llm/_torch/_experimental/modeling_v2/**`, `cpp/tensorrt_llm/modeling_v2/**`, `cpp/cmake/modules/modelingV2*.cmake` (excl. docs); per family when only one family changed | | `agent_flow.py` | `AgentFlowRule` | `agentflowonly` | `agent-flow/**` (excl. docs and out-of-scope metadata) → the single `CPU-AgentFlow-UnitTest` stage; not test-db-driven | | `openengine.py` | `OpenEngineRule` | `openengineonly` | `tensorrt_llm/grpc/openengine/**` (excl. docs) → the `l0_cpu` block containing `unittest/grpc/openengine/` | | `docs.py` | `DocsRule` | `docsonly` | Outside noop paths: `docs/**` and verified Python docstring edits → `CPU-Build_Docs` plus complete `l0_cpu`; other `**/*.md` / `**/*.rst` → `CPU-Build_Docs` only | @@ -303,7 +303,20 @@ Outcomes: Path-only rule. Claims non-documentation source changes under `tensorrt_llm/_torch/_experimental/modeling_v2/`, the self-contained second modeling path (one flat forward per checkpoint/arch/parallel triple, assembled -from a catalog of op wrappers). +from a catalog of op wrappers), under `cpp/tensorrt_llm/modeling_v2/`, its +native code (one library per model family, loaded only when routing picks +one of the family's targets), and the native build module +`cpp/cmake/modules/modelingV2*.cmake`. + +Scope — per family, else all of modeling_v2: a change confined to one +family's directories (`models//` in Python, +`cpp/tensorrt_llm/modeling_v2//` in C++, the family named by the +Python package whose camelCase form the C++ directory is) keeps that family's +entries -- `accuracy/modeling_v2/test_.py` and +`unittest/_torch/modeling_v2/models/` -- plus the modeling_v2 CPU +checks (`unittest/_torch/modeling_v2/test_modeling_v2_*.py`). Several +families union. Any other modeling_v2 change keeps every modeling_v2 entry. +A claimed family with no entry in any test-db yaml falls back to baseline. `.md` exclusion carries more weight here than elsewhere: every catalog entry ships a contract document, so roughly a fifth of the subtree is @@ -318,9 +331,10 @@ in `backend: pytorch` blocks beside everything else. A block belongs to modeling_v2 iff one of its `tests:` entries matches `_MV2_ENTRY_PATTERNS`: -- `unittest/_torch/modeling_v2/` — the op-level catalog matrix, carried - as whole-directory entries (one on `l0_gb300`, one on - `l0_gb300_multi_gpus` for the 4-rank collectives). +- `unittest/_torch/modeling_v2/` — the op-level catalog matrix (one + directory entry with `--ignore` of `comm/` and `models/`, plus `comm/` + on `l0_gb300_multi_gpus` for the 4-rank collectives) and one entry per + family under `models/`, so a family scope can keep its own. - `test_modeling_v2_` — the accuracy gates, and any future unit file. Both markers are exact by construction rather than by luck: every test diff --git a/jenkins/scripts/cbts/cbts/rules/modeling_v2.py b/jenkins/scripts/cbts/cbts/rules/modeling_v2.py index 791827e35e28..9db8beae3026 100644 --- a/jenkins/scripts/cbts/cbts/rules/modeling_v2.py +++ b/jenkins/scripts/cbts/cbts/rules/modeling_v2.py @@ -11,74 +11,46 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""ModelingV2Rule — narrows CI when the modeling_v2 subtree changes. - -modeling_v2 is a second modeling path living entirely under -`tensorrt_llm/_torch/_experimental/modeling_v2/`: one self-contained forward per -(checkpoint, GPU arch, parallel topology), assembled from a catalog of -op wrappers. - -Block selection: a block belongs to modeling_v2 iff one of its `tests:` -entries matches a marker in `_MV2_ENTRY_PATTERNS`. Those markers are exact -by construction rather than by luck: every unit test file in the subtree is -named `test_modeling_v2_*` precisely so it cannot collide with the upstream -test of the same op, and the accuracy gates live in their own -`accuracy/modeling_v2/` directory. - -Entry selection: each changed file maps to the tests it can affect. - -- `models/` is a tree of nodes. A directory is a node when it, or a - directory below it, holds a target's `modeling.py`; a family directory - (`models//`) is always one. Other directories -- `ops/` and the - like -- serve the node above them. A changed file belongs to the nearest - node at or above it, and selects that node's unit tests - (`unittest/_torch/modeling_v2/models/`) plus the accuracy gate of - every target under it. A target's gate is `Test` in - `accuracy/modeling_v2/test_.py`, the class name read from the - import line in the family's `routing.py` that routes to it. -- Anything else -- the catalog, the resolver, `models/__init__.py` -- can - reach every target and selects every modeling_v2 entry. - -The tests directly under `unittest/_torch/modeling_v2/` check the tree as a -whole (claims, routing, contracts), so any change keeps them. - -A unit test prefix that no entry of a block sits at or under keeps the -longest entry of that block that covers it instead: a family is one -directory entry, and a change to one of its targets runs the family's -unit tests. A test file listed at a directory above a node -- a family's -own test file -- belongs to every node under that directory, so a change -to any of them keeps it. Whenever the tree on disk cannot answer -- a node directory -deleted with its `routing.py` -- the file falls back to the wider -selection. So does a node holding a `modeling.py` that its family's -`routing.py` does not import, such as a target nested below the family's -first level: its accuracy gate cannot be named. - -Outward fallback: not needed, and that is a property of the design -rather than an accident. Nothing beyond the resolver module is imported -unless `LLM(modeling_v2=...)` is on: the model loader calls -`modeling_v2_resolve`, which returns immediately when the switch is off, -and the routing modules are imported lazily behind it. The callers -outside the subtree are `tensorrt_llm/_torch/pyexecutor/model_loader.py` -(decides) and `tensorrt_llm/_torch/models/modeling_auto.py` (reads the -decision off `ModelConfig`), neither of which this rule claims -- a PR -touching them falls back to baseline, which is what a change to the -shared resolver deserves. - -`.md` exclusion matters more here than for most rules: the catalog -carries a contract document per entry, so roughly a fifth of the files -in the subtree are Markdown. Claiming them would make a +"""ModelingV2Rule — narrows CI when only modeling_v2 code changes. + +modeling_v2 is a second modeling path: one self-contained forward per +(checkpoint, GPU arch, parallel topology), assembled from a catalog of op +wrappers. Its Python lives under `tensorrt_llm/_torch/_experimental/modeling_v2/` +and its native code under `cpp/tensorrt_llm/modeling_v2/`, which builds one +library per model family, loaded only when routing picks one of the family's +targets. + +Scope — per family, else all of modeling_v2: +A change confined to one family's directories (`models//` in Python, +`cpp/tensorrt_llm/modeling_v2//` in C++) selects that family's test +entries -- `accuracy/modeling_v2/test_.py` and +`unittest/_torch/modeling_v2/models//` -- plus the modeling_v2 CPU +checks. Any other modeling_v2 change (catalog, resolver, loader, the native +build module and contract surface) selects every modeling_v2 entry. Several +families in one PR union their entries. + +Why no outward fallback is needed: nothing beyond the resolver is imported, and +no family library is loaded, unless `LLM(modeling_v2=...)` is on -- the model +loader calls the resolver, which returns immediately when the switch is off, +and a target's library opens only when routing imports that target. Family +libraries are separate modules, never linked into libtensorrt_llm.so or +libth_common.so. The claims tests under `tests/unittest/_torch/modeling_v2/` +keep that true (stock never references a target, targets never reach a +sibling, `modeling_v2` defaults to `off`); they run in every scope this rule +selects. The callers outside the subtree (`_torch/pyexecutor/model_loader.py`, +`_torch/models/modeling_auto.py`, the `trtllm::mv2_contract_fingerprint` op in +`cpp/tensorrt_llm/thop/`) stay unclaimed: a change to them runs the baseline. + +`.md` exclusion matters here: every catalog entry carries a contract document, +so a fifth of the Python subtree is Markdown; claiming it would let a documentation-only PR pull in multi-GPU GB300 stages. -PerfSanity policy: `perfsanity_relevant` is dynamic, True only when a -matched block lives in a `*_perf_sanity*` yaml -- same as VisualGenRule -/ SpecDecRule. modeling_v2 has no perf-sanity entries -today, so this aggregates to False and Groovy Layer 2 drops the -force-keep of `*-PerfSanity-*` stages. +PerfSanity policy: `perfsanity_relevant` is True only when a selected block +lives in a `*_perf_sanity*` yaml -- same as VisualGenRule / SpecDecRule. -Sanity policy: `sanity_relevant=False`. The subtree ships no -user-facing entry point and is not imported by `trtllm-serve` or by -`import tensorrt_llm`, so nothing it contains is what PackageSanityCheck -verifies about the wheel. +Sanity policy: `sanity_relevant=False`. Nothing here is imported by +`trtllm-serve` or `import tensorrt_llm`, and the family libraries are not +loaded by either, so nothing here is what PackageSanityCheck verifies. """ from __future__ import annotations @@ -93,28 +65,54 @@ from .base import PRInputs, Rule, RuleResult from .docs import is_docs_path -# Source-path prefixes the rule may claim. Tests under tests/** are left -# to TestsDefRule; the two scopes combine via _TESTSONLY_FAMILY. -_MV2_SRC_PREFIXES: tuple[str, ...] = ("tensorrt_llm/_torch/_experimental/modeling_v2/",) +_PY_ROOT = "tensorrt_llm/_torch/_experimental/modeling_v2/" +_PY_MODELS = _PY_ROOT + "models/" +_CPP_ROOT = "cpp/tensorrt_llm/modeling_v2/" +# Directories of the native tree that are not families. +_CPP_NON_FAMILY_DIRS = frozenset({"common", "catalog"}) +# The native build module: shared by every family library. +_CPP_BUILD_FILES = frozenset( + { + "cpp/cmake/modules/modelingV2.cmake", + "cpp/cmake/modules/modelingV2Fingerprint.cmake", + } +) + +# Source-path prefixes the rule may claim. Tests under tests/** are left to +# TestsDefRule; the two scopes combine via _TESTSONLY_FAMILY. +_MV2_SRC_PREFIXES: tuple[str, ...] = (_PY_ROOT, _CPP_ROOT) _UNIT_ROOT = "unittest/_torch/modeling_v2" _ACCURACY_ROOT = "accuracy/modeling_v2" # Substrings that mark a test entry as modeling_v2. All are unambiguous: -# - "unittest/_torch/modeling_v2/" → the op and target unit tests +# - "unittest/_torch/modeling_v2/" → the op-level catalog matrix and the +# per-family unit tests (`.../models/`) # - "accuracy/modeling_v2/" → the per-model accuracy gates -# - "test_modeling_v2_" → any future unit file named by the subtree's -# own convention +# - "test_modeling_v2_" → any unit file named by the subtree's own convention _MV2_ENTRY_PATTERNS: tuple[str, ...] = ( f"{_UNIT_ROOT}/", f"{_ACCURACY_ROOT}/", "test_modeling_v2_", ) -# A family's `routing.py` imports each target it routes to by this line, from -# the target directory directly under the family. The claims test reads the -# same line with the same pattern. -_TARGET_IMPORT = re.compile(r"^\s*from \.(\w+)\.modeling import (\w+)$", re.MULTILINE) +# The modeling_v2 CPU checks (claims, routing, native loader): selected in +# every scope, because they are what keeps the narrowing sound. +_MV2_CHECK_ENTRY = re.compile(r"^unittest/_torch/modeling_v2/test_modeling_v2_[\w]+\.py$") + + +def _camel(name: str) -> str: + """The C++ directory of a family: `minimax_m3` -> `minimaxM3`. + + Mirrors `_trtllm_mv2_camel` in cpp/cmake/modules/modelingV2.cmake. + """ + parts = [p for p in name.split("_") if p] + return parts[0] + "".join(p[:1].upper() + p[1:] for p in parts[1:]) + + +def _snake(camel_name: str) -> str: + """Best-effort inverse of `_camel`, used only without a repo root.""" + return re.sub(r"(?<=[a-z0-9])([A-Z])", lambda m: "_" + m.group(1).lower(), camel_name) def _is_mv2_claim(path: str) -> bool: @@ -126,41 +124,15 @@ def _is_mv2_claim(path: str) -> bool: fixture, so the rule keeps claiming it and re-runs the stages (safe over-run). """ - if not path.startswith(_MV2_SRC_PREFIXES): + if not (path.startswith(_MV2_SRC_PREFIXES) or path in _CPP_BUILD_FILES): return False - if is_docs_path(path): - return False - return True + return not is_docs_path(path) def _entry_is_mv2(entry: str) -> bool: return any(p in entry for p in _MV2_ENTRY_PATTERNS) -def _mv2_entries(block) -> list[str]: - return [t for t in block.tests if _entry_is_mv2(t)] - - -def _is_whole_tree_test(target: str) -> bool: - """A test directly under the unit root, which checks the tree as a whole.""" - rest = target.removeprefix(f"{_UNIT_ROOT}/").split("::")[0] - return target.startswith(f"{_UNIT_ROOT}/") and "/" not in rest and rest.endswith(".py") - - -def _is_ancestor_file(target: str, prefix: str) -> bool: - """A test file in a directory below the models root that holds `prefix`. - - A test file at a family (or an intermediate) directory belongs to every - node under it, so a change to any of those nodes runs it. - """ - parent, _, name = target.split("::")[0].rpartition("/") - return ( - name.endswith(".py") - and parent.startswith(f"{_UNIT_ROOT}/models/") - and _target_in_filter_subtree(prefix, parent) - ) - - def _is_perf_sanity_stem(stem: str) -> bool: """True for perf-sanity yaml stems (`l0_*_perf_sanity*`).""" return "perf_sanity" in stem @@ -190,97 +162,99 @@ class ModelingV2Rule(Rule): name = "modelingv2" needs_diff_for: tuple[str, ...] = () - def __init__(self, yaml_index: YAMLIndex, stages: dict[str, Stage], repo_root: Path) -> None: + def __init__( + self, + yaml_index: YAMLIndex, + stages: dict[str, Stage], + repo_root: Optional[Path] = None, + ) -> None: self.yaml_index = yaml_index self._stages_by_yaml = stages_by_yaml_stem(stages) - self._src_root = Path(repo_root) / _MV2_SRC_PREFIXES[0] - self._tests_root = Path(repo_root) / "tests" - - def _is_node(self, rel_dir: Path) -> bool: - """Whether `models/` is a node: a family, or holds a target at or below it.""" - path = self._src_root / "models" / rel_dir - return len(rel_dir.parts) == 1 or (path.is_dir() and any(path.rglob("modeling.py"))) + # camelCase native directory -> family, from the Python family packages. + self._family_of_cpp_dir: dict[str, str] = {} + if repo_root is not None: + models = Path(repo_root) / _PY_MODELS + if models.is_dir(): + for family_dir in models.iterdir(): + if family_dir.is_dir() and (family_dir / "__init__.py").is_file(): + self._family_of_cpp_dir[_camel(family_dir.name)] = family_dir.name + + def family_of(self, path: str) -> Optional[str]: + """The family a claimed path belongs to, or None for shared modeling_v2 code.""" + if path.startswith(_PY_MODELS): + parts = path[len(_PY_MODELS) :].split("/") + return parts[0] if len(parts) > 1 else None + if path.startswith(_CPP_ROOT): + parts = path[len(_CPP_ROOT) :].split("/") + if len(parts) < 2 or parts[0] in _CPP_NON_FAMILY_DIRS: + return None + return self._family_of_cpp_dir.get(parts[0], _snake(parts[0])) + return None + + @staticmethod + def _entry_in_family(target: str, family: str) -> bool: + accuracy = f"accuracy/modeling_v2/test_{family}.py" + unit = f"unittest/_torch/modeling_v2/models/{family}" + return ( + target == accuracy + or target.startswith(accuracy + "::") + or target == unit + or target.startswith(unit + "/") + ) - def _gates_under(self, node: Path) -> Optional[set[str]]: - """Accuracy-gate prefixes of every target at or under `node`. + def _select(self, families: Optional[set[str]]): + """Select the modeling_v2 entries a change runs. - None if unreadable: no `routing.py`, or a `modeling.py` under `node` - that no import line in it routes to, whose gate cannot be named. + Every modeling_v2 entry when `families` is None, else the entries of + `families` plus the CPU checks. Also returns the families that matched + no entry. """ - models = self._src_root / "models" - family = node.parts[0] - routing = models / family / "routing.py" - if not routing.is_file(): - return None - routed, gates = set(), set() - for target, target_cls in _TARGET_IMPORT.findall(routing.read_text(encoding="utf-8")): - leaf = Path(family, target) - routed.add(leaf) - if leaf == node or node in leaf.parents: - gates.add(f"{_ACCURACY_ROOT}/test_{family}.py::Test{target_cls}") - if any( - m.parent.relative_to(models) not in routed for m in (models / node).rglob("modeling.py") - ): - return None - return gates - - def _prefixes_for(self, path: str) -> Optional[set[str]]: - """Filter prefixes a changed file selects; None selects every modeling_v2 entry.""" - rel = Path(path[len(_MV2_SRC_PREFIXES[0]) :]) - if rel.parts[0] != "models" or len(rel.parts) < 3: - return None - node = Path(*rel.parts[1:-1]) - while not self._is_node(node): - node = node.parent - gates = self._gates_under(node) - if gates is None: - return None - return {f"{_UNIT_ROOT}/models/{node.as_posix()}"} | gates + block_filters: dict[tuple[str, int], dict[str, set[str]]] = {} + matched_families: set[str] = set() + for block in self.yaml_index.blocks: + for entry in block.tests: + if not _entry_is_mv2(entry): + continue + target = _entry_target(entry) + if not target: + continue + if families is not None: + owners = {f for f in families if self._entry_in_family(target, f)} + if not owners and not _MV2_CHECK_ENTRY.match(target): + continue + matched_families |= owners + key = (block.yaml_stem, block.block_index) + block_filters.setdefault(key, {}).setdefault(target, set()).add(entry) + unmatched = set() if families is None else families - matched_families + return block_filters, unmatched def apply(self, pr: PRInputs) -> Optional[RuleResult]: claimed = {f for f in pr.changed_files if _is_mv2_claim(f)} if not claimed: return None - prefixes: Optional[set[str]] = set() - for path in sorted(claimed): - selected = self._prefixes_for(path) - if selected is None: - prefixes = None - break - prefixes |= selected + owners = {self.family_of(f) for f in claimed} + families = None if None in owners else set(owners) + block_filters, unmatched = self._select(families) - block_filters: dict[tuple[str, int], dict[str, set[str]]] = {} - for block in self.yaml_index.blocks: - entries = _mv2_entries(block) - if prefixes is not None: - entries = _kept_entries(entries, prefixes) - if not entries: - continue - prefix_dict = block_filters.setdefault((block.yaml_stem, block.block_index), {}) - for entry in entries: - target = _entry_target(entry) - if target: - prefix_dict.setdefault(target, set()).add(entry) - - if not block_filters: - # Defensive: modeling_v2 source changed but no modeling_v2 block - # exists in any yaml. Do not fabricate stages -- fall back to - # baseline so the change still gets coverage. Reachable if the - # subtree's entries are ever removed from the test-db without - # the subtree going with them. + if not block_filters or unmatched: + # Do not fabricate stages -- fall back to baseline so the change + # still gets coverage. Reachable if a family registers no test + # entry, or the subtree's entries leave the test-db. + missing = f" for {sorted(unmatched)}" if unmatched else "" return RuleResult( handled_files=claimed, affected_stages=set(), scope=None, reason=( f"modelingv2: {len(claimed)} modeling_v2 source file(s); " - "no modeling_v2 block matched in any test-db yaml — fallback" + f"no modeling_v2 test entry{missing} in any test-db yaml — fallback" ), ) affected = resolve_affected_stages(block_filters, self.yaml_index, self._stages_by_yaml) perfsanity_relevant = any(_is_perf_sanity_stem(stem) for stem, _ in block_filters) + scope_text = "all modeling_v2" if families is None else f"families {sorted(families)}" return RuleResult( handled_files=claimed, @@ -290,8 +264,7 @@ def apply(self, pr: PRInputs) -> Optional[RuleResult]: sanity_relevant=False, perfsanity_relevant=perfsanity_relevant, reason=( - f"modelingv2: {len(claimed)} modeling_v2 source file(s) → " - f"{'every modeling_v2 entry' if prefixes is None else f'{len(prefixes)} test prefix(es)'}, " - f"{len(block_filters)} block(s), {len(affected)} stage(s)" + f"modelingv2: {len(claimed)} modeling_v2 source file(s), {scope_text} → " + f"{len(block_filters)} modeling_v2 block(s), {len(affected)} stage(s)" ), ) diff --git a/scripts/build_wheel.py b/scripts/build_wheel.py index 76f3c1cdd261..d12a33a67e26 100755 --- a/scripts/build_wheel.py +++ b/scripts/build_wheel.py @@ -792,6 +792,38 @@ def has_sm90_or_newer(cuda_architectures: str) -> bool: return False +def install_modeling_v2_libs(mv2_build_dir: Path, lib_dir: Path, + install_file) -> None: + """Install the modeling_v2 family libraries and their manifest. + + The manifest is what the loader reads + (tensorrt_llm/_torch/_experimental/modeling_v2/_native.py). This is also + the packaging completeness check: the manifest lists, per family, the + library its built targets need, and a listed library that is missing fails + the build here rather than turning into a modeling_v2 fallback at run time. + """ + manifest_path = mv2_build_dir / "modeling_v2_manifest.json" + if not manifest_path.is_file(): + raise RuntimeError( + f"{manifest_path} is missing; the cmake configure step writes it " + "(cpp/tensorrt_llm/modeling_v2/CMakeLists.txt)") + manifest = json.loads(manifest_path.read_text()) + install_file(manifest_path, lib_dir / manifest_path.name) + missing = [] + for family, entry in sorted(manifest.get("families", {}).items()): + library = entry.get("library") + if library is None: + continue + if not (mv2_build_dir / library).is_file(): + missing.append(f"{family}: {library}") + continue + install_file(mv2_build_dir / library, lib_dir / library) + if missing: + raise RuntimeError( + "modeling_v2 packaging check: the manifest lists libraries the build " + f"did not produce: {', '.join(missing)}") + + def stage_nccl_extensions_package(wheel: Path, staging_dir: Path) -> None: """Stage NCCL-EP namespace packages from the intermediate wheel.""" package_root = staging_dir / "3rdparty" / "nccl_extensions" @@ -1018,7 +1050,7 @@ def main(*, targets = [ "tensorrt_llm", "th_common", "bindings", "deep_ep", "deep_gemm", - "pg_utils", "flash_mla" + "pg_utils", "flash_mla", "modeling_v2_libs" ] build_nccl_extensions_enabled = has_sm90_or_newer(cuda_architectures) if build_nccl_extensions_enabled: @@ -1341,6 +1373,8 @@ def copy_resolving_symlink(src_path, dst_path): lib_dir / "libtensorrt_llm.so") install_file(build_dir / "tensorrt_llm/thop/libth_common.so", lib_dir / "libth_common.so") + install_modeling_v2_libs(build_dir / "tensorrt_llm/modeling_v2", + lib_dir, install_file) if os.path.exists( build_dir / "tensorrt_llm/executor/cache_transmission/ucx_utils/libtensorrt_llm_ucx_wrapper.so" diff --git a/setup.py b/setup.py index 6ff03926441a..8dddac528eac 100644 --- a/setup.py +++ b/setup.py @@ -179,6 +179,8 @@ def has_ext_modules(self): package_data = [ 'libs/libtensorrt_llm.so', 'libs/libth_common.so', + 'libs/libtrtllm_mv2_*.so', + 'libs/modeling_v2_manifest.json', 'libs/libtensorrt_llm_ucx_wrapper.so', 'libs/libdecoder_attention_0.so', 'libs/libtensorrt_llm_nixl_wrapper.so', diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/README.md b/tensorrt_llm/_torch/_experimental/modeling_v2/README.md index f9c8568348f3..b7dee6dbd746 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/README.md +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/README.md @@ -111,14 +111,29 @@ checkpoint of the same shape is ungated. ``` _router_index.py architectures[0] -> routing module. Small, stable, test-guarded. +_native.py loads a target's native code (see "Native code" below) models// routing.py one forward-reading decision tree per architecture family ____/ modeling.py weights.py ops// entries only this target calls + fake_ops.py fakes of the target's native ops, if it has any catalog// entries more than one target calls ``` +### Native code + +A target's C++/CUDA lives beside the stock tree, not in it: +`cpp/tensorrt_llm/modeling_v2///`, built into one +library per family, `libtrtllm_mv2_.so`. Nothing of it is linked into +`libtensorrt_llm.so` or `libth_common.so`, and nothing loads it until routing +imports one of the family's targets, whose `__init__.py` starts with +`native_ops = load_target_ops(__name__)`. A target this build or this device +cannot run is a `NoTarget`, so `auto` keeps the built-in model; a broken +installation is a `NativeLoadError`, never a fallback. +`cpp/tensorrt_llm/modeling_v2/README.md` is the guide, and +`models/fixture/` is the smallest complete example. + An entry, shared or owned, is one `OpWrapper` subclass: `__call__` is the op call a target makes, `reference` is what it is supposed to compute in plain torch, `is_valid` names the inputs it accepts and answers wrongly, and `CELLS` diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/__init__.py b/tensorrt_llm/_torch/_experimental/modeling_v2/__init__.py index 96e5209c99dc..16d312d3e7b1 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/__init__.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/__init__.py @@ -31,6 +31,7 @@ ModelingV2Mode, NoTarget, modeling_v2_resolve, + modeling_v2_resolve_on_all_ranks, step_contract_enabled, ) @@ -39,5 +40,6 @@ "ModelingV2Mode", "NoTarget", "modeling_v2_resolve", + "modeling_v2_resolve_on_all_ranks", "step_contract_enabled", ] diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/_native.py b/tensorrt_llm/_torch/_experimental/modeling_v2/_native.py new file mode 100644 index 000000000000..ee872fd8bbff --- /dev/null +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/_native.py @@ -0,0 +1,304 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""A target's native (C++/CUDA) code: where it is built, and how the target loads it. + +Native code of a model family lives in ``cpp/tensorrt_llm/modeling_v2//`` +and builds into one library, ``libtrtllm_mv2_.so``, installed next to +``libth_common.so`` together with ``modeling_v2_manifest.json``. Nothing of it is in +the stock libraries, and nothing loads it until routing picks a target of the +family: a target package that has native code starts with:: + + from tensorrt_llm._torch._experimental.modeling_v2._native import load_target_ops + + ops = load_target_ops(__name__) # torch.ops.mv2__ + +``route`` imports the target only on the branch that returns it, so loading is part +of routing: a target whose code this wheel did not build raises :class:`NoTarget` +(``auto`` keeps the built-in model, ``require`` reports why), while a library that +is present but unusable raises :class:`NativeLoadError`, which is never a fallback. + +The checks, in order: + +1. the manifest lists the target as built (otherwise ``NoTarget``); +2. the device's compute capability is covered by an architecture the target was + built for (otherwise ``NoTarget``); +3. the family library opens from the fixed package path; +4. its contract fingerprint equals the loaded ``libth_common.so``'s -- the two were + compiled against the same stock headers, definitions, torch and CUDA; +5. the target's ``fake_ops.py`` registers, which fails if an op it declares is + missing from the library. + +``cpp/tensorrt_llm/modeling_v2/README.md`` is the guide for authors. +""" + +from __future__ import annotations + +import importlib +import importlib.util +import json +import sys +from functools import lru_cache +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional, Tuple + +from ._router_index import NoTarget + +MANIFEST_NAME = "modeling_v2_manifest.json" +MANIFEST_SCHEMA = 1 + +_PACKAGE = __name__.rpartition(".")[0] +_MODELS_PACKAGE = f"{_PACKAGE}.models" + +# tensorrt_llm/libs, where build_wheel.py installs the family libraries. +_LIBS_DIR = Path(__file__).resolve().parents[3] / "libs" + +_loaded_libraries: set = set() + + +class NativeLoadError(RuntimeError): + """A family library the manifest lists as built cannot be used. + + Raised for a broken or mismatched installation: a missing library, one that + fails to open, a contract-fingerprint mismatch, an op ``fake_ops.py`` declares + but the library lacks. It is deliberately not a :class:`NoTarget`, so ``auto`` + does not hide it behind the built-in model. + """ + + +# --- Naming (mirrors cpp/cmake/modules/modelingV2.cmake) --------------------- + + +def camel(name: str) -> str: + """The C++ directory of a family or target: ``scale_add__sm_80__default`` -> + ``scaleAddSm80Default``.""" + parts = [p for p in name.split("_") if p] + return parts[0] + "".join(p[:1].upper() + p[1:] for p in parts[1:]) + + +def identifier(name: str) -> str: + """``name`` with ``__`` collapsed, usable as a C++ or torch identifier.""" + while "__" in name: + name = name.replace("__", "_") + return name + + +def torch_namespace(family: str, target: Optional[str] = None) -> str: + """The torch namespace of a target's ops, or of the family's shared ops.""" + if target is None: + return f"mv2_{family}" + return f"mv2_{family}_{identifier(target)}" + + +def split_target_package(package: str) -> Tuple[str, str]: + """``(family, target)`` of a target package name.""" + prefix = f"{_MODELS_PACKAGE}." + if not package.startswith(prefix): + raise ValueError(f"{package!r} is not a modeling_v2 target package") + parts = package[len(prefix) :].split(".") + if len(parts) != 2: + raise ValueError(f"{package!r} is not a models.. package") + return parts[0], parts[1] + + +def namespace_of(package: str) -> str: + """The torch namespace of the target package ``package``, e.g. for its fake_ops.py.""" + return torch_namespace(*split_target_package(package)) + + +# --- Architectures --------------------------------------------------------------- + + +def arch_covers(arch: str, capability: Tuple[int, int]) -> bool: + """Whether code built for ``arch`` (a manifest entry such as ``100f-real``) runs on + a device of compute capability ``capability``. + + ``NNa`` runs on exactly NN; ``NN`` on NN and later minor versions of the same + major version; ``NNf`` on every member of family NN, which this build treats the + same way. + """ + name = arch[: -len("-real")] if arch.endswith("-real") else arch + suffix = name[-1] if name[-1] in "af" else "" + number = int(name[:-1] if suffix else name) + major, minor = divmod(number, 10) + if suffix == "a": + return capability == (major, minor) + return capability[0] == major and capability[1] >= minor + + +# --- Manifest ----------------------------------------------------------------- + + +@lru_cache(maxsize=None) +def read_manifest(path: Optional[Path] = None) -> Dict[str, Any]: + """The manifest the build wrote next to the family libraries.""" + path = path or (_LIBS_DIR / MANIFEST_NAME) + try: + manifest = json.loads(path.read_text()) + except FileNotFoundError: + raise NativeLoadError( + f"{path} is missing: this installation of TensorRT-LLM has no modeling_v2 " + "manifest. Rebuild with scripts/build_wheel.py." + ) from None + except (OSError, ValueError) as error: + raise NativeLoadError(f"{path} cannot be read: {error}") from None + if manifest.get("schema") != MANIFEST_SCHEMA: + raise NativeLoadError( + f"{path} has schema {manifest.get('schema')!r}; this loader reads schema " + f"{MANIFEST_SCHEMA}. The Python package and the native libraries are from " + "different builds." + ) + return manifest + + +def available_targets(manifest: Optional[Dict[str, Any]] = None) -> List[Dict[str, Any]]: + """Every target the manifest knows, with its status, architectures and reason -- + what this installation supports.""" + manifest = read_manifest() if manifest is None else manifest + rows = [] + for family, entry in sorted(manifest.get("families", {}).items()): + for target, unit in sorted(entry.get("targets", {}).items()): + rows.append( + { + "family": family, + "target": target, + "status": unit["status"], + "archs": list(unit["archs"]), + "reason": unit["reason"], + "library": entry.get("library"), + } + ) + return rows + + +def _family_entry(manifest: Dict[str, Any], family: str) -> Dict[str, Any]: + if not manifest.get("enabled", False): + raise NoTarget( + f"this TensorRT-LLM build has no modeling_v2 libraries: {manifest.get('reason')}" + ) + entry = manifest.get("families", {}).get(family) + if entry is None: + raise NativeLoadError( + f"modeling_v2 family {family!r} has native code but is not in the manifest; " + "the Python package and the native libraries are from different builds" + ) + return entry + + +def _target_entry(manifest: Dict[str, Any], family: str, target: str) -> Dict[str, Any]: + entry = _family_entry(manifest, family) + unit = entry.get("targets", {}).get(target) + if unit is None: + raise NativeLoadError( + f"modeling_v2 target {family}/{target} has native code but is not in the " + "manifest; the Python package and the native libraries are from different builds" + ) + if unit["status"] != "built": + raise NoTarget( + f"{family}/{target}: native code not built in this TensorRT-LLM build " + f"({unit['status']}: {unit['reason']})" + ) + return unit + + +# --- Loading ------------------------------------------------------------------ + + +def _ops_namespace(namespace: str) -> Any: + import torch + + return getattr(torch.ops, namespace) + + +def load_family_library(family: str, manifest: Optional[Dict[str, Any]] = None) -> None: + """Open ``libtrtllm_mv2_.so`` and check its contract fingerprint. + + Idempotent. Does not look at a device, so it can run anywhere the stock + libraries load; :func:`load_target_ops` adds the target-level checks. + """ + manifest = read_manifest() if manifest is None else manifest + entry = _family_entry(manifest, family) + library = entry.get("library") + if library is None: + raise NoTarget(f"modeling_v2 family {family!r}: no target of it is built in this build") + path = _LIBS_DIR / library + if path in _loaded_libraries: + return + if not path.is_file(): + raise NativeLoadError(f"{path} is listed in the manifest but missing") + + import torch + + try: + torch.ops.load_library(str(path)) + except OSError as error: + raise NativeLoadError(f"{path} cannot be loaded: {error}") from None + + try: + stock = torch.ops.trtllm.mv2_contract_fingerprint() + except (AttributeError, RuntimeError) as error: + raise NativeLoadError( + f"libth_common.so does not report a modeling_v2 contract fingerprint ({error}); " + f"it predates {path.name}" + ) from None + fingerprint = _ops_namespace(torch_namespace(family)).contract_fingerprint() + if fingerprint != stock: + raise NativeLoadError( + f"{path.name} was built against a different stock contract than the loaded " + f"libth_common.so (fingerprint {fingerprint} vs {stock}); install libraries " + "from one build" + ) + _loaded_libraries.add(path) + + +def _device_capability() -> Optional[Tuple[int, int]]: + import torch + + if not torch.cuda.is_available(): + return None + return torch.cuda.get_device_capability() + + +def load_target_ops(package: str, manifest: Optional[Dict[str, Any]] = None) -> Any: + """Load the native code of the target package ``package`` and return its ops + namespace, ``torch.ops.mv2__``. + + Call it from the target's ``__init__.py`` with ``__name__``. Raises + :class:`NoTarget` when this build or this device cannot run the target and + :class:`NativeLoadError` when the installation is broken. + """ + if sys.platform == "win32": + raise NoTarget("modeling_v2 native code is not built on Windows") + family, target = split_target_package(package) + manifest = read_manifest() if manifest is None else manifest + unit = _target_entry(manifest, family, target) + + capability = _device_capability() + if capability is not None and not any(arch_covers(a, capability) for a in unit["archs"]): + raise NoTarget( + f"{family}/{target}: built for {', '.join(unit['archs'])}, which does not run " + f"on this sm_{capability[0]}{capability[1]} device" + ) + + load_family_library(family, manifest) + + fake_ops = f"{package}.fake_ops" + if importlib.util.find_spec(fake_ops) is not None: + try: + importlib.import_module(fake_ops) + except (AttributeError, RuntimeError, ValueError) as error: + raise NativeLoadError( + f"{fake_ops} does not match the ops in the family library: {error}" + ) from None + return _ops_namespace(unit["namespace"]) + + +def built_families(manifest: Optional[Dict[str, Any]] = None) -> Iterable[str]: + """Families whose library this build produced.""" + manifest = read_manifest() if manifest is None else manifest + if not manifest.get("enabled", False): + return [] + return sorted( + family + for family, entry in manifest.get("families", {}).items() + if entry.get("library") is not None + ) diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/_router_index.py b/tensorrt_llm/_torch/_experimental/modeling_v2/_router_index.py index b9cd4f09fac1..339208634476 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/_router_index.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/_router_index.py @@ -175,6 +175,33 @@ def engine_kv_cache_quant_algo(config: "ModelConfig", llm_args: "TorchLlmArgs"): return {"fp8": QuantAlgo.FP8, "nvfp4": QuantAlgo.NVFP4}.get(dtype, dtype) +def _route( + config: "ModelConfig", llm_args: "TorchLlmArgs", mode: ModelingV2Mode +) -> Tuple[Optional[type], Optional[str]]: + """``(target class, None)``, or ``(None, why)`` for the built-in implementation. + + ``why`` is None when the checkpoint names no architecture, which is not a + fallback worth reporting. Under ``require`` a missing target raises instead. + A target's native code loads when ``route`` imports the target (see + ``_native``), so its ``NoTarget`` lands here like any other. + """ + pretrained_config = config.pretrained_config + if not getattr(pretrained_config, "architectures", None): + return None, None + arch = pretrained_config.architectures[0] + + try: + route = registered_routes().get(arch) + if route is None: + known = ", ".join(sorted(registered_routes())) or "(none)" + raise NoTarget(f"no route for this architecture; routed architectures: {known}") + return route(config, llm_args, _device_compute_capability()), None + except NoTarget as why: + if mode is ModelingV2Mode.REQUIRE: + raise ValueError(f"modeling_v2='require', but no target fits {arch!r}: {why}") from None + return None, f"no target fits {arch!r} ({why})" + + def modeling_v2_resolve(config: "ModelConfig", llm_args: "TorchLlmArgs") -> Optional[type]: """Decide which modeling_v2 target, if any, this deployment builds. @@ -182,26 +209,80 @@ def modeling_v2_resolve(config: "ModelConfig", llm_args: "TorchLlmArgs") -> Opti when ``llm_args.modeling_v2`` is ``off``, when no ``route`` is registered for the architecture, or when that ``route`` raises ``NoTarget``. In ``require`` every one of those but ``off`` raises ``ValueError`` instead. + + This decides for the calling rank only; the model loader uses + :func:`modeling_v2_resolve_on_all_ranks`. """ mode = ModelingV2Mode.of(llm_args) if mode is ModelingV2Mode.OFF: return None + target, fallback = _route(config, llm_args, mode) + if fallback is not None: + logger.info(f"modeling_v2: {fallback}; using the built-in implementation") + return target - pretrained_config = config.pretrained_config - if not getattr(pretrained_config, "architectures", None): + +def _qualified(target: Optional[type]) -> Optional[str]: + return None if target is None else f"{target.__module__}.{target.__qualname__}" + + +def modeling_v2_resolve_on_all_ranks( + config: "ModelConfig", llm_args: "TorchLlmArgs" +) -> Optional[type]: + """:func:`modeling_v2_resolve`, agreed by every rank of the engine. + + Each rank routes on its own facts -- its device, and whether this + installation built the target's native code -- so one rank can land on the + built-in model while its peers build the target, and the first collective + then hangs instead of failing. Every rank therefore reports its outcome, and + all of them raise unless the outcomes agree. A rank's error is reported + after the exchange, not before, so its peers are never left waiting in it. + + ``off`` returns before any exchange: nothing in this package runs then. + """ + mode = ModelingV2Mode.of(llm_args) + if mode is ModelingV2Mode.OFF: return None - arch = pretrained_config.architectures[0] + target = fallback = error = None try: - route = registered_routes().get(arch) - if route is None: - known = ", ".join(sorted(registered_routes())) or "(none)" - raise NoTarget(f"no route for this architecture; routed architectures: {known}") - return route(config, llm_args, _device_compute_capability()) - except NoTarget as why: - if mode is ModelingV2Mode.REQUIRE: - raise ValueError(f"modeling_v2='require', but no target fits {arch!r}: {why}") from None - logger.info( - f"modeling_v2: no target fits {arch!r} ({why}); using the built-in implementation" + target, fallback = _route(config, llm_args, mode) + except Exception as caught: # noqa: BLE001 -- re-raised below, after every rank has reported + error = caught + outcome = ( + _qualified(target), + fallback, + None if error is None else f"{type(error).__name__}: {error}", + ) + + mapping = config.mapping + outcomes = [outcome] + if mapping.world_size > 1: + from tensorrt_llm._torch.distributed.communicator import Distributed + + outcomes = Distributed.get(mapping).allgather(outcome) + + if error is not None: + raise error + failed = {rank: o[2] for rank, o in enumerate(outcomes) if o[2] is not None} + if failed: + detail = "; ".join(f"rank {rank}: {why}" for rank, why in sorted(failed.items())) + raise RuntimeError(f"modeling_v2: other ranks failed to resolve a target ({detail})") + if len({o[0] for o in outcomes}) > 1: + detail = "; ".join( + f"rank {rank}: {o[0] or 'built-in (' + str(o[1]) + ')'}" + for rank, o in enumerate(outcomes) ) - return None + raise RuntimeError( + f"modeling_v2: ranks disagree on the implementation to build ({detail}); " + "every rank has to run the same code" + ) + + if mapping.rank == 0: + if target is not None: + logger.warning( + f"modeling_v2: building {_qualified(target)} on all {len(outcomes)} rank(s)" + ) + elif fallback is not None: + logger.warning(f"modeling_v2: {fallback}; using the built-in implementation") + return target diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/__init__.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/__init__.py new file mode 100644 index 000000000000..22de297f5244 --- /dev/null +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/__init__.py @@ -0,0 +1,13 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""The native-code fixture: a family with no model behind it. + +Its one target, ``scale_add__sm_80__default``, carries one native op so that CI +drives the whole native path -- the family library's build and packaging, the +loader's checks, the fake op, the GPU op -- on every GPU it has. No routing +module names it; tests import the target directly. + +It is also the smallest complete example of a target with native code; its C++ +half is ``cpp/tensorrt_llm/modeling_v2/fixture/`` and +``cpp/tensorrt_llm/modeling_v2/README.md`` walks through both. +""" diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_80__default/__init__.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_80__default/__init__.py new file mode 100644 index 000000000000..4a04b0fb8089 --- /dev/null +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_80__default/__init__.py @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Fixture / sm_80 / default: the native-code fixture's only target. + +Loading the package loads its native code: ``native_ops`` is +``torch.ops.mv2_fixture_scale_add_sm_80_default``, built from +``cpp/tensorrt_llm/modeling_v2/fixture/scaleAddSm80Default/``. A target with +native code starts this way, so that ``route`` importing the target is what +loads -- and checks -- its library. (Not ``ops``: that name is the target's +``ops/`` package, which would replace it on import.) +""" + +from tensorrt_llm._torch._experimental.modeling_v2._native import load_target_ops + +native_ops = load_target_ops(__name__) diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_80__default/fake_ops.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_80__default/fake_ops.py new file mode 100644 index 000000000000..454ba14dfcde --- /dev/null +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_80__default/fake_ops.py @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Fake (meta) implementations of this target's native ops. + +The loader imports this module right after it opens the family library, so a +fake for an op the library does not define fails the load. Every op the +target's C++ registers has a fake here; the native claims test checks that by +reading both sides. +""" + +import torch + +from tensorrt_llm._torch._experimental.modeling_v2._native import namespace_of + +_NS = namespace_of(__package__) + + +@torch.library.register_fake(f"{_NS}::scale_add") +def _(x, alpha, beta): + return torch.empty_like(x) diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_80__default/ops/__init__.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_80__default/ops/__init__.py new file mode 100644 index 000000000000..1059445198db --- /dev/null +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_80__default/ops/__init__.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Ops used only by this target. Shared ops stay in the catalog.""" diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_80__default/ops/elementwise/__init__.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_80__default/ops/elementwise/__init__.py new file mode 100644 index 000000000000..52a7a9daf028 --- /dev/null +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_80__default/ops/elementwise/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_80__default/ops/elementwise/scale_add.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_80__default/ops/elementwise/scale_add.py new file mode 100644 index 000000000000..456801e1d9b0 --- /dev/null +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_80__default/ops/elementwise/scale_add.py @@ -0,0 +1,56 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""`x * alpha + beta`: the native-code fixture's op, an entry like any other. + +The kernel is ``cpp/tensorrt_llm/modeling_v2/fixture/scaleAddSm80Default/``; the +op is reached through the target's ``native_ops`` namespace, which loading the +target package filled. Nothing here is specific to native code beyond that one name. +""" + +from __future__ import annotations + +import torch + +from tensorrt_llm._torch._experimental.modeling_v2.catalog._op import Arch, Cell, OpWrapper + +from ... import native_ops + + +class ScaleAdd(OpWrapper): + """`(x.float() * alpha + beta).to(x.dtype)`, elementwise, new tensor out. + + The product and the sum are each rounded to fp32 and the result once to + `x`'s dtype; the kernel does the same with no fused multiply-add, so the two + agree bitwise. + """ + + ARCHS = frozenset({Arch.SM_103}) + + CELLS: tuple[Cell, ...] = ( + Cell(why="fp32, one element", spec=dict(numel=1, dtype=torch.float32)), + Cell( + why="fp32, not a multiple of the block size", + spec=dict(numel=1000, dtype=torch.float32), + ), + Cell( + why="bf16, more elements than one grid covers", + spec=dict(numel=65535 * 256 + 3, dtype=torch.bfloat16), + ), + Cell(why="empty", spec=dict(numel=0, dtype=torch.bfloat16)), + ) + + note = """ + A fixture: it proves the native path works, not anything about a model. + """ + + def __call__(self, x: torch.Tensor, alpha: float, beta: float) -> torch.Tensor: + return native_ops.scale_add(x, alpha, beta) + + def reference(self, x: torch.Tensor, alpha: float, beta: float) -> torch.Tensor: + return (x.float() * alpha + beta).to(x.dtype) + + def is_valid(self, x: torch.Tensor, alpha: float, beta: float) -> None: + assert x.is_contiguous(), "the kernel indexes x as one contiguous run" + + +scale_add = ScaleAdd() diff --git a/tensorrt_llm/_torch/pyexecutor/model_loader.py b/tensorrt_llm/_torch/pyexecutor/model_loader.py index 67a3fbba4a6f..f967e92d7069 100644 --- a/tensorrt_llm/_torch/pyexecutor/model_loader.py +++ b/tensorrt_llm/_torch/pyexecutor/model_loader.py @@ -43,7 +43,7 @@ from tensorrt_llm.quantization.utils.fp4_utils import float4_e2m1x2 from ...llmapi.llm_args import LoadFormat -from .._experimental.modeling_v2 import modeling_v2_resolve +from .._experimental.modeling_v2 import modeling_v2_resolve_on_all_ranks from ..locality_domain.policy import LocalityDomainPolicy from ..model_config import ModelConfig from ..models import AutoModelForCausalLM @@ -2096,7 +2096,7 @@ def _load_and_validate_config( # modeling_v2 target reads the arguments as the engine will run with # them and never overrides those defaults. config = replace(config, - modeling_v2_target=modeling_v2_resolve( + modeling_v2_target=modeling_v2_resolve_on_all_ranks( config, self.llm_args)) if uses_mtp_head_checkpoint(self.spec_config): diff --git a/tests/integration/test_lists/test-db/l0_b300.yml b/tests/integration/test_lists/test-db/l0_b300.yml index 8ec539b91268..621c6cd9d79d 100644 --- a/tests/integration/test_lists/test-db/l0_b300.yml +++ b/tests/integration/test_lists/test-db/l0_b300.yml @@ -135,34 +135,16 @@ l0_b300: - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_chunked_prefill[quant_dtype=fp8-kv_cache_reuse=True-fp8kv=True-overlap_scheduler=True] # Cover nvbugs 6084445 - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_flashinfer[enable_chunked_prefill=False] - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_flashinfer[enable_chunked_prefill=True] - # modeling_v2 unit tests, one entry per catalog op test and one per model - # family that has unit tests, so CBTS can select the ones a change reaches. - # The whole-tree tests that need no GPU are in l0_cpu; the comm tests in - # l0_gb300_multi_gpus. - - unittest/_torch/modeling_v2/activation/test_modeling_v2_flashinfer_silu_and_mul.py - - unittest/_torch/modeling_v2/attention/test_modeling_v2_fused_qk_norm_rope.py - - unittest/_torch/modeling_v2/attention/test_modeling_v2_load_paged_kv_cache_for_mla.py - - unittest/_torch/modeling_v2/attention/test_modeling_v2_mla_rope_append_paged_kv_assign_q.py - - unittest/_torch/modeling_v2/attention/test_modeling_v2_mla_rope_generation.py - - unittest/_torch/modeling_v2/attention/test_modeling_v2_thop_attention.py - - unittest/_torch/modeling_v2/gemm/test_modeling_v2_bmm_out.py - - unittest/_torch/modeling_v2/gemm/test_modeling_v2_cublas_mm.py - - unittest/_torch/modeling_v2/gemm/test_modeling_v2_nvfp4_gemm.py - - unittest/_torch/modeling_v2/graph/test_modeling_v2_eager_on_graph.py - - unittest/_torch/modeling_v2/moe/test_modeling_v2_fp4_block_scale_moe_runner.py - - unittest/_torch/modeling_v2/moe/test_modeling_v2_mxe4m3_mxe2m1_block_scale_moe_runner.py - - unittest/_torch/modeling_v2/moe/test_modeling_v2_noaux_tc_op.py - - unittest/_torch/modeling_v2/norm/test_modeling_v2_flashinfer_fused_add_rmsnorm.py - - unittest/_torch/modeling_v2/norm/test_modeling_v2_flashinfer_rmsnorm.py - - unittest/_torch/modeling_v2/quantization/test_modeling_v2_fp4_quantize.py - - unittest/_torch/modeling_v2/quantization/test_modeling_v2_mxfp8_quantize.py - - unittest/_torch/modeling_v2/models/deepseek_v3 - - unittest/_torch/modeling_v2/models/qwen4_exp - - unittest/_torch/modeling_v2/test_modeling_v2_deepseek_weight_table.py - - unittest/_torch/modeling_v2/test_modeling_v2_mnnvl_workspace.py - - unittest/_torch/modeling_v2/test_modeling_v2_no_stale_claims.py - - unittest/_torch/modeling_v2/test_modeling_v2_target_contract.py - - unittest/_torch/modeling_v2/test_modeling_v2_weight_table.py + # modeling_v2 unit tests: the catalog (shared by every target) as one entry and + # each family as its own, so that CBTS can select one family's tests for a + # change confined to that family (jenkins/scripts/cbts/cbts/rules/modeling_v2.py). + # TIMEOUT measured for the whole directory: 3122 cases in 13m54s on one GB300 (the + # cpu_only files run on the CPU stage instead), so 30 per entry is at least 2.2x headroom. + - unittest/_torch/modeling_v2 --ignore=unittest/_torch/modeling_v2/comm --ignore=unittest/_torch/modeling_v2/models TIMEOUT (30) + - unittest/_torch/modeling_v2/models/deepseek_v3 TIMEOUT (30) + - unittest/_torch/modeling_v2/models/fixture TIMEOUT (30) + - unittest/_torch/modeling_v2/models/glm_moe_dsa TIMEOUT (30) + - unittest/_torch/modeling_v2/models/qwen4_exp TIMEOUT (30) - accuracy/modeling_v2/test_gpt_oss.py::TestModelingV2GptOss120bSm10xDefault::test_gsm8k[tp1] # qwen4_exp target, pre-merge: gsm8k; one MTP engine for gsm8k with the acceptance-length # floor (the only gate that sees a miscomputed draft layer), mmmu (the vision tower) and diff --git a/tests/integration/test_lists/test-db/l0_cpu.yml b/tests/integration/test_lists/test-db/l0_cpu.yml index b14442eab318..5b3ec00ab910 100644 --- a/tests/integration/test_lists/test-db/l0_cpu.yml +++ b/tests/integration/test_lists/test-db/l0_cpu.yml @@ -69,6 +69,9 @@ l0_cpu: # derivation and attention-call arguments, its routed-expert scalars and its weight table. # The rest of unittest/_torch/modeling_v2 needs a GPU. - unittest/_torch/modeling_v2/test_modeling_v2_claims.py + - unittest/_torch/modeling_v2/test_modeling_v2_native_claims.py + - unittest/_torch/modeling_v2/test_modeling_v2_native_loader.py + - unittest/_torch/modeling_v2/test_modeling_v2_rank_agreement.py - unittest/_torch/modeling_v2/test_modeling_v2_routing.py - unittest/_torch/modeling_v2/test_modeling_v2_deepseek_default_shards.py - unittest/_torch/modeling_v2/test_modeling_v2_gpt_oss_default_shards.py diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index d97277c60fd0..8b0251c968da 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -174,9 +174,13 @@ unittest/_torch/kernels/parallel/test_fp4_linear.py::test_fp4_gemm_bias_per_back unittest/_torch/kernels/parallel/test_fp4_linear.py::test_fp4_gemm_bias_per_backend[mnk3-cublaslt] SKIP (https://nvbugs/6581067) unittest/_torch/kernels/parallel/test_fp4_linear.py::test_fp4_gemm_bias_per_backend[mnk3-cutlass] SKIP (https://nvbugs/6581067) unittest/_torch/modeling/test_modeling_nemotron_h_multimodal.py::test_nemotron_nano_v2_vl_image_batch_equivalence SKIP (https://nvbugs/6758946) +unittest/_torch/modeling_v2 --ignore=unittest/_torch/modeling_v2/comm --ignore=unittest/_torch/modeling_v2/models SKIP (https://nvbugs/6853741) unittest/_torch/modeling_v2/comm/test_modeling_v2_allgather_op_matrix.py::test_allgather_op_matrix SKIP (https://nvbugs/6843474) unittest/_torch/modeling_v2/comm/test_modeling_v2_reducescatter_op_matrix.py::test_reducescatter_op_matrix SKIP (https://nvbugs/6843474) +unittest/_torch/modeling_v2/models/deepseek_v3 SKIP (https://nvbugs/6853741) +unittest/_torch/modeling_v2/models/glm_moe_dsa SKIP (https://nvbugs/6853741) unittest/_torch/modeling_v2/models/glm_moe_dsa/test_routing.py SKIP (https://nvbugs/6986044) +unittest/_torch/modeling_v2/models/qwen4_exp SKIP (https://nvbugs/6853741) unittest/_torch/modules/mamba/test_flashinfer_gdn_decode.py SKIP (https://nvbugs/6842831) unittest/_torch/modules/tests_lora_modules/test_nemotron_h_lora_sanity.py::TestNemotronHLoRA::test_lora_pp2_sanity SKIP (https://nvbugs/6428124) unittest/_torch/moe/test_moe_backend.py::test_moe_backend[act=Relu2-e60_k4_h2048_i1408-seq=8-dtype=torch.bfloat16-backend=TRTLLM-quant=NVFP4-routing=Renormalize] SKIP (https://nvbugs/5989912) diff --git a/tests/unittest/_torch/modeling_v2/models/fixture/scale_add__sm_80__default/elementwise/test_modeling_v2_scale_add.py b/tests/unittest/_torch/modeling_v2/models/fixture/scale_add__sm_80__default/elementwise/test_modeling_v2_scale_add.py new file mode 100644 index 000000000000..15b357cbc7e5 --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/fixture/scale_add__sm_80__default/elementwise/test_modeling_v2_scale_add.py @@ -0,0 +1,67 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""GPU test for the native-code fixture's scale_add entry. + +Importing the entry imports its target package, which loads the fixture family +library through ``_native.load_target_ops``: a pass here means the library was +built for this GPU, packaged, opened, matched the stock contract fingerprint and +registered its fake op -- before the first cell runs. +""" + +import zlib + +import pytest +import torch + +from tensorrt_llm._torch._experimental.modeling_v2.models.fixture.scale_add__sm_80__default import ( # noqa: E501 + native_ops, +) +from tensorrt_llm._torch._experimental.modeling_v2.models.fixture.scale_add__sm_80__default.ops.elementwise.scale_add import ( # noqa: E501 + scale_add as op, +) + +__extra_import_path__ = ["../../../.."] +from _validating import certifying, validating # noqa: E402,F401 — path above; fixture by name + +assert torch.cuda.is_available(), "scale_add requires a CUDA device" + +_ALPHA, _BETA = 1.5, -0.25 + + +def _input(spec, why): + g = torch.Generator(device="cuda").manual_seed(zlib.crc32(why.encode())) + return torch.randn(spec["numel"], generator=g, device="cuda").to(spec["dtype"]) + + +@pytest.mark.parametrize("cell", op.CELLS, ids=[c.why for c in op.CELLS]) +def test_certified_cells(cell, certifying) -> None: # noqa: F811 — pytest resolves the fixture by this name + certifying(op) + x = _input(cell.spec, cell.why) + with validating(op): + result = op(x, _ALPHA, _BETA) + expected = op.reference(x, _ALPHA, _BETA) + assert result.dtype == x.dtype and result.shape == x.shape + # Both sides round the product and the sum to fp32 separately, then once to + # the output dtype: the band is zero. + assert torch.equal(result, expected) + + +def test_is_valid_rejects_a_strided_input() -> None: + x = torch.randn(64, device="cuda")[::2] + with pytest.raises(AssertionError, match="contiguous"): + with validating(op): + op(x, _ALPHA, _BETA) + + +def test_the_op_rejects_an_unsupported_dtype() -> None: + with pytest.raises(RuntimeError, match="float32 or bfloat16"): + native_ops.scale_add(torch.ones(4, device="cuda", dtype=torch.float16), _ALPHA, _BETA) + + +def test_the_fake_op_describes_the_output() -> None: + from torch._subclasses.fake_tensor import FakeTensorMode + + with FakeTensorMode(): + x = torch.empty(10, 3, device="cuda", dtype=torch.bfloat16) + out = native_ops.scale_add(x, _ALPHA, _BETA) + assert out.shape == (10, 3) and out.dtype == torch.bfloat16 diff --git a/tests/unittest/_torch/modeling_v2/test_modeling_v2_claims.py b/tests/unittest/_torch/modeling_v2/test_modeling_v2_claims.py index b0f0d86d4c06..5f79f65e7115 100644 --- a/tests/unittest/_torch/modeling_v2/test_modeling_v2_claims.py +++ b/tests/unittest/_torch/modeling_v2/test_modeling_v2_claims.py @@ -121,9 +121,11 @@ def test_every_target_ships_both_products(): def test_targets_do_not_share_files(): """Isolation is the property being demonstrated; measure it. - Targets are allowed to import the catalog and nothing else of each - other's. A shared helper between two targets is the first step back to - the abstraction this package exists to avoid. + Targets are allowed to import the catalog, and the loader of their own + native code (``_native``), and nothing else of each other's. A shared + helper between two targets is the first step back to the abstraction this + package exists to avoid. The native half of the same rule is + ``test_modeling_v2_native_claims.py``. """ for arch in _ARCHS: for target_dir, name in _routed_targets(arch): @@ -133,10 +135,10 @@ def test_targets_do_not_share_files(): dots, tail = match.group(1), match.group(2) if len(dots) == 1: continue # sibling within the target - assert tail.startswith("catalog"), ( + assert tail.startswith(("catalog", "_native")), ( f"{name}: {source_file.name} reaches outside its own " f"directory for {tail!r}; targets may import the " - f"catalog and nothing else" + f"catalog and _native and nothing else" ) diff --git a/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_claims.py b/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_claims.py new file mode 100644 index 000000000000..b57113f35de4 --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_claims.py @@ -0,0 +1,413 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""The boundary between stock code and modeling_v2 native code, read off disk. + +modeling_v2 native code lives in ``cpp/tensorrt_llm/modeling_v2/`` and builds +one library per family that loads only when routing picks one of the family's +targets (``_native``). Three things rest on that staying true: ``off`` loading +nothing, a target never depending on a sibling, and CBTS testing a +family-only change with that family's tests alone +(``jenkins/scripts/cbts/cbts/rules/modeling_v2.py``). Each test here guards one +of the rules that keeps it true; none imports a target or needs a GPU. + +The paths are the source checkout's, found from this file: these are claims +about the tree a PR changes, C++ included, not about an installed wheel. +""" + +from __future__ import annotations + +import ast +import fnmatch +import re +import shutil +import subprocess +from pathlib import Path + +import pytest + +from tensorrt_llm._torch._experimental.modeling_v2._native import ( + arch_covers, + camel, + identifier, + torch_namespace, +) + +_REPO = Path(__file__).resolve().parents[4] +_NATIVE = _REPO / "cpp/tensorrt_llm/modeling_v2" +_PY = _REPO / "tensorrt_llm/_torch/_experimental/modeling_v2" +_MODELS = _PY / "models" +_CMAKE_MODULE = _REPO / "cpp/cmake/modules/modelingV2.cmake" +_TEST_DB = _REPO / "tests/integration/test_lists/test-db" + +pytestmark = [ + pytest.mark.cpu_only, + pytest.mark.skipif(not _NATIVE.is_dir(), reason="needs the source checkout"), +] + +_NATIVE_SUFFIXES = (".cu", ".cuh", ".h", ".hpp", ".inl", ".cpp") +# Native directories that are not families. +_NON_FAMILY_DIRS = {"common", "catalog"} +# A target name's sm segment that names a family of architectures. +_SM_GROUPS = {"10x": ((10, 0), (10, 3))} + + +# --- Reading the CMake declarations ------------------------------------------ + + +def _calls(text: str, function: str) -> list[list[str]]: + """Argument tokens of every ``function(...)`` call in a CMakeLists.txt.""" + text = re.sub(r"#.*", "", text) + return [m.group(1).split() for m in re.finditer(rf"\b{function}\s*\(([^)]*)\)", text)] + + +def _keyword_values(tokens: list[str], keyword: str) -> list[str]: + keywords = {"ARCHS", "SOURCES", "DEPS", "REQUIRES"} + values, active = [], False + for token in tokens: + if token in keywords: + active = token == keyword + elif active: + values.append(token) + return values + + +def _families() -> dict[str, Path]: + """family -> native family directory, from every family CMakeLists.txt.""" + families = {} + for cmake in sorted(_NATIVE.glob("*/CMakeLists.txt")): + calls = _calls(cmake.read_text(), "trtllm_mv2_family") + assert len(calls) == 1 and len(calls[0]) == 1, ( + f"{cmake.relative_to(_REPO)} must call trtllm_mv2_family() exactly once" + ) + families[calls[0][0]] = cmake.parent + return families + + +def _targets() -> list[tuple[str, str, list[str]]]: + """(family, target, tokens) of every trtllm_mv2_target call.""" + out = [] + for family_dir in _families().values(): + for tokens in _calls((family_dir / "CMakeLists.txt").read_text(), "trtllm_mv2_target"): + out.append((tokens[0], tokens[1], tokens[2:])) + return out + + +def _native_files(root: Path) -> list[Path]: + return sorted(p for p in root.rglob("*") if p.is_file() and p.suffix in _NATIVE_SUFFIXES) + + +def _contract_surface() -> list[str]: + lines = (_NATIVE / "contractSurface.txt").read_text().splitlines() + return [line.strip() for line in lines if line.strip() and not line.startswith("#")] + + +# --- Layout ------------------------------------------------------------------ + + +def test_every_native_family_is_a_python_family(): + for family, family_dir in _families().items(): + assert family_dir.name == camel(family), ( + f"family {family!r} must live in {_NATIVE.relative_to(_REPO)}/{camel(family)}" + ) + assert (_MODELS / family / "__init__.py").is_file(), ( + f"native family {family!r} has no Python family package models/{family}/" + ) + + +def test_every_native_target_is_a_python_target_that_loads_it(): + declared = set() + for family, target, _ in _targets(): + declared.add((family, target)) + package = _MODELS / family / target / "__init__.py" + assert package.is_file(), f"native target {family}/{target} has no Python target package" + assert "load_target_ops(__name__)" in package.read_text(), ( + f"{package.relative_to(_REPO)} must start by loading its native code: " + "native_ops = load_target_ops(__name__)" + ) + for family, family_dir in _families().items(): + for sub in sorted(p for p in family_dir.iterdir() if p.is_dir()): + if sub.name == "shared": + continue + assert any(camel(t) == sub.name for f, t in declared if f == family), ( + f"{sub.relative_to(_REPO)} is not the directory of any trtllm_mv2_target " + f"of family {family!r}" + ) + + +def test_python_targets_that_load_native_code_have_it(): + declared = {(family, target) for family, target, _ in _targets()} + for package in sorted(_MODELS.glob("*/*/__init__.py")): + if "load_target_ops(" not in package.read_text(): + continue + family, target = package.parent.parent.name, package.parent.name + assert (family, target) in declared, ( + f"{package.relative_to(_REPO)} loads native code, but no " + f"trtllm_mv2_target({family} {target} ...) declares it" + ) + + +def test_declared_archs_cover_the_sm_in_the_target_name(): + for family, target, tokens in _targets(): + archs = _keyword_values(tokens, "ARCHS") + sm = target.split("__")[1][len("sm_") :] + if sm in _SM_GROUPS: + needed = _SM_GROUPS[sm] + else: + assert sm.isdigit(), ( + f"{family}/{target}: sm segment 'sm_{sm}' is neither a number nor in _SM_GROUPS" + ) + needed = (divmod(int(sm), 10),) + for capability in needed: + assert any(arch_covers(a, capability) for a in archs), ( + f"{family}/{target} declares ARCHS {archs}, none of which runs on " + f"sm_{capability[0]}{capability[1]}, the architecture in its name" + ) + + +# --- What native code may reach ----------------------------------------------- + +_INCLUDE = re.compile(r'^\s*#\s*include\s*"([^"]+)"', re.MULTILINE) + + +def test_native_code_includes_only_its_own_code_and_the_contract_surface(): + surface = _contract_surface() + problems = [] + for family, family_dir in _families().items(): + for unit_dir in sorted(p for p in family_dir.iterdir() if p.is_dir()): + own = f"tensorrt_llm/modeling_v2/{family_dir.name}/{unit_dir.name}/" + allowed = (own, "tensorrt_llm/modeling_v2/common/") + if unit_dir.name != "shared": + allowed += (f"tensorrt_llm/modeling_v2/{family_dir.name}/shared/",) + for path in _native_files(unit_dir): + for include in _INCLUDE.findall(path.read_text()): + where = f'{path.relative_to(_REPO)}: "{include}"' + if include.startswith("tensorrt_llm/modeling_v2/"): + if not include.startswith(allowed): + problems.append(f"{where} reaches another target or family") + elif include.startswith("tensorrt_llm/"): + if not any(fnmatch.fnmatch(include, p) for p in surface): + problems.append(f"{where} is not in contractSurface.txt") + elif ".." in include: + problems.append(f"{where} leaves its directory") + assert not problems, "\n".join(problems) + + +def test_contract_surface_patterns_name_existing_headers(): + for pattern in _contract_surface(): + assert any((_REPO / root).glob(pattern) for root in ("cpp/include", "cpp")), ( + f"contractSurface.txt pattern {pattern!r} matches no header" + ) + + +_RAW_REGISTRATION = re.compile(r"\bTORCH_LIBRARY(?:_FRAGMENT|_IMPL)?\s*\(") + + +def test_native_code_registers_only_into_its_own_namespace(): + """Raw TORCH_LIBRARY* would let a file register into trtllm:: or a sibling's + namespace; the TRTLLM_MV2_ macros register into the unit's own only.""" + offenders = [ + str(path.relative_to(_REPO)) + for path in _native_files(_NATIVE) + if not path.is_relative_to(_NATIVE / "common") + and _RAW_REGISTRATION.search(path.read_text()) + ] + assert not offenders, ( + f"use TRTLLM_MV2_LIBRARY_FRAGMENT / TRTLLM_MV2_LIBRARY_IMPL, not TORCH_LIBRARY*: {offenders}" + ) + + +_OP_DEF = re.compile(r'\bm\.def\(\s*"(\w+)\(') + + +def test_every_native_op_has_a_fake(): + """fake_ops.py doubles as the Python side's list of a target's ops: the + loader imports it, so an op without a fake is one torch.compile and fake + tracing cannot see, and a fake without an op fails the load.""" + for family, target, _ in _targets(): + target_dir = _families()[family] / camel(target) + ops = sorted( + {op for p in _native_files(target_dir) for op in _OP_DEF.findall(p.read_text())} + ) + if not ops: + continue + fake_ops = _MODELS / family / target / "fake_ops.py" + assert fake_ops.is_file(), f"{family}/{target} defines {ops} but has no fake_ops.py" + text = fake_ops.read_text() + missing = [op for op in ops if f'::{op}"' not in text] + assert not missing, f"{fake_ops.relative_to(_REPO)} has no fake for {missing}" + + +# --- What stock code may reach ------------------------------------------------ + +_STOCK_PY_REFERENCE = re.compile(r"modeling_v2[./]models\b|torch\.ops\.mv2_") +_STOCK_CPP_REFERENCE = re.compile(r'#\s*include\s*"tensorrt_llm/modeling_v2/') +_STOCK_CMAKE_REFERENCE = re.compile(r"modeling_v2/\w") + + +def test_stock_python_does_not_reference_targets(): + offenders = [] + for path in sorted((_REPO / "tensorrt_llm").rglob("*.py")): + if path.is_relative_to(_PY): + continue + if _STOCK_PY_REFERENCE.search(path.read_text(errors="ignore")): + offenders.append(str(path.relative_to(_REPO))) + assert not offenders, ( + f"stock code imports or calls modeling_v2 targets: {offenders}. Stock reaches " + "modeling_v2 only through the resolver; a target hands stock a callable instead." + ) + + +def test_stock_native_code_does_not_reference_targets(): + offenders = [] + for root in ("cpp/tensorrt_llm", "cpp/include", "cpp/tests"): + for path in sorted((_REPO / root).rglob("*")): + if not path.is_file() or path.is_relative_to(_NATIVE): + continue + # Skip generated cubin sources; they include nothing of interest. + if path.suffix in _NATIVE_SUFFIXES and path.stat().st_size < (1 << 20): + if _STOCK_CPP_REFERENCE.search(path.read_text(errors="ignore")): + offenders.append(str(path.relative_to(_REPO))) + elif path.name == "CMakeLists.txt": + if _STOCK_CMAKE_REFERENCE.search(path.read_text()): + offenders.append(str(path.relative_to(_REPO))) + assert not offenders, ( + f"stock native code reaches into cpp/tensorrt_llm/modeling_v2: {offenders}" + ) + + +def test_stock_custom_ops_register_no_fakes_for_targets(): + offenders = [ + str(path.relative_to(_REPO)) + for path in sorted((_REPO / "tensorrt_llm/_torch/custom_ops").rglob("*.py")) + if re.search(r"register_fake\(\s*f?[\"']mv2_", path.read_text()) + ] + assert not offenders, f"target fakes belong in the target's fake_ops.py: {offenders}" + + +# --- Targets ------------------------------------------------------------------ + + +def test_no_op_probing(): + """A target calls exactly the ops it declares. Choosing a path by whether an + op exists means `require` no longer promises the gated code ran.""" + # hasattr, or getattr with a default: an absent op becomes a path choice + # instead of an error. The loader's two-argument getattr raises instead. + probe = re.compile(r"\bhasattr\(\s*torch\.ops\b|\bgetattr\(\s*torch\.ops[\w.]*\s*,[^,()]+,") + offenders = [ + str(path.relative_to(_REPO)) + for path in sorted(_PY.rglob("*.py")) + if probe.search(path.read_text()) + ] + assert not offenders, f"op probing in {offenders}" + + +def test_family_packages_have_no_side_effects(): + """Native code loads per target; a family package that imported anything + would load it for every target of the family, or for none of them.""" + for init in [_MODELS / "__init__.py", *sorted(_MODELS.glob("*/__init__.py"))]: + body = ast.parse(init.read_text()).body + for node in body: + docstring = isinstance(node, ast.Expr) and isinstance(node.value, ast.Constant) + future = isinstance(node, ast.ImportFrom) and node.module == "__future__" + assert docstring or future, ( + f"{init.relative_to(_REPO)}:{node.lineno} -- a family package holds a docstring only" + ) + + +def test_modeling_v2_defaults_to_off(): + """`off` loads nothing, which is what lets CBTS run only modeling_v2 tests + for a modeling_v2 change. Changing the default changes that rule too.""" + from tensorrt_llm.llmapi.llm_args import TorchLlmArgs + + assert TorchLlmArgs.model_fields["modeling_v2"].default == "off" + + +# --- Test lists ---------------------------------------------------------------- + +_MV2_ENTRY = re.compile(r"^\s*-\s+((?:unittest/_torch/modeling_v2|accuracy/modeling_v2)\S*)(.*)$") + + +def _test_db_entries() -> list[tuple[str, str, str]]: + entries = [] + for yml in sorted(_TEST_DB.glob("*.yml")): + for line in yml.read_text().splitlines(): + match = _MV2_ENTRY.match(line) + if match: + entries.append((yml.name, match.group(1), match.group(2))) + return entries + + +def test_test_lists_keep_family_tests_in_their_own_entries(): + """CBTS selects one family's tests by entry; an entry covering models/ from + above would hide a family's unit tests inside the catalog's.""" + models = "unittest/_torch/modeling_v2/models" + for yml, target, options in _test_db_entries(): + if target in ("unittest/_torch/modeling_v2", "unittest/_torch/modeling_v2/"): + assert f"--ignore={models}" in options, ( + f"{yml}: {target} must carry --ignore={models}; list each family on its own" + ) + assert target != models, f"{yml}: list each family under {models}/ on its own" + + +def test_family_unit_entries_point_at_tests(): + """A family entry naming a directory with no tests is a collection error in CI.""" + for yml, target, _ in _test_db_entries(): + if target.startswith("unittest/_torch/modeling_v2/models/"): + path = _REPO / "tests" / target + assert path.exists() and (path.is_file() or any(path.rglob("test_*.py"))), ( + f"{yml}: {target} names no test file or directory with tests" + ) + + +def test_families_with_native_code_have_test_entries(): + targets = {target for _, target, _ in _test_db_entries()} + for family in _families(): + unit = f"unittest/_torch/modeling_v2/models/{family}" + accuracy = f"accuracy/modeling_v2/test_{family}.py" + assert any(t == unit or t.startswith((unit + "/", accuracy)) for t in targets), ( + f"family {family!r} has native code but no test-db entry under {unit} or {accuracy}" + ) + + +def test_accuracy_entries_name_a_family(): + families = {p.name for p in _MODELS.iterdir() if (p / "__init__.py").is_file()} + for yml, target, _ in _test_db_entries(): + match = re.match(r"accuracy/modeling_v2/test_(\w+)\.py", target) + if match: + assert match.group(1) in families, ( + f"{yml}: {target} is named after no modeling_v2 family; CBTS selects " + "accuracy gates by family name" + ) + + +# --- CMake and Python agree ------------------------------------------------------ + + +@pytest.mark.skipif(shutil.which("cmake") is None, reason="needs cmake") +def test_cmake_and_python_agree_on_names_and_architectures(tmp_path): + """The loader and CBTS compute directory, namespace and architecture + coverage in Python; the build computes them in modelingV2.cmake.""" + names = ["fixture", "minimax_m3", "scale_add__sm_80__default", "r1_0528_nvfp4__sm_10x__default"] + script = tmp_path / "probe.cmake" + lines = [f'include("{_CMAKE_MODULE}")'] + for name in names: + lines += [ + f'_trtllm_mv2_camel(c "{name}")', + f'_trtllm_mv2_ident(i "{name}")', + f'message(STATUS "name {name} ${{c}} ${{i}}")', + ] + lines += [ + "_trtllm_mv2_resolve_archs(r DECLARED 80 90a 100f 103a 120f REQUESTED 86 90 103 107)", + 'message(STATUS "archs ${r}")', + ] + script.write_text("\n".join(lines) + "\n") + output = subprocess.run( + ["cmake", "-P", str(script)], capture_output=True, text=True, check=True + ).stdout + for name in names: + assert f"name {name} {camel(name)} {identifier(name)}" in output, output + assert "archs 90a-real;100f-real;103a-real" in output, output + # The family namespace the build registers is the loader's. + assert torch_namespace("fixture", "scale_add__sm_80__default") == ( + "mv2_fixture_scale_add_sm_80_default" + ) diff --git a/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_loader.py b/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_loader.py new file mode 100644 index 000000000000..927f78289104 --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_loader.py @@ -0,0 +1,234 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""What ``_native`` does with a manifest, and that the installed libraries load. + +The first half drives the loader's decisions with synthetic manifests: which +outcomes are ``NoTarget`` (this build or this device cannot run the target, so +``auto`` keeps the built-in model) and which are ``NativeLoadError`` (the +installation is broken, which is never a fallback). Nothing is opened. + +The second half is the load smoke test of the installed wheel: every family +library the manifest lists opens in a fresh process, reports the stock contract +fingerprint, and defines no op its target's fake_ops.py does not fake. A family +library aborts the process from a static initializer if two libraries claim one +torch namespace, so each one is opened in a child process. +""" + +from __future__ import annotations + +import json +import subprocess +import sys +import textwrap +from pathlib import Path + +import pytest + +from tensorrt_llm._torch._experimental.modeling_v2 import _native +from tensorrt_llm._torch._experimental.modeling_v2._native import ( + NativeLoadError, + arch_covers, + available_targets, + camel, + identifier, + load_target_ops, + namespace_of, + read_manifest, + split_target_package, + torch_namespace, +) +from tensorrt_llm._torch._experimental.modeling_v2._router_index import NoTarget + +pytestmark = pytest.mark.cpu_only + +_FIXTURE = "tensorrt_llm._torch._experimental.modeling_v2.models.fixture.scale_add__sm_80__default" + + +def _manifest(status="built", archs=("100f-real",), library="libtrtllm_mv2_fixture.so", **extra): + return { + "schema": _native.MANIFEST_SCHEMA, + "enabled": True, + "requested_archs": ["100", "103"], + "families": { + "fixture": { + "library": library, + "namespace": "mv2_fixture", + "shared": None, + "targets": { + "scale_add__sm_80__default": { + "status": status, + "namespace": "mv2_fixture_scale_add_sm_80_default", + "declared_archs": ["100f"], + "archs": list(archs), + "reason": extra.get("reason", ""), + } + }, + } + }, + } + + +# --- Naming ---------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "name, expected", + [ + ("fixture", "fixture"), + ("minimax_m3", "minimaxM3"), + ("glm_moe_dsa", "glmMoeDsa"), + ("minimax_m3_nvfp4__sm_107__tp2", "minimaxM3Nvfp4Sm107Tp2"), + ("r1_0528_nvfp4__sm_10x__default", "r10528Nvfp4Sm10xDefault"), + ], +) +def test_camel_is_the_native_directory(name, expected): + assert camel(name) == expected + + +def test_namespaces_collapse_double_underscores(): + assert identifier("scale_add__sm_80__default") == "scale_add_sm_80_default" + assert torch_namespace("fixture") == "mv2_fixture" + assert namespace_of(_FIXTURE) == "mv2_fixture_scale_add_sm_80_default" + assert split_target_package(_FIXTURE) == ("fixture", "scale_add__sm_80__default") + with pytest.raises(ValueError): + split_target_package("tensorrt_llm._torch.models") + + +@pytest.mark.parametrize( + "arch, capability, runs", + [ + ("100f-real", (10, 0), True), + ("100f-real", (10, 3), True), + ("100f-real", (10, 7), True), + ("100f-real", (12, 0), False), + ("103a-real", (10, 3), True), + ("103a-real", (10, 0), False), + ("80-real", (8, 6), True), + ("80-real", (9, 0), False), + ("90a-real", (9, 0), True), + ("120f", (12, 1), True), + ], +) +def test_arch_coverage(arch, capability, runs): + assert arch_covers(arch, capability) is runs + + +# --- Decisions ----------------------------------------------------------------- + + +def test_a_missing_manifest_is_a_broken_installation(tmp_path): + with pytest.raises(NativeLoadError, match="no modeling_v2"): + read_manifest(tmp_path / "absent.json") + + +def test_a_manifest_from_another_schema_is_refused(tmp_path): + path = tmp_path / "manifest.json" + path.write_text(json.dumps({**_manifest(), "schema": 999})) + with pytest.raises(NativeLoadError, match="schema"): + read_manifest(path) + + +def test_a_build_without_modeling_v2_is_no_target(): + manifest = { + "schema": _native.MANIFEST_SCHEMA, + "enabled": False, + "reason": "off", + "families": {}, + } + with pytest.raises(NoTarget, match="no modeling_v2 libraries"): + load_target_ops(_FIXTURE, manifest) + + +@pytest.mark.parametrize("status", ["skipped", "excluded"]) +def test_a_target_this_build_skipped_is_no_target(status): + manifest = _manifest(status=status, archs=(), reason="needs sm_107") + with pytest.raises(NoTarget, match="needs sm_107"): + load_target_ops(_FIXTURE, manifest) + + +def test_a_target_the_manifest_does_not_know_is_a_broken_installation(): + manifest = _manifest() + manifest["families"]["fixture"]["targets"] = {} + with pytest.raises(NativeLoadError, match="different builds"): + load_target_ops(_FIXTURE, manifest) + with pytest.raises(NativeLoadError, match="different builds"): + load_target_ops(_FIXTURE, {**manifest, "families": {}}) + + +def test_a_device_the_target_was_not_built_for_is_no_target(monkeypatch): + monkeypatch.setattr(_native, "_device_capability", lambda: (8, 0)) + with pytest.raises(NoTarget, match="does not run on this sm_80"): + load_target_ops(_FIXTURE, _manifest(archs=("100f-real",))) + + +def test_a_listed_library_that_is_missing_is_a_broken_installation(monkeypatch, tmp_path): + monkeypatch.setattr(_native, "_device_capability", lambda: None) + monkeypatch.setattr(_native, "_LIBS_DIR", tmp_path) + with pytest.raises(NativeLoadError, match="missing"): + load_target_ops(_FIXTURE, _manifest()) + + +def test_available_targets_lists_every_target(): + rows = available_targets(_manifest(status="skipped", archs=(), reason="needs sm_107")) + assert rows == [ + { + "family": "fixture", + "target": "scale_add__sm_80__default", + "status": "skipped", + "archs": [], + "reason": "needs sm_107", + "library": "libtrtllm_mv2_fixture.so", + } + ] + + +# --- The installed wheel --------------------------------------------------------- + +_LOAD_ONE_FAMILY = textwrap.dedent( + """ + import json, sys + import torch + import tensorrt_llm # noqa: F401 -- loads libth_common.so + from tensorrt_llm._torch._experimental.modeling_v2 import _native + + family = sys.argv[1] + _native.load_family_library(family) + schemas = [s.name for s in torch._C._jit_get_all_schemas()] + print(json.dumps(sorted(n for n in schemas if n.startswith("mv2_")))) + """ +) + + +def _installed_manifest(): + path = _native._LIBS_DIR / _native.MANIFEST_NAME + assert path.is_file(), ( + f"{path} is missing: scripts/build_wheel.py installs it with the family libraries" + ) + return read_manifest(path) + + +def test_every_built_family_library_loads(): + manifest = _installed_manifest() + families = list(_native.built_families(manifest)) + if not families: + pytest.skip(f"this build has no modeling_v2 family library: {manifest.get('reason')}") + models = Path(_native.__file__).resolve().parent / "models" + for family in families: + result = subprocess.run( + [sys.executable, "-c", _LOAD_ONE_FAMILY, family], + capture_output=True, + text=True, + ) + assert result.returncode == 0, ( + f"libtrtllm_mv2_{family}.so failed to load:\n{result.stdout}\n{result.stderr}" + ) + ops = json.loads(result.stdout.strip().splitlines()[-1]) + for target, unit in manifest["families"][family]["targets"].items(): + if unit["status"] != "built": + continue + prefix = f"{unit['namespace']}::" + target_ops = [op[len(prefix) :] for op in ops if op.startswith(prefix)] + fake_ops = models / family / target / "fake_ops.py" + text = fake_ops.read_text() if fake_ops.is_file() else "" + unfaked = [op for op in target_ops if f'::{op}"' not in text] + assert not unfaked, f"{family}/{target} defines {unfaked} with no fake in fake_ops.py" diff --git a/tests/unittest/_torch/modeling_v2/test_modeling_v2_rank_agreement.py b/tests/unittest/_torch/modeling_v2/test_modeling_v2_rank_agreement.py new file mode 100644 index 000000000000..c5cda015a399 --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/test_modeling_v2_rank_agreement.py @@ -0,0 +1,113 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""``modeling_v2_resolve_on_all_ranks``: every rank builds the same code, or every rank fails. + +Each rank routes on its own facts -- its device, whether this installation built +the target's native code -- so ranks can disagree, and a disagreement surfaces +as a hang in the first collective unless the resolver catches it. These tests +stand in for the other ranks with a fake ``Distributed`` whose ``allgather`` +returns their outcomes. +""" + +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +from tensorrt_llm._torch._experimental.modeling_v2 import _router_index +from tensorrt_llm._torch._experimental.modeling_v2._router_index import ( + modeling_v2_resolve_on_all_ranks, +) +from tensorrt_llm._torch.distributed import communicator + +pytestmark = pytest.mark.cpu_only + + +class Target: + """Stands in for a target class.""" + + +_TARGET = f"{Target.__module__}.{Target.__qualname__}" + + +class _FakeDist: + def __init__(self, others): + self.others = others + self.calls = 0 + + def allgather(self, outcome): + self.calls += 1 + return [outcome, *self.others] + + +def _config(rank=0, world_size=2): + return SimpleNamespace(mapping=SimpleNamespace(rank=rank, world_size=world_size)) + + +def _args(mode): + return SimpleNamespace(modeling_v2=mode) + + +@pytest.fixture +def dist(monkeypatch): + fake = _FakeDist(others=[]) + monkeypatch.setattr(communicator.Distributed, "get", staticmethod(lambda mapping: fake)) + return fake + + +def _route_to(monkeypatch, target=None, fallback=None, error=None): + def route(config, llm_args, mode): + if error is not None: + raise error + return target, fallback + + monkeypatch.setattr(_router_index, "_route", route) + + +def test_off_exchanges_nothing(monkeypatch, dist): + _route_to(monkeypatch, error=AssertionError("routing ran under off")) + assert modeling_v2_resolve_on_all_ranks(_config(), _args("off")) is None + assert dist.calls == 0 + + +def test_one_rank_exchanges_nothing(monkeypatch, dist): + _route_to(monkeypatch, target=Target) + assert modeling_v2_resolve_on_all_ranks(_config(world_size=1), _args("auto")) is Target + assert dist.calls == 0 + + +def test_agreeing_ranks_build_the_target(monkeypatch, dist): + dist.others = [(_TARGET, None, None)] + _route_to(monkeypatch, target=Target) + assert modeling_v2_resolve_on_all_ranks(_config(), _args("require")) is Target + assert dist.calls == 1 + + +def test_agreeing_fallbacks_build_the_builtin_model(monkeypatch, dist): + dist.others = [(None, "no target fits 'X' (needs sm_107)", None)] + _route_to(monkeypatch, fallback="no target fits 'X' (needs sm_107)") + assert modeling_v2_resolve_on_all_ranks(_config(), _args("auto")) is None + + +def test_disagreeing_ranks_all_fail(monkeypatch, dist): + dist.others = [(None, "no target fits 'X' (native code not built)", None)] + _route_to(monkeypatch, target=Target) + with pytest.raises(RuntimeError, match="ranks disagree"): + modeling_v2_resolve_on_all_ranks(_config(), _args("auto")) + + +def test_another_ranks_error_fails_this_rank(monkeypatch, dist): + dist.others = [(None, None, "NativeLoadError: libtrtllm_mv2_x.so cannot be loaded")] + _route_to(monkeypatch, target=Target) + with pytest.raises(RuntimeError, match="rank 1: NativeLoadError"): + modeling_v2_resolve_on_all_ranks(_config(), _args("auto")) + + +def test_this_ranks_error_is_raised_after_the_exchange(monkeypatch, dist): + """Raising before the exchange would leave the other ranks waiting in it.""" + dist.others = [(_TARGET, None, None)] + _route_to(monkeypatch, error=ValueError("modeling_v2='require', but no target fits")) + with pytest.raises(ValueError, match="no target fits"): + modeling_v2_resolve_on_all_ranks(_config(), _args("require")) + assert dist.calls == 1 diff --git a/tests/unittest/scripts/cbts/test_modeling_v2_rule.py b/tests/unittest/scripts/cbts/test_modeling_v2_rule.py new file mode 100644 index 000000000000..0c031fb83536 --- /dev/null +++ b/tests/unittest/scripts/cbts/test_modeling_v2_rule.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests for CBTS ModelingV2Rule: per-family selection over Python and native paths.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +__extra_import_path__ = ["~/jenkins/scripts/cbts"] +from cbts.blocks import Block, Stage, YAMLIndex +from cbts.rules.base import PRInputs +from cbts.rules.modeling_v2 import ModelingV2Rule, _is_mv2_claim, _snake + +pytestmark = pytest.mark.cpu_only + +_PY = "tensorrt_llm/_torch/_experimental/modeling_v2" +_CPP = "cpp/tensorrt_llm/modeling_v2" +_CATALOG_UNIT = ( + "unittest/_torch/modeling_v2 --ignore=unittest/_torch/modeling_v2/comm " + "--ignore=unittest/_torch/modeling_v2/models TIMEOUT (30)" +) +_FIXTURE_UNIT = "unittest/_torch/modeling_v2/models/fixture" +_GPT_OSS_UNIT = "unittest/_torch/modeling_v2/models/gpt_oss" +_GPT_OSS_ACCURACY = "accuracy/modeling_v2/test_gpt_oss.py::TestGptOss::test_gsm8k[tp1]" +_CLAIMS = "unittest/_torch/modeling_v2/test_modeling_v2_claims.py" + + +@pytest.fixture +def repo_root(tmp_path: Path) -> Path: + for family in ("fixture", "gpt_oss", "minimax_m3"): + package = tmp_path / _PY / "models" / family + package.mkdir(parents=True) + (package / "__init__.py").write_text('"""Family."""\n') + return tmp_path + + +@pytest.fixture +def rule(repo_root: Path) -> ModelingV2Rule: + yaml_index = YAMLIndex() + yaml_index.blocks = [ + Block( + yaml_stem="l0_b300", + block_index=0, + condition={}, + tests=[ + "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4", + _CATALOG_UNIT, + _FIXTURE_UNIT, + _GPT_OSS_UNIT, + _GPT_OSS_ACCURACY, + ], + ), + Block(yaml_stem="l0_cpu", block_index=0, condition={}, tests=["unittest/tools", _CLAIMS]), + ] + stages = { + "B300-PyTorch-1": Stage("B300-PyTorch-1", "l0_b300", "aarch64", 1, 2), + "CPU-Generic-x86-1": Stage("CPU-Generic-x86-1", "l0_cpu", "x86_64", 1, 1), + } + return ModelingV2Rule(yaml_index, stages, repo_root=repo_root) + + +def _apply(rule: ModelingV2Rule, *files: str): + return rule.apply(PRInputs(changed_files=list(files), diffs={})) + + +def _selected(result) -> set[str]: + return { + entry + for prefixes in result.block_filters.values() + for entries in prefixes.values() + for entry in entries + } + + +def test_claims_python_and_native_sources_but_not_docs_or_stock(): + assert _is_mv2_claim(f"{_PY}/models/fixture/scale_add__sm_80__default/__init__.py") + assert _is_mv2_claim(f"{_CPP}/fixture/scaleAddSm80Default/kernels/scaleAdd.cu") + assert _is_mv2_claim(f"{_CPP}/contractSurface.txt") + assert _is_mv2_claim("cpp/cmake/modules/modelingV2.cmake") + assert not _is_mv2_claim(f"{_CPP}/README.md") + assert not _is_mv2_claim("cpp/tensorrt_llm/thop/modelingV2ContractOp.cpp") + assert not _is_mv2_claim("tensorrt_llm/_torch/pyexecutor/model_loader.py") + + +def test_family_of_reads_both_trees(rule): + assert rule.family_of(f"{_PY}/models/minimax_m3/routing.py") == "minimax_m3" + assert rule.family_of(f"{_CPP}/minimaxM3/shared/attnMega.cu") == "minimax_m3" + assert rule.family_of(f"{_PY}/catalog/_op.py") is None + assert rule.family_of(f"{_PY}/models/__init__.py") is None + assert rule.family_of(f"{_CPP}/common/torchLibrary.h") is None + assert rule.family_of(f"{_CPP}/CMakeLists.txt") is None + + +def test_snake_inverts_camel_without_a_repo_root(): + assert _snake("glmMoeDsa") == "glm_moe_dsa" + assert _snake("minimaxM3") == "minimax_m3" + assert _snake("kimiK3Vl") == "kimi_k3_vl" + + +@pytest.mark.parametrize( + "changed", + [ + f"{_PY}/models/fixture/scale_add__sm_80__default/ops/elementwise/scale_add.py", + f"{_CPP}/fixture/scaleAddSm80Default/kernels/scaleAdd.cu", + f"{_CPP}/fixture/CMakeLists.txt", + ], +) +def test_a_family_change_selects_that_family_and_the_checks(rule, changed): + result = _apply(rule, changed) + assert result.scope == "modelingv2only" + assert _selected(result) == {_FIXTURE_UNIT, _CLAIMS} + assert result.sanity_relevant is False + + +def test_two_families_union(rule): + result = _apply( + rule, + f"{_PY}/models/fixture/scale_add__sm_80__default/__init__.py", + f"{_PY}/models/gpt_oss/routing.py", + ) + assert _selected(result) == {_FIXTURE_UNIT, _GPT_OSS_UNIT, _GPT_OSS_ACCURACY, _CLAIMS} + + +@pytest.mark.parametrize( + "changed", + [ + f"{_PY}/catalog/_op.py", + f"{_PY}/_native.py", + f"{_CPP}/common/torchLibrary.h", + f"{_CPP}/contractSurface.txt", + "cpp/cmake/modules/modelingV2.cmake", + ], +) +def test_shared_modeling_v2_code_selects_everything_of_modeling_v2(rule, changed): + result = _apply(rule, changed) + assert result.scope == "modelingv2only" + assert _selected(result) == { + _CATALOG_UNIT, + _FIXTURE_UNIT, + _GPT_OSS_UNIT, + _GPT_OSS_ACCURACY, + _CLAIMS, + } + + +def test_a_family_with_no_test_entry_falls_back(rule): + result = _apply(rule, f"{_CPP}/minimaxM3/shared/attnMega.cu") + assert result.scope is None + assert "minimax_m3" in result.reason + + +def test_stock_changes_are_not_claimed(rule): + assert _apply(rule, "tensorrt_llm/_torch/pyexecutor/model_loader.py") is None From 80f93f868f955c867e820a01fa275a0b16768ad6 Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Sun, 25 Oct 2026 22:40:09 -0700 Subject: [PATCH 02/20] [None][feat] modeling_v2: build, package and load family libraries like stock's optional libraries Align the per-family native libraries with how the stock tree already handles its optional torch-op libraries (flash_mla, deep_ep, th_common): - Names: libth_modeling_v2_.so, cpp/cmake/modules/modeling_v2.cmake, add_modeling_v2_family/_shared/_target/_family_libraries, a BUILD_MODELING_V2 option; the stock hook is one add_subdirectory next to thop. - Packaging: each family build directory holds cuda_architectures.txt; build_wheel.py installs the family library only when it is non-empty and requires exactly one, as for flash_mla and deep_ep. The JSON manifest is removed. - Built targets: each library answers torch.ops.mv2_.built_targets() from its generated registration code, the way th_common and deep_ep answer build-time feature queries. - Loading: a library absent from the installation is NoTarget, as a missing optional module is ImportError; one that is present but fails to load is NativeLoadError. The device check is left to routing. - The contract fingerprint (th_common op, generated header, CMake module) is removed: stock and family libraries are built from one tree, and setup.py's NATIVE_BUILD_INPUTS already covers precompiled-wheel skew. Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- .cmake-format.json | 6 +- .github/CODEOWNERS | 3 +- cpp/CMakeLists.txt | 6 +- cpp/cmake/modules/modelingV2.cmake | 833 ------------------ cpp/cmake/modules/modelingV2Fingerprint.cmake | 66 -- cpp/cmake/modules/modeling_v2.cmake | 638 ++++++++++++++ cpp/tensorrt_llm/CMakeLists.txt | 6 +- cpp/tensorrt_llm/modeling_v2/CMakeLists.txt | 42 +- cpp/tensorrt_llm/modeling_v2/README.md | 72 +- .../modeling_v2/common/familyLibrary.version | 2 +- .../modeling_v2/common/namespace.h | 6 +- .../modeling_v2/common/torchLibrary.h | 6 +- .../modeling_v2/contractSurface.txt | 11 +- .../modeling_v2/fixture/CMakeLists.txt | 6 +- cpp/tensorrt_llm/thop/CMakeLists.txt | 10 +- .../thop/modelingV2ContractOp.cpp | 46 - jenkins/scripts/cbts/README.md | 4 +- jenkins/scripts/cbts/cbts/rules/README.md | 6 +- .../scripts/cbts/cbts/rules/modeling_v2.py | 19 +- scripts/build_wheel.py | 36 +- setup.py | 3 +- .../_experimental/modeling_v2/README.md | 14 +- .../_experimental/modeling_v2/_native.py | 268 ++---- .../elementwise/test_modeling_v2_scale_add.py | 4 +- .../test_modeling_v2_native_claims.py | 42 +- .../test_modeling_v2_native_loader.py | 201 ++--- .../test_modeling_v2_rank_agreement.py | 2 +- .../scripts/cbts/test_modeling_v2_rule.py | 6 +- 28 files changed, 922 insertions(+), 1442 deletions(-) delete mode 100644 cpp/cmake/modules/modelingV2.cmake delete mode 100644 cpp/cmake/modules/modelingV2Fingerprint.cmake create mode 100644 cpp/cmake/modules/modeling_v2.cmake delete mode 100644 cpp/tensorrt_llm/thop/modelingV2ContractOp.cpp diff --git a/.cmake-format.json b/.cmake-format.json index 194be4f77f9c..21bf59072b21 100644 --- a/.cmake-format.json +++ b/.cmake-format.json @@ -26,10 +26,10 @@ "ARCHS": "+" } }, - "trtllm_mv2_family": { + "add_modeling_v2_family": { "pargs": 1 }, - "trtllm_mv2_shared": { + "add_modeling_v2_shared": { "pargs": 1, "kwargs": { "ARCHS": "+", @@ -38,7 +38,7 @@ "REQUIRES": "+" } }, - "trtllm_mv2_target": { + "add_modeling_v2_target": { "pargs": 2, "kwargs": { "ARCHS": "+", diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 6bf54f3e0e2a..aef368040f09 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -390,8 +390,7 @@ # modeling_v2 native code: the modeling_v2 owners for the targets, the kernels # owners because it is CUDA and links against the stock libraries. /cpp/tensorrt_llm/modeling_v2 @tianyuxbear @Wanli-Jiang @WeiHaocheng @litaotju @NVIDIA/trt-llm-kernels-devs -/cpp/cmake/modules/modelingV2.cmake @tianyuxbear @Wanli-Jiang @WeiHaocheng @litaotju @NVIDIA/trt-llm-kernels-devs -/cpp/cmake/modules/modelingV2Fingerprint.cmake @tianyuxbear @Wanli-Jiang @WeiHaocheng @litaotju @NVIDIA/trt-llm-kernels-devs +/cpp/cmake/modules/modeling_v2.cmake @tianyuxbear @Wanli-Jiang @WeiHaocheng @litaotju @NVIDIA/trt-llm-kernels-devs /tests/unittest/_torch/modeling_v2 @tianyuxbear @Wanli-Jiang @WeiHaocheng @litaotju /tests/integration/defs/accuracy/modeling_v2 @tianyuxbear @Wanli-Jiang @WeiHaocheng @litaotju # The two catalog categories whose contracts state kernel behaviour the diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 2074f7122614..064f80b4f5e3 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -49,8 +49,10 @@ option(BUILD_DEEP_EP "Build the Deep EP module" ON) option(BUILD_NCCL_EXTENSIONS "Build the nccl-extensions Python wheel" ON) option(BUILD_DEEP_GEMM "Build the DeepGEMM module" ON) option(BUILD_FLASH_MLA "Build the FlashMLA module" ON) -option(BUILD_MODELING_V2 - "Build the modeling_v2 per-family native libraries (libtrtllm_mv2_*)" ON) +option( + BUILD_MODELING_V2 + "Build the modeling_v2 per-family torch-op libraries (libth_modeling_v2_*)" + ON) set(TRTLLM_MODELING_V2_TARGETS "all" CACHE diff --git a/cpp/cmake/modules/modelingV2.cmake b/cpp/cmake/modules/modelingV2.cmake deleted file mode 100644 index 9904cc64728e..000000000000 --- a/cpp/cmake/modules/modelingV2.cmake +++ /dev/null @@ -1,833 +0,0 @@ -# -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -# All rights reserved. SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. -# - -#[[ -Native (C++/CUDA) code of modeling_v2 targets. - -Each model family under cpp/tensorrt_llm/modeling_v2// builds into -one MODULE library, libtrtllm_mv2_.so. Nothing in it is linked into -libtensorrt_llm.so or libth_common.so; the Python loader -(tensorrt_llm/_torch/_experimental/modeling_v2/_native.py) opens it only when -routing picks one of the family's targets. cpp/tensorrt_llm/modeling_v2/README.md -is the guide for authors. - -A family's CMakeLists.txt uses three functions. Names are the Python package -names (snake_case); directories are their camelCase form: - - trtllm_mv2_family() - trtllm_mv2_shared( ARCHS ... SOURCES ... [DEPS ...] [REQUIRES ...]) - trtllm_mv2_target( ARCHS ... SOURCES ... [DEPS ...] [REQUIRES ...]) - -ARCHS NN, NNa or NNf. Each is intersected with the build's requested - architectures; a target with an empty intersection is not built and - the manifest says why. No placeholder kernels are generated. -SOURCES Paths or globs relative to the target directory (shared/ for - trtllm_mv2_shared). .cu files are compiled for ARCHS without torch; - .cpp files are the torch bindings. -DEPS From a closed list: cublas, cublaslt, cuda_driver, internal_cutlass, - nccl, trtllm_gen_fmha_interface. -REQUIRES Build features: MULTI_DEVICE, NVSHMEM. Unmet requirements skip the - target. - -The stock side of the build calls trtllm_mv2_define_contract_fingerprint() -before thop, then adds cpp/tensorrt_llm/modeling_v2, which calls -trtllm_mv2_finalize() after the families. -#]] - -include_guard(GLOBAL) - -set(_TRTLLM_MV2_MODULE_DIR "${CMAKE_CURRENT_LIST_DIR}") -set(_TRTLLM_MV2_KNOWN_DEPS cublas cublaslt cuda_driver internal_cutlass nccl - trtllm_gen_fmha_interface) -set(_TRTLLM_MV2_KNOWN_REQUIRES MULTI_DEVICE NVSHMEM) -set(_TRTLLM_MV2_MANIFEST_SCHEMA 1) - -# --- Naming ------------------------------------------------------------------ - -# camelCase form of a snake_case name: minimax_m3_nvfp4__sm_107__tp2 -> -# minimaxM3Nvfp4Sm107Tp2. The same mapping as camel() in the Python checks. -function(_trtllm_mv2_camel out name) - string(REPLACE "_" ";" parts "${name}") - set(result "") - foreach(part IN LISTS parts) - if(part STREQUAL "") - continue() - endif() - if(result STREQUAL "") - set(result "${part}") - else() - string(SUBSTRING "${part}" 0 1 head) - string(SUBSTRING "${part}" 1 -1 tail) - string(TOUPPER "${head}" head) - string(APPEND result "${head}${tail}") - endif() - endforeach() - set(${out} - "${result}" - PARENT_SCOPE) -endfunction() - -# A name usable as a C++ or torch identifier: "__" collapsed to "_", since C++ -# reserves identifiers that contain a double underscore. -function(_trtllm_mv2_ident out name) - set(result "${name}") - while(result MATCHES "__") - string(REPLACE "__" "_" result "${result}") - endwhile() - set(${out} - "${result}" - PARENT_SCOPE) -endfunction() - -function(_trtllm_mv2_check_family_name family) - if(NOT family MATCHES "^[a-z][a-z0-9_]*[a-z0-9]$" OR family MATCHES "__") - message( - FATAL_ERROR - "modeling_v2: family '${family}' is not a snake_case package name") - endif() -endfunction() - -# ____, the identity the Python target directory -# carries (see tensorrt_llm/_torch/_experimental/modeling_v2/README.md). -function(_trtllm_mv2_check_target_name target) - string(REPLACE "__" ";" segments "${target}") - list(LENGTH segments count) - if(NOT count EQUAL 3 OR NOT target MATCHES "^[a-z][a-z0-9_]*$") - message( - FATAL_ERROR - "modeling_v2: target '${target}' is not a ____ name" - ) - endif() - list(GET segments 1 sm) - if(NOT sm MATCHES "^sm_[0-9]+[a-z0-9]*$") - message( - FATAL_ERROR - "modeling_v2: target '${target}' has no sm_ segment (got '${sm}')" - ) - endif() -endfunction() - -# --- Selection, requirements, architectures ----------------------------------- - -# Whether TRTLLM_MODELING_V2_TARGETS selects /. -function(_trtllm_mv2_selected out family target) - set(filter "${TRTLLM_MODELING_V2_TARGETS}") - if(filter STREQUAL "" OR filter STREQUAL "all") - set(${out} - TRUE - PARENT_SCOPE) - return() - endif() - if(filter STREQUAL "none") - set(${out} - FALSE - PARENT_SCOPE) - return() - endif() - set(selected FALSE) - foreach(pattern IN LISTS filter) - # Glob to regex: escape the dots, then * -> [^/]*, ? -> [^/]. - string(REPLACE "." "\\." regex "${pattern}") - string(REPLACE "*" "[^/]*" regex "${regex}") - string(REPLACE "?" "[^/]" regex "${regex}") - if("${family}/${target}" MATCHES "^${regex}$") - set(selected TRUE) - endif() - endforeach() - set(${out} - ${selected} - PARENT_SCOPE) -endfunction() - -# Empty when every requirement holds, otherwise the reason. -function(_trtllm_mv2_unmet_requirements out) - set(reasons "") - foreach(requirement IN LISTS ARGN) - if(NOT requirement IN_LIST _TRTLLM_MV2_KNOWN_REQUIRES) - message( - FATAL_ERROR - "modeling_v2: unknown REQUIRES '${requirement}' (known: ${_TRTLLM_MV2_KNOWN_REQUIRES})" - ) - endif() - if(requirement STREQUAL "MULTI_DEVICE" AND NOT ENABLE_MULTI_DEVICE) - list(APPEND reasons "built with ENABLE_MULTI_DEVICE=OFF") - elseif(requirement STREQUAL "NVSHMEM" AND NOT ENABLE_NVSHMEM) - list(APPEND reasons "built with ENABLE_NVSHMEM=OFF") - endif() - endforeach() - list(JOIN reasons ", " reasons) - set(${out} - "${reasons}" - PARENT_SCOPE) -endfunction() - -#[[ -Intersect declared architectures with the requested ones. - - NN -> NN-real if NN is requested - NNa -> NNa-real if NN is requested - NNf -> NNf-real if a requested architecture is in family NN: the same - major version and a minor version >= NN's - -`requested` defaults to CMAKE_CUDA_ARCHITECTURES_ORIG, the plain numbers the -build was asked for. This deliberately does not reuse set_cuda_architectures(), -which drops NNa silently and falls back to placeholder kernels. -#]] -function(_trtllm_mv2_resolve_archs out) - cmake_parse_arguments(PARSE_ARGV 1 arg "" "" "DECLARED;REQUESTED") - if(NOT DEFINED arg_REQUESTED) - set(arg_REQUESTED ${CMAKE_CUDA_ARCHITECTURES_ORIG}) - endif() - set(resolved "") - foreach(arch IN LISTS arg_DECLARED) - if(NOT arch MATCHES "^([0-9]+)([af]?)$") - message( - FATAL_ERROR "modeling_v2: ARCHS entry '${arch}' is not NN, NNa or NNf") - endif() - set(number "${CMAKE_MATCH_1}") - set(suffix "${CMAKE_MATCH_2}") - if(suffix STREQUAL "f") - math(EXPR family_major "${number} / 10") - set(in_family FALSE) - foreach(requested IN LISTS arg_REQUESTED) - math(EXPR requested_major "${requested} / 10") - if(requested_major EQUAL family_major AND requested GREATER_EQUAL - number) - set(in_family TRUE) - endif() - endforeach() - if(in_family) - list(APPEND resolved "${number}f-real") - endif() - elseif(number IN_LIST arg_REQUESTED) - list(APPEND resolved "${number}${suffix}-real") - endif() - endforeach() - list(REMOVE_DUPLICATES resolved) - set(${out} - "${resolved}" - PARENT_SCOPE) -endfunction() - -# --- Contract fingerprint -# ------------------------------------------------------ - -#[[ -Define the target `trtllm_mv2_contract_fingerprint`, which generates -${TRTLLM_MV2_GENERATED_DIR}/mv2ContractFingerprint.h. - -The fingerprint hashes what a family library is compiled against on the stock -side: the stock headers listed in cpp/tensorrt_llm/modeling_v2/contractSurface.txt, -the global compile definitions, and the torch and CUDA versions. th_common and -every family library report it, and the loader refuses a family library whose -fingerprint differs from the loaded th_common's. - -Call from cpp/tensorrt_llm/CMakeLists.txt before thop; it sets -TRTLLM_MV2_GENERATED_DIR in the caller's scope. -#]] -function(trtllm_mv2_define_contract_fingerprint) - set(generated_dir "${CMAKE_BINARY_DIR}/tensorrt_llm/modeling_v2/generated") - set(surface_file - "${PROJECT_SOURCE_DIR}/tensorrt_llm/modeling_v2/contractSurface.txt") - set_property( - DIRECTORY - APPEND - PROPERTY CMAKE_CONFIGURE_DEPENDS "${surface_file}") - - file(STRINGS "${surface_file}" patterns REGEX "^[^#]") - set(headers "") - foreach(pattern IN LISTS patterns) - string(STRIP "${pattern}" pattern) - if(pattern STREQUAL "") - continue() - endif() - set(matched_any FALSE) - foreach(root "${PROJECT_SOURCE_DIR}/include" "${PROJECT_SOURCE_DIR}") - file(GLOB matched CONFIGURE_DEPENDS "${root}/${pattern}") - if(matched) - set(matched_any TRUE) - list(APPEND headers ${matched}) - endif() - endforeach() - if(NOT matched_any) - message( - FATAL_ERROR - "modeling_v2: contract surface pattern '${pattern}' matches no header" - ) - endif() - endforeach() - list(REMOVE_DUPLICATES headers) - list(SORT headers) - - get_directory_property(definitions COMPILE_DEFINITIONS) - set(inputs "torch:${TORCH_VERSION}\ncuda:${CUDAToolkit_VERSION_MAJOR}\n") - foreach(definition IN LISTS definitions) - string(APPEND inputs "definition:${definition}\n") - endforeach() - foreach(header IN LISTS headers) - string(APPEND inputs "file:${header}\n") - endforeach() - file(WRITE "${generated_dir}/contractInputs.txt.tmp" "${inputs}") - configure_file("${generated_dir}/contractInputs.txt.tmp" - "${generated_dir}/contractInputs.txt" COPYONLY) - - set(header "${generated_dir}/mv2ContractFingerprint.h") - set(stamp "${generated_dir}/mv2ContractFingerprint.stamp") - add_custom_command( - OUTPUT "${stamp}" - BYPRODUCTS "${header}" - COMMAND - ${CMAKE_COMMAND} "-DINPUTS=${generated_dir}/contractInputs.txt" - "-DROOT=${PROJECT_SOURCE_DIR}" "-DOUTPUT=${header}" -P - "${_TRTLLM_MV2_MODULE_DIR}/modelingV2Fingerprint.cmake" - COMMAND ${CMAKE_COMMAND} -E touch "${stamp}" - DEPENDS ${headers} "${generated_dir}/contractInputs.txt" - "${_TRTLLM_MV2_MODULE_DIR}/modelingV2Fingerprint.cmake" - COMMENT "Computing the modeling_v2 contract fingerprint" - VERBATIM) - add_custom_target(trtllm_mv2_contract_fingerprint DEPENDS "${stamp}") - - set(TRTLLM_MV2_GENERATED_DIR - "${generated_dir}" - PARENT_SCOPE) -endfunction() - -# --- Families, shared code and targets ---------------------------------------- - -function(trtllm_mv2_family family) - _trtllm_mv2_check_family_name("${family}") - _trtllm_mv2_camel(camel "${family}") - get_filename_component(dir_name "${CMAKE_CURRENT_SOURCE_DIR}" NAME) - if(NOT dir_name STREQUAL camel) - message( - FATAL_ERROR - "modeling_v2: family '${family}' must live in directory '${camel}', not '${dir_name}'" - ) - endif() - get_property(families GLOBAL PROPERTY TRTLLM_MV2_FAMILIES) - if(family IN_LIST families) - message(FATAL_ERROR "modeling_v2: family '${family}' is declared twice") - endif() - set_property(GLOBAL APPEND PROPERTY TRTLLM_MV2_FAMILIES "${family}") - set_property(GLOBAL PROPERTY TRTLLM_MV2_${family}_DIR - "${CMAKE_CURRENT_SOURCE_DIR}") -endfunction() - -function(_trtllm_mv2_require_family family) - get_property(families GLOBAL PROPERTY TRTLLM_MV2_FAMILIES) - if(NOT family IN_LIST families) - message( - FATAL_ERROR - "modeling_v2: call trtllm_mv2_family(${family}) before declaring its code" - ) - endif() - get_property(family_dir GLOBAL PROPERTY TRTLLM_MV2_${family}_DIR) - if(NOT family_dir STREQUAL CMAKE_CURRENT_SOURCE_DIR) - message( - FATAL_ERROR - "modeling_v2: declare family '${family}' code in ${family_dir}/CMakeLists.txt" - ) - endif() -endfunction() - -# Expand SOURCES (paths or globs) relative to `base`. -function(_trtllm_mv2_expand_sources out base) - set(sources "") - foreach(item IN LISTS ARGN) - if(IS_ABSOLUTE "${item}") - message( - FATAL_ERROR - "modeling_v2: SOURCES must be relative to ${base}, got '${item}'") - endif() - if(item MATCHES "[*?]") - file(GLOB matched CONFIGURE_DEPENDS "${base}/${item}") - if(NOT matched) - message(FATAL_ERROR "modeling_v2: '${base}/${item}' matches no file") - endif() - list(SORT matched) - list(APPEND sources ${matched}) - else() - if(NOT EXISTS "${base}/${item}") - message(FATAL_ERROR "modeling_v2: '${base}/${item}' does not exist") - endif() - list(APPEND sources "${base}/${item}") - endif() - endforeach() - set(${out} - "${sources}" - PARENT_SCOPE) -endfunction() - -function(_trtllm_mv2_check_deps) - foreach(dep IN LISTS ARGN) - if(NOT dep IN_LIST _TRTLLM_MV2_KNOWN_DEPS) - message( - FATAL_ERROR - "modeling_v2: unknown DEPS '${dep}' (known: ${_TRTLLM_MV2_KNOWN_DEPS})" - ) - endif() - endforeach() -endfunction() - -# Compile-time usage requirements of DEPS on one OBJECT library. -function(_trtllm_mv2_apply_compile_deps object_lib) - foreach(dep IN LISTS ARGN) - if(dep STREQUAL "internal_cutlass") - target_include_directories( - ${object_lib} - PRIVATE - $ - ) - elseif(dep STREQUAL "trtllm_gen_fmha_interface") - target_link_libraries(${object_lib} PRIVATE trtllm_gen_fmha_interface) - elseif(dep STREQUAL "nccl" AND ENABLE_MULTI_DEVICE) - target_include_directories(${object_lib} PRIVATE ${MPI_C_INCLUDE_DIRS}) - endif() - endforeach() -endfunction() - -# Link-time libraries of DEPS, for the family library. -function(_trtllm_mv2_link_libraries out) - set(libraries "") - foreach(dep IN LISTS ARGN) - if(dep STREQUAL "cublas") - list(APPEND libraries ${CUBLAS_LIB}) - elseif(dep STREQUAL "cublaslt") - list(APPEND libraries ${CUBLASLT_LIB}) - elseif(dep STREQUAL "cuda_driver") - list(APPEND libraries ${CUDA_DRV_LIB}) - elseif(dep STREQUAL "nccl" AND ENABLE_MULTI_DEVICE) - list(APPEND libraries ${NCCL_LIB} ${MPI_C_LIBRARIES}) - endif() - endforeach() - set(${out} - "${libraries}" - PARENT_SCOPE) -endfunction() - -#[[ -Create the OBJECT libraries of one unit (a target, or a family's shared code): -`_kernels` for the .cu sources, compiled for `archs` without torch, and -`_ops` for the .cpp torch bindings. Both get the unit's namespaces as -compile definitions (see common/torchLibrary.h and common/namespace.h). -#]] -function( - _trtllm_mv2_add_objects - name - family - torch_ns - scope - archs - sources - deps) - set(cu_sources "") - set(cpp_sources "") - foreach(source IN LISTS sources) - if(source MATCHES "\\.cu$") - list(APPEND cu_sources "${source}") - elseif(source MATCHES "\\.cpp$") - list(APPEND cpp_sources "${source}") - elseif(NOT source MATCHES "\\.(h|cuh|hpp|inl)$") - message( - FATAL_ERROR - "modeling_v2: '${source}' is neither a .cu kernel source nor a .cpp binding" - ) - endif() - endforeach() - - set(definitions - TRTLLM_MV2_TORCH_NS=${torch_ns} TRTLLM_MV2_FAMILY_TORCH_NS=mv2_${family} - TRTLLM_MV2_SCOPE=${scope}) - set(objects "") - if(cu_sources) - add_library(${name}_kernels OBJECT ${cu_sources}) - set_target_properties( - ${name}_kernels PROPERTIES POSITION_INDEPENDENT_CODE ON CUDA_ARCHITECTURES - "${archs}") - target_compile_definitions(${name}_kernels PRIVATE ${definitions}) - _trtllm_mv2_apply_compile_deps(${name}_kernels ${deps}) - list(APPEND objects ${name}_kernels) - endif() - if(cpp_sources) - add_library(${name}_ops OBJECT ${cpp_sources}) - set_target_properties(${name}_ops PROPERTIES POSITION_INDEPENDENT_CODE ON) - target_compile_definitions(${name}_ops PRIVATE ${definitions}) - target_link_libraries(${name}_ops PRIVATE ${TORCH_LIBRARIES}) - _trtllm_mv2_apply_compile_deps(${name}_ops ${deps}) - list(APPEND objects ${name}_ops) - endif() - set_property(GLOBAL APPEND PROPERTY TRTLLM_MV2_${family}_OBJECTS ${objects}) -endfunction() - -# Record a unit's manifest entry and, when it is built, its link libraries. -function( - _trtllm_mv2_record - family - key - status - declared - archs - reason - torch_ns - deps) - set_property(GLOBAL PROPERTY TRTLLM_MV2_${family}_${key}_STATUS "${status}") - set_property(GLOBAL PROPERTY TRTLLM_MV2_${family}_${key}_DECLARED - "${declared}") - set_property(GLOBAL PROPERTY TRTLLM_MV2_${family}_${key}_ARCHS "${archs}") - set_property(GLOBAL PROPERTY TRTLLM_MV2_${family}_${key}_REASON "${reason}") - set_property(GLOBAL PROPERTY TRTLLM_MV2_${family}_${key}_NAMESPACE - "${torch_ns}") - if(status STREQUAL "built") - set_property(GLOBAL APPEND PROPERTY TRTLLM_MV2_${family}_DEPS ${deps}) - endif() -endfunction() - -function(trtllm_mv2_shared family) - cmake_parse_arguments(PARSE_ARGV 1 arg "" "" "ARCHS;SOURCES;DEPS;REQUIRES") - _trtllm_mv2_require_family("${family}") - get_property(already GLOBAL PROPERTY TRTLLM_MV2_${family}_HAS_SHARED) - if(already) - message( - FATAL_ERROR - "modeling_v2: family '${family}' declares trtllm_mv2_shared twice") - endif() - set_property(GLOBAL PROPERTY TRTLLM_MV2_${family}_HAS_SHARED TRUE) - if(NOT arg_ARCHS OR NOT arg_SOURCES) - message( - FATAL_ERROR - "modeling_v2: trtllm_mv2_shared(${family}) needs ARCHS and SOURCES") - endif() - _trtllm_mv2_check_deps(${arg_DEPS}) - if(NOT IS_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/shared") - message( - FATAL_ERROR - "modeling_v2: family '${family}' shared code lives in ${CMAKE_CURRENT_SOURCE_DIR}/shared" - ) - endif() - - # Shared code is built when a target of the family is; trtllm_mv2_finalize - # drops it otherwise. Selection and requirements still apply to it here. - _trtllm_mv2_unmet_requirements(unmet ${arg_REQUIRES}) - _trtllm_mv2_resolve_archs(archs DECLARED ${arg_ARCHS}) - if(NOT unmet STREQUAL "") - set(status "skipped") - set(reason "${unmet}") - set(archs "") - elseif(NOT archs) - set(status "skipped") - set(reason - "declares ${arg_ARCHS}; this build requests ${CMAKE_CUDA_ARCHITECTURES_ORIG}" - ) - else() - set(status "built") - set(reason "") - _trtllm_mv2_expand_sources(sources "${CMAKE_CURRENT_SOURCE_DIR}/shared" - ${arg_SOURCES}) - _trtllm_mv2_add_objects( - trtllm_mv2_${family}_shared - "${family}" - "mv2_${family}" - "${family}::shared" - "${archs}" - "${sources}" - "${arg_DEPS}") - endif() - _trtllm_mv2_record( - "${family}" - "SHARED" - "${status}" - "${arg_ARCHS}" - "${archs}" - "${reason}" - "mv2_${family}" - "${arg_DEPS}") -endfunction() - -function(trtllm_mv2_target family target) - cmake_parse_arguments(PARSE_ARGV 2 arg "" "" "ARCHS;SOURCES;DEPS;REQUIRES") - _trtllm_mv2_require_family("${family}") - _trtllm_mv2_check_target_name("${target}") - if(NOT arg_ARCHS OR NOT arg_SOURCES) - message( - FATAL_ERROR - "modeling_v2: trtllm_mv2_target(${family} ${target}) needs ARCHS and SOURCES" - ) - endif() - _trtllm_mv2_check_deps(${arg_DEPS}) - _trtllm_mv2_camel(target_dir_name "${target}") - set(target_dir "${CMAKE_CURRENT_SOURCE_DIR}/${target_dir_name}") - if(NOT IS_DIRECTORY "${target_dir}") - message( - FATAL_ERROR - "modeling_v2: target '${target}' must live in directory ${target_dir}") - endif() - - get_property(targets GLOBAL PROPERTY TRTLLM_MV2_${family}_TARGETS) - if(target IN_LIST targets) - message( - FATAL_ERROR "modeling_v2: target '${family}/${target}' is declared twice") - endif() - set_property(GLOBAL APPEND PROPERTY TRTLLM_MV2_${family}_TARGETS "${target}") - - _trtllm_mv2_ident(target_ident "${target}") - set(torch_ns "mv2_${family}_${target_ident}") - get_property(namespaces GLOBAL PROPERTY TRTLLM_MV2_NAMESPACES) - if(torch_ns IN_LIST namespaces) - message( - FATAL_ERROR - "modeling_v2: torch namespace '${torch_ns}' of ${family}/${target} is already taken" - ) - endif() - set_property(GLOBAL APPEND PROPERTY TRTLLM_MV2_NAMESPACES "${torch_ns}") - - _trtllm_mv2_selected(selected "${family}" "${target}") - _trtllm_mv2_unmet_requirements(unmet ${arg_REQUIRES}) - _trtllm_mv2_resolve_archs(archs DECLARED ${arg_ARCHS}) - if(NOT selected) - set(status "excluded") - set(reason - "not selected by TRTLLM_MODELING_V2_TARGETS=${TRTLLM_MODELING_V2_TARGETS}" - ) - set(archs "") - elseif(NOT unmet STREQUAL "") - set(status "skipped") - set(reason "${unmet}") - set(archs "") - elseif(NOT archs) - set(status "skipped") - set(reason - "declares ${arg_ARCHS}; this build requests ${CMAKE_CUDA_ARCHITECTURES_ORIG}" - ) - else() - set(status "built") - set(reason "") - _trtllm_mv2_expand_sources(sources "${target_dir}" ${arg_SOURCES}) - _trtllm_mv2_add_objects( - trtllm_mv2_${family}_${target_ident} - "${family}" - "${torch_ns}" - "${family}::${target_ident}" - "${archs}" - "${sources}" - "${arg_DEPS}") - endif() - _trtllm_mv2_record( - "${family}" - "TARGET_${target}" - "${status}" - "${arg_ARCHS}" - "${archs}" - "${reason}" - "${torch_ns}" - "${arg_DEPS}") -endfunction() - -# --- Family libraries and the manifest --------------------------------------- - -function(_trtllm_mv2_json_list out) - set(json "[]") - set(index 0) - foreach(item IN LISTS ARGN) - string(JSON json SET "${json}" ${index} "\"${item}\"") - math(EXPR index "${index} + 1") - endforeach() - set(${out} - "${json}" - PARENT_SCOPE) -endfunction() - -function(_trtllm_mv2_json_unit out family key) - get_property(status GLOBAL PROPERTY TRTLLM_MV2_${family}_${key}_STATUS) - get_property(declared GLOBAL PROPERTY TRTLLM_MV2_${family}_${key}_DECLARED) - get_property(archs GLOBAL PROPERTY TRTLLM_MV2_${family}_${key}_ARCHS) - get_property(reason GLOBAL PROPERTY TRTLLM_MV2_${family}_${key}_REASON) - get_property(torch_ns GLOBAL PROPERTY TRTLLM_MV2_${family}_${key}_NAMESPACE) - _trtllm_mv2_json_list(declared_json ${declared}) - _trtllm_mv2_json_list(archs_json ${archs}) - set(json "{}") - string(JSON json SET "${json}" status "\"${status}\"") - string(JSON json SET "${json}" namespace "\"${torch_ns}\"") - string(JSON json SET "${json}" declared_archs "${declared_json}") - string(JSON json SET "${json}" archs "${archs_json}") - string(JSON json SET "${json}" reason "\"${reason}\"") - set(${out} - "${json}" - PARENT_SCOPE) -endfunction() - -function(_trtllm_mv2_write_manifest json) - file( - GENERATE - OUTPUT - "${CMAKE_BINARY_DIR}/tensorrt_llm/modeling_v2/modeling_v2_manifest.json" - CONTENT "${json}\n") -endfunction() - -# The manifest of a build without modeling_v2 libraries. -function(trtllm_mv2_write_disabled_manifest reason) - set(json "{}") - string(JSON json SET "${json}" schema ${_TRTLLM_MV2_MANIFEST_SCHEMA}) - string(JSON json SET "${json}" enabled false) - string(JSON json SET "${json}" reason "\"${reason}\"") - string(JSON json SET "${json}" families "{}") - _trtllm_mv2_write_manifest("${json}") -endfunction() - -# The registration unit of a family library: it claims the family's torch -# namespace (with contract_fingerprint) and every built target's namespace, so a -# second library claiming any of them fails when it is loaded. -function(_trtllm_mv2_registration_source out family) - set(source - "// Generated by cpp/cmake/modules/modelingV2.cmake for family ${family}. Do not edit.\n" - ) - string(APPEND source "#include \"mv2ContractFingerprint.h\"\n\n") - string(APPEND source "#include \n#include \n\n") - string(APPEND source "namespace\n{\nstd::string contractFingerprint()\n{\n") - string(APPEND source - " return TRTLLM_MV2_CONTRACT_FINGERPRINT;\n}\n} // namespace\n\n") - string(APPEND source "TORCH_LIBRARY(mv2_${family}, m)\n{\n") - string(APPEND source " m.def(\"contract_fingerprint() -> str\");\n}\n\n") - string(APPEND source - "TORCH_LIBRARY_IMPL(mv2_${family}, CompositeExplicitAutograd, m)\n{\n") - string(APPEND source - " m.impl(\"contract_fingerprint\", &contractFingerprint);\n}\n") - foreach(torch_ns IN LISTS ARGN) - string(APPEND source - "\nTORCH_LIBRARY(${torch_ns}, m)\n{\n (void) m;\n}\n") - endforeach() - set(path "${CMAKE_CURRENT_BINARY_DIR}/${family}/registration.cpp") - file( - GENERATE - OUTPUT "${path}" - CONTENT "${source}") - set_source_files_properties("${path}" PROPERTIES GENERATED TRUE) - set(${out} - "${path}" - PARENT_SCOPE) -endfunction() - -function(_trtllm_mv2_add_family_library family) - get_property(objects GLOBAL PROPERTY TRTLLM_MV2_${family}_OBJECTS) - get_property(targets GLOBAL PROPERTY TRTLLM_MV2_${family}_TARGETS) - get_property(deps GLOBAL PROPERTY TRTLLM_MV2_${family}_DEPS) - - set(built_namespaces "") - foreach(target IN LISTS targets) - get_property(status GLOBAL - PROPERTY TRTLLM_MV2_${family}_TARGET_${target}_STATUS) - if(status STREQUAL "built") - get_property(torch_ns GLOBAL - PROPERTY TRTLLM_MV2_${family}_TARGET_${target}_NAMESPACE) - list(APPEND built_namespaces "${torch_ns}") - endif() - endforeach() - - # Shared code without a built target has nothing to serve. - get_property(shared_status GLOBAL PROPERTY TRTLLM_MV2_${family}_SHARED_STATUS) - if(NOT built_namespaces) - if(shared_status STREQUAL "built") - set_property(GLOBAL PROPERTY TRTLLM_MV2_${family}_SHARED_STATUS "skipped") - set_property(GLOBAL PROPERTY TRTLLM_MV2_${family}_SHARED_REASON - "no target of the family is built") - set_property(GLOBAL PROPERTY TRTLLM_MV2_${family}_SHARED_ARCHS "") - endif() - set_property(GLOBAL PROPERTY TRTLLM_MV2_${family}_LIBRARY "") - return() - endif() - - set(library trtllm_mv2_${family}) - _trtllm_mv2_registration_source(registration "${family}" ${built_namespaces}) - set(object_sources "") - foreach(object IN LISTS objects) - list(APPEND object_sources $) - endforeach() - add_library(${library} MODULE "${registration}" ${object_sources}) - add_dependencies(${library} trtllm_mv2_contract_fingerprint) - target_include_directories(${library} PRIVATE "${TRTLLM_MV2_GENERATED_DIR}") - list(REMOVE_DUPLICATES deps) - _trtllm_mv2_link_libraries(dep_libraries ${deps}) - target_link_libraries( - ${library} PRIVATE ${TORCH_LIBRARIES} th_utils ${SHARED_TARGET} - ${CUDA_RT_LIB} ${CUDA_DRV_LIB} ${dep_libraries}) - set(version_script - "${PROJECT_SOURCE_DIR}/tensorrt_llm/modeling_v2/common/familyLibrary.version" - ) - set_target_properties( - ${library} - PROPERTIES OUTPUT_NAME ${library} - PREFIX "lib" - LIBRARY_OUTPUT_DIRECTORY - "${CMAKE_BINARY_DIR}/tensorrt_llm/modeling_v2" - POSITION_INDEPENDENT_CODE ON - LINKER_LANGUAGE CXX - LINK_DEPENDS "${version_script}" - BUILD_RPATH "$ORIGIN;$ORIGIN/../../nvidia/nccl/lib") - target_link_options( - ${library} PRIVATE "-Wl,--version-script=${version_script}" - "-Wl,--no-undefined") - add_dependencies(modeling_v2_libs ${library}) - set_property(GLOBAL PROPERTY TRTLLM_MV2_${family}_LIBRARY "lib${library}.so") -endfunction() - -# Build every declared family's library and write the manifest. Called by -# cpp/tensorrt_llm/modeling_v2/CMakeLists.txt after it added the families. -function(trtllm_mv2_finalize) - get_property(families GLOBAL PROPERTY TRTLLM_MV2_FAMILIES) - _trtllm_mv2_json_list(requested_json ${CMAKE_CUDA_ARCHITECTURES_ORIG}) - set(json "{}") - string(JSON json SET "${json}" schema ${_TRTLLM_MV2_MANIFEST_SCHEMA}) - string(JSON json SET "${json}" enabled true) - string(JSON json SET "${json}" requested_archs "${requested_json}") - string(JSON json SET "${json}" families "{}") - - foreach(family IN LISTS families) - _trtllm_mv2_add_family_library("${family}") - get_property(library GLOBAL PROPERTY TRTLLM_MV2_${family}_LIBRARY) - get_property(targets GLOBAL PROPERTY TRTLLM_MV2_${family}_TARGETS) - get_property(has_shared GLOBAL PROPERTY TRTLLM_MV2_${family}_HAS_SHARED) - - set(family_json "{}") - if(library STREQUAL "") - string(JSON family_json SET "${family_json}" library null) - else() - string(JSON family_json SET "${family_json}" library "\"${library}\"") - endif() - string(JSON family_json SET "${family_json}" namespace "\"mv2_${family}\"") - if(has_shared) - _trtllm_mv2_json_unit(shared_json "${family}" "SHARED") - string(JSON family_json SET "${family_json}" shared "${shared_json}") - else() - string(JSON family_json SET "${family_json}" shared null) - endif() - set(targets_json "{}") - foreach(target IN LISTS targets) - _trtllm_mv2_json_unit(target_json "${family}" "TARGET_${target}") - string(JSON targets_json SET "${targets_json}" "${target}" - "${target_json}") - endforeach() - string(JSON family_json SET "${family_json}" targets "${targets_json}") - string( - JSON - json - SET - "${json}" - families - "${family}" - "${family_json}") - endforeach() - _trtllm_mv2_write_manifest("${json}") -endfunction() diff --git a/cpp/cmake/modules/modelingV2Fingerprint.cmake b/cpp/cmake/modules/modelingV2Fingerprint.cmake deleted file mode 100644 index e7860cf1c897..000000000000 --- a/cpp/cmake/modules/modelingV2Fingerprint.cmake +++ /dev/null @@ -1,66 +0,0 @@ -# -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. -# All rights reserved. SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. -# - -#[[ -Write the modeling_v2 contract fingerprint header (build step of -trtllm_mv2_define_contract_fingerprint in modelingV2.cmake). - - cmake -DINPUTS= -DROOT= -DOUTPUT=
- -P modelingV2Fingerprint.cmake - -INPUTS has one entry per line: `file:` lines are hashed by -content under their path relative to ROOT, every other line is hashed as is. -The header is rewritten only when the fingerprint changes, so an unchanged -contract recompiles nothing. -#]] - -foreach(required INPUTS ROOT OUTPUT) - if(NOT DEFINED ${required}) - message( - FATAL_ERROR "modelingV2Fingerprint.cmake: -D${required} is required") - endif() -endforeach() - -file(STRINGS "${INPUTS}" lines) -set(digest_input "") -foreach(line IN LISTS lines) - if(line MATCHES "^file:(.*)$") - set(path "${CMAKE_MATCH_1}") - file(SHA256 "${path}" content_digest) - file(RELATIVE_PATH relative "${ROOT}" "${path}") - string(APPEND digest_input "${relative} ${content_digest}\n") - else() - string(APPEND digest_input "${line}\n") - endif() -endforeach() -string(SHA256 fingerprint "${digest_input}") -string(SUBSTRING "${fingerprint}" 0 16 fingerprint) - -set(content - "// Generated by cpp/cmake/modules/modelingV2Fingerprint.cmake. Do not edit.\n" -) -string(APPEND content "#pragma once\n\n") -string(APPEND content - "#define TRTLLM_MV2_CONTRACT_FINGERPRINT \"${fingerprint}\"\n") - -if(EXISTS "${OUTPUT}") - file(READ "${OUTPUT}" previous) - if(previous STREQUAL content) - return() - endif() -endif() -file(WRITE "${OUTPUT}" "${content}") diff --git a/cpp/cmake/modules/modeling_v2.cmake b/cpp/cmake/modules/modeling_v2.cmake new file mode 100644 index 000000000000..937f9f402b24 --- /dev/null +++ b/cpp/cmake/modules/modeling_v2.cmake @@ -0,0 +1,638 @@ +# +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# + +#[[ +Native (C++/CUDA) code of modeling_v2 targets. + +Each model family under cpp/tensorrt_llm/modeling_v2// builds into +one torch-op library, libth_modeling_v2_.so, installed next to +libth_common.so. Nothing in it is linked into libtensorrt_llm.so or +libth_common.so; the Python loader +(tensorrt_llm/_torch/_experimental/modeling_v2/_native.py) opens it only when +routing picks one of the family's targets. cpp/tensorrt_llm/modeling_v2/README.md +is the guide for authors. + +A family's CMakeLists.txt uses three functions. Names are the Python package +names (snake_case); directories are their camelCase form: + + add_modeling_v2_family() + add_modeling_v2_shared( ARCHS ... SOURCES ... [DEPS ...] [REQUIRES ...]) + add_modeling_v2_target( ARCHS ... SOURCES ... [DEPS ...] [REQUIRES ...]) + +ARCHS NN, NNa or NNf. Each is intersected with the build's requested + architectures, the way flash_mla and deep_ep choose theirs; a target + with an empty intersection is not built and its family library + does not carry it. +SOURCES Paths or globs relative to the target directory (shared/ for + add_modeling_v2_shared). .cu files are compiled for ARCHS without + torch; .cpp files are the torch bindings. +DEPS From a closed list: cublas, cublaslt, cuda_driver, internal_cutlass, + nccl, trtllm_gen_fmha_interface. +REQUIRES Build features: MULTI_DEVICE, NVSHMEM. Unmet requirements skip the + target. + +Like deep_ep and flash_mla, each family writes cuda_architectures.txt into its +build directory: the architectures its library was built for, empty when it +has none. scripts/build_wheel.py installs the library only when that list is +not empty. +#]] + +include_guard(GLOBAL) + +set(_MODELING_V2_KNOWN_DEPS cublas cublaslt cuda_driver internal_cutlass nccl + trtllm_gen_fmha_interface) +set(_MODELING_V2_KNOWN_REQUIRES MULTI_DEVICE NVSHMEM) + +# --- Naming ------------------------------------------------------------------ + +# camelCase form of a snake_case name: minimax_m3_nvfp4__sm_107__tp2 -> +# minimaxM3Nvfp4Sm107Tp2. The same mapping as camel() in the Python checks. +function(_modeling_v2_camel out name) + string(REPLACE "_" ";" parts "${name}") + set(result "") + foreach(part IN LISTS parts) + if(part STREQUAL "") + continue() + endif() + if(result STREQUAL "") + set(result "${part}") + else() + string(SUBSTRING "${part}" 0 1 head) + string(SUBSTRING "${part}" 1 -1 tail) + string(TOUPPER "${head}" head) + string(APPEND result "${head}${tail}") + endif() + endforeach() + set(${out} + "${result}" + PARENT_SCOPE) +endfunction() + +# A name usable as a C++ or torch identifier: "__" collapsed to "_", since C++ +# reserves identifiers that contain a double underscore. +function(_modeling_v2_ident out name) + set(result "${name}") + while(result MATCHES "__") + string(REPLACE "__" "_" result "${result}") + endwhile() + set(${out} + "${result}" + PARENT_SCOPE) +endfunction() + +function(_modeling_v2_check_family_name family) + if(NOT family MATCHES "^[a-z][a-z0-9_]*[a-z0-9]$" OR family MATCHES "__") + message( + FATAL_ERROR + "modeling_v2: family '${family}' is not a snake_case package name") + endif() +endfunction() + +# ____, the identity the Python target directory +# carries (see tensorrt_llm/_torch/_experimental/modeling_v2/README.md). +function(_modeling_v2_check_target_name target) + string(REPLACE "__" ";" segments "${target}") + list(LENGTH segments count) + if(NOT count EQUAL 3 OR NOT target MATCHES "^[a-z][a-z0-9_]*$") + message( + FATAL_ERROR + "modeling_v2: target '${target}' is not a ____ name" + ) + endif() + list(GET segments 1 sm) + if(NOT sm MATCHES "^sm_[0-9]+[a-z0-9]*$") + message( + FATAL_ERROR + "modeling_v2: target '${target}' has no sm_ segment (got '${sm}')" + ) + endif() +endfunction() + +# --- Selection, requirements, architectures ----------------------------------- + +# Whether TRTLLM_MODELING_V2_TARGETS selects /. +function(_modeling_v2_selected out family target) + set(filter "${TRTLLM_MODELING_V2_TARGETS}") + if(filter STREQUAL "" OR filter STREQUAL "all") + set(${out} + TRUE + PARENT_SCOPE) + return() + endif() + if(filter STREQUAL "none") + set(${out} + FALSE + PARENT_SCOPE) + return() + endif() + set(selected FALSE) + foreach(pattern IN LISTS filter) + # Glob to regex: escape the dots, then * -> [^/]*, ? -> [^/]. + string(REPLACE "." "\\." regex "${pattern}") + string(REPLACE "*" "[^/]*" regex "${regex}") + string(REPLACE "?" "[^/]" regex "${regex}") + if("${family}/${target}" MATCHES "^${regex}$") + set(selected TRUE) + endif() + endforeach() + set(${out} + ${selected} + PARENT_SCOPE) +endfunction() + +# Empty when every requirement holds, otherwise the reason. +function(_modeling_v2_unmet_requirements out) + set(reasons "") + foreach(requirement IN LISTS ARGN) + if(NOT requirement IN_LIST _MODELING_V2_KNOWN_REQUIRES) + message( + FATAL_ERROR + "modeling_v2: unknown REQUIRES '${requirement}' (known: ${_MODELING_V2_KNOWN_REQUIRES})" + ) + endif() + if(requirement STREQUAL "MULTI_DEVICE" AND NOT ENABLE_MULTI_DEVICE) + list(APPEND reasons "built with ENABLE_MULTI_DEVICE=OFF") + elseif(requirement STREQUAL "NVSHMEM" AND NOT ENABLE_NVSHMEM) + list(APPEND reasons "built with ENABLE_NVSHMEM=OFF") + endif() + endforeach() + list(JOIN reasons ", " reasons) + set(${out} + "${reasons}" + PARENT_SCOPE) +endfunction() + +#[[ +Intersect declared architectures with the requested ones. + + NN -> NN-real if NN is requested + NNa -> NNa-real if NN is requested + NNf -> NNf-real if a requested architecture is in family NN: the same + major version and a minor version >= NN's + +`requested` defaults to CMAKE_CUDA_ARCHITECTURES_ORIG, the plain numbers the +build was asked for -- the list flash_mla intersects with too. This does not +reuse set_cuda_architectures(), which drops NNa silently and falls back to +placeholder kernels. +#]] +function(_modeling_v2_resolve_archs out) + cmake_parse_arguments(PARSE_ARGV 1 arg "" "" "DECLARED;REQUESTED") + if(NOT DEFINED arg_REQUESTED) + set(arg_REQUESTED ${CMAKE_CUDA_ARCHITECTURES_ORIG}) + endif() + set(resolved "") + foreach(arch IN LISTS arg_DECLARED) + if(NOT arch MATCHES "^([0-9]+)([af]?)$") + message( + FATAL_ERROR "modeling_v2: ARCHS entry '${arch}' is not NN, NNa or NNf") + endif() + set(number "${CMAKE_MATCH_1}") + set(suffix "${CMAKE_MATCH_2}") + if(suffix STREQUAL "f") + math(EXPR family_major "${number} / 10") + set(in_family FALSE) + foreach(requested IN LISTS arg_REQUESTED) + math(EXPR requested_major "${requested} / 10") + if(requested_major EQUAL family_major AND requested GREATER_EQUAL + number) + set(in_family TRUE) + endif() + endforeach() + if(in_family) + list(APPEND resolved "${number}f-real") + endif() + elseif(number IN_LIST arg_REQUESTED) + list(APPEND resolved "${number}${suffix}-real") + endif() + endforeach() + list(REMOVE_DUPLICATES resolved) + set(${out} + "${resolved}" + PARENT_SCOPE) +endfunction() + +# --- Families, shared code and targets ---------------------------------------- + +function(add_modeling_v2_family family) + _modeling_v2_check_family_name("${family}") + _modeling_v2_camel(camel "${family}") + get_filename_component(dir_name "${CMAKE_CURRENT_SOURCE_DIR}" NAME) + if(NOT dir_name STREQUAL camel) + message( + FATAL_ERROR + "modeling_v2: family '${family}' must live in directory '${camel}', not '${dir_name}'" + ) + endif() + get_property(families GLOBAL PROPERTY MODELING_V2_FAMILIES) + if(family IN_LIST families) + message(FATAL_ERROR "modeling_v2: family '${family}' is declared twice") + endif() + set_property(GLOBAL APPEND PROPERTY MODELING_V2_FAMILIES "${family}") + set_property(GLOBAL PROPERTY MODELING_V2_${family}_SOURCE_DIR + "${CMAKE_CURRENT_SOURCE_DIR}") + set_property(GLOBAL PROPERTY MODELING_V2_${family}_BINARY_DIR + "${CMAKE_CURRENT_BINARY_DIR}") +endfunction() + +function(_modeling_v2_require_family family) + get_property(families GLOBAL PROPERTY MODELING_V2_FAMILIES) + if(NOT family IN_LIST families) + message( + FATAL_ERROR + "modeling_v2: call add_modeling_v2_family(${family}) before declaring its code" + ) + endif() + get_property(family_dir GLOBAL PROPERTY MODELING_V2_${family}_SOURCE_DIR) + if(NOT family_dir STREQUAL CMAKE_CURRENT_SOURCE_DIR) + message( + FATAL_ERROR + "modeling_v2: declare family '${family}' code in ${family_dir}/CMakeLists.txt" + ) + endif() +endfunction() + +# Expand SOURCES (paths or globs) relative to `base`. +function(_modeling_v2_expand_sources out base) + set(sources "") + foreach(item IN LISTS ARGN) + if(IS_ABSOLUTE "${item}") + message( + FATAL_ERROR + "modeling_v2: SOURCES must be relative to ${base}, got '${item}'") + endif() + if(item MATCHES "[*?]") + file(GLOB matched CONFIGURE_DEPENDS "${base}/${item}") + if(NOT matched) + message(FATAL_ERROR "modeling_v2: '${base}/${item}' matches no file") + endif() + list(SORT matched) + list(APPEND sources ${matched}) + else() + if(NOT EXISTS "${base}/${item}") + message(FATAL_ERROR "modeling_v2: '${base}/${item}' does not exist") + endif() + list(APPEND sources "${base}/${item}") + endif() + endforeach() + set(${out} + "${sources}" + PARENT_SCOPE) +endfunction() + +function(_modeling_v2_check_deps) + foreach(dep IN LISTS ARGN) + if(NOT dep IN_LIST _MODELING_V2_KNOWN_DEPS) + message( + FATAL_ERROR + "modeling_v2: unknown DEPS '${dep}' (known: ${_MODELING_V2_KNOWN_DEPS})" + ) + endif() + endforeach() +endfunction() + +# Compile-time usage requirements of DEPS on one OBJECT library. +function(_modeling_v2_apply_compile_deps object_lib) + foreach(dep IN LISTS ARGN) + if(dep STREQUAL "internal_cutlass") + target_include_directories( + ${object_lib} + PRIVATE + $ + ) + elseif(dep STREQUAL "trtllm_gen_fmha_interface") + target_link_libraries(${object_lib} PRIVATE trtllm_gen_fmha_interface) + elseif(dep STREQUAL "nccl" AND ENABLE_MULTI_DEVICE) + target_include_directories(${object_lib} PRIVATE ${MPI_C_INCLUDE_DIRS}) + endif() + endforeach() +endfunction() + +# Link-time libraries of DEPS, for the family library. +function(_modeling_v2_link_libraries out) + set(libraries "") + foreach(dep IN LISTS ARGN) + if(dep STREQUAL "cublas") + list(APPEND libraries ${CUBLAS_LIB}) + elseif(dep STREQUAL "cublaslt") + list(APPEND libraries ${CUBLASLT_LIB}) + elseif(dep STREQUAL "cuda_driver") + list(APPEND libraries ${CUDA_DRV_LIB}) + elseif(dep STREQUAL "nccl" AND ENABLE_MULTI_DEVICE) + list(APPEND libraries ${NCCL_LIB} ${MPI_C_LIBRARIES}) + endif() + endforeach() + set(${out} + "${libraries}" + PARENT_SCOPE) +endfunction() + +#[[ +Create the OBJECT libraries of one unit (a target, or a family's shared code): +`_kernels` for the .cu sources, compiled for `archs` without torch, and +`_ops` for the .cpp torch bindings. Both get the unit's namespaces as +compile definitions (see common/torchLibrary.h and common/namespace.h). +#]] +function( + _modeling_v2_add_objects + name + family + torch_ns + scope + archs + sources + deps) + set(cu_sources "") + set(cpp_sources "") + foreach(source IN LISTS sources) + if(source MATCHES "\\.cu$") + list(APPEND cu_sources "${source}") + elseif(source MATCHES "\\.cpp$") + list(APPEND cpp_sources "${source}") + elseif(NOT source MATCHES "\\.(h|cuh|hpp|inl)$") + message( + FATAL_ERROR + "modeling_v2: '${source}' is neither a .cu kernel source nor a .cpp binding" + ) + endif() + endforeach() + + set(definitions + TRTLLM_MV2_TORCH_NS=${torch_ns} TRTLLM_MV2_FAMILY_TORCH_NS=mv2_${family} + TRTLLM_MV2_SCOPE=${scope}) + set(objects "") + if(cu_sources) + add_library(${name}_kernels OBJECT ${cu_sources}) + set_target_properties( + ${name}_kernels PROPERTIES POSITION_INDEPENDENT_CODE ON CUDA_ARCHITECTURES + "${archs}") + target_compile_definitions(${name}_kernels PRIVATE ${definitions}) + _modeling_v2_apply_compile_deps(${name}_kernels ${deps}) + list(APPEND objects ${name}_kernels) + endif() + if(cpp_sources) + add_library(${name}_ops OBJECT ${cpp_sources}) + set_target_properties(${name}_ops PROPERTIES POSITION_INDEPENDENT_CODE ON) + target_compile_definitions(${name}_ops PRIVATE ${definitions}) + target_link_libraries(${name}_ops PRIVATE ${TORCH_LIBRARIES}) + _modeling_v2_apply_compile_deps(${name}_ops ${deps}) + list(APPEND objects ${name}_ops) + endif() + set_property(GLOBAL APPEND PROPERTY MODELING_V2_${family}_OBJECTS ${objects}) + set_property(GLOBAL APPEND PROPERTY MODELING_V2_${family}_DEPS ${deps}) + set_property(GLOBAL APPEND PROPERTY MODELING_V2_${family}_ARCHS ${archs}) +endfunction() + +# Decide whether a unit builds: "" when it does, otherwise why not. +function(_modeling_v2_skip_reason out family target declared requires) + _modeling_v2_selected(selected "${family}" "${target}") + _modeling_v2_unmet_requirements(unmet ${requires}) + _modeling_v2_resolve_archs(archs DECLARED ${declared}) + if(NOT selected) + set(reason + "not selected by TRTLLM_MODELING_V2_TARGETS=${TRTLLM_MODELING_V2_TARGETS}" + ) + elseif(NOT unmet STREQUAL "") + set(reason "${unmet}") + elseif(NOT archs) + set(reason + "declares ${declared}; this build requests ${CMAKE_CUDA_ARCHITECTURES_ORIG}" + ) + else() + set(reason "") + endif() + set(${out} + "${reason}" + PARENT_SCOPE) +endfunction() + +function(add_modeling_v2_shared family) + cmake_parse_arguments(PARSE_ARGV 1 arg "" "" "ARCHS;SOURCES;DEPS;REQUIRES") + _modeling_v2_require_family("${family}") + get_property(already GLOBAL PROPERTY MODELING_V2_${family}_HAS_SHARED) + if(already) + message( + FATAL_ERROR + "modeling_v2: family '${family}' declares add_modeling_v2_shared twice") + endif() + set_property(GLOBAL PROPERTY MODELING_V2_${family}_HAS_SHARED TRUE) + if(NOT arg_ARCHS OR NOT arg_SOURCES) + message( + FATAL_ERROR + "modeling_v2: add_modeling_v2_shared(${family}) needs ARCHS and SOURCES" + ) + endif() + _modeling_v2_check_deps(${arg_DEPS}) + if(NOT IS_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/shared") + message( + FATAL_ERROR + "modeling_v2: family '${family}' shared code lives in ${CMAKE_CURRENT_SOURCE_DIR}/shared" + ) + endif() + # Recorded now, built by _modeling_v2_add_family_library only when a target of + # the family builds: shared code with no target has nothing to serve. + foreach(field ARCHS SOURCES DEPS REQUIRES) + set_property(GLOBAL PROPERTY MODELING_V2_${family}_SHARED_${field} + "${arg_${field}}") + endforeach() +endfunction() + +function(add_modeling_v2_target family target) + cmake_parse_arguments(PARSE_ARGV 2 arg "" "" "ARCHS;SOURCES;DEPS;REQUIRES") + _modeling_v2_require_family("${family}") + _modeling_v2_check_target_name("${target}") + if(NOT arg_ARCHS OR NOT arg_SOURCES) + message( + FATAL_ERROR + "modeling_v2: add_modeling_v2_target(${family} ${target}) needs ARCHS and SOURCES" + ) + endif() + _modeling_v2_check_deps(${arg_DEPS}) + _modeling_v2_camel(target_dir_name "${target}") + set(target_dir "${CMAKE_CURRENT_SOURCE_DIR}/${target_dir_name}") + if(NOT IS_DIRECTORY "${target_dir}") + message( + FATAL_ERROR + "modeling_v2: target '${target}' must live in directory ${target_dir}") + endif() + + get_property(targets GLOBAL PROPERTY MODELING_V2_${family}_TARGETS) + if(target IN_LIST targets) + message( + FATAL_ERROR "modeling_v2: target '${family}/${target}' is declared twice") + endif() + set_property(GLOBAL APPEND PROPERTY MODELING_V2_${family}_TARGETS "${target}") + + _modeling_v2_ident(target_ident "${target}") + set(torch_ns "mv2_${family}_${target_ident}") + get_property(namespaces GLOBAL PROPERTY MODELING_V2_NAMESPACES) + if(torch_ns IN_LIST namespaces) + message( + FATAL_ERROR + "modeling_v2: torch namespace '${torch_ns}' of ${family}/${target} is already taken" + ) + endif() + set_property(GLOBAL APPEND PROPERTY MODELING_V2_NAMESPACES "${torch_ns}") + + _modeling_v2_skip_reason(reason "${family}" "${target}" "${arg_ARCHS}" + "${arg_REQUIRES}") + if(NOT reason STREQUAL "") + message(STATUS "modeling_v2: ${family}/${target} not built: ${reason}") + return() + endif() + _modeling_v2_resolve_archs(archs DECLARED ${arg_ARCHS}) + message(STATUS "modeling_v2: ${family}/${target} built for ${archs}") + _modeling_v2_expand_sources(sources "${target_dir}" ${arg_SOURCES}) + _modeling_v2_add_objects( + th_modeling_v2_${family}_${target_ident} + "${family}" + "${torch_ns}" + "${family}::${target_ident}" + "${archs}" + "${sources}" + "${arg_DEPS}") + set_property(GLOBAL APPEND PROPERTY MODELING_V2_${family}_BUILT "${target}") +endfunction() + +# --- Family libraries --------------------------------------------------------- + +# The registration unit of a family library: it claims the family's torch +# namespace and every built target's, so a second library claiming any of them +# fails when it is loaded, and it registers built_targets() -- which targets +# this library carries -- the way th_common answers feature queries such as +# mhc_fused_hc_mma_enabled(). +function(_modeling_v2_registration_source out family binary_dir) + get_property(built GLOBAL PROPERTY MODELING_V2_${family}_BUILT) + list(JOIN built "," built_csv) + set(source + "// Generated by cpp/cmake/modules/modeling_v2.cmake for family ${family}. Do not edit.\n" + ) + string(APPEND source "#include \n#include \n\n") + string(APPEND source "namespace\n{\n") + string(APPEND source + "//! The targets of this family compiled into this library.\n") + string(APPEND source "std::string builtTargets()\n{\n") + string(APPEND source " return \"${built_csv}\";\n}\n} // namespace\n\n") + string(APPEND source "TORCH_LIBRARY(mv2_${family}, m)\n{\n") + string(APPEND source " m.def(\"built_targets() -> str\");\n}\n\n") + string(APPEND source + "TORCH_LIBRARY_IMPL(mv2_${family}, CompositeExplicitAutograd, m)\n{\n") + string(APPEND source " m.impl(\"built_targets\", &builtTargets);\n}\n") + foreach(target IN LISTS built) + _modeling_v2_ident(target_ident "${target}") + string( + APPEND source + "\nTORCH_LIBRARY(mv2_${family}_${target_ident}, m)\n{\n (void) m;\n}\n" + ) + endforeach() + set(path "${binary_dir}/registration.cpp") + file( + GENERATE + OUTPUT "${path}" + CONTENT "${source}") + set_source_files_properties("${path}" PROPERTIES GENERATED TRUE) + set(${out} + "${path}" + PARENT_SCOPE) +endfunction() + +function(_modeling_v2_add_family_library family) + get_property(binary_dir GLOBAL PROPERTY MODELING_V2_${family}_BINARY_DIR) + get_property(built GLOBAL PROPERTY MODELING_V2_${family}_BUILT) + if(NOT built) + # Like deep_ep and flash_mla without a matching architecture: an empty + # cuda_architectures.txt, and nothing for build_wheel.py to install. + file(WRITE "${binary_dir}/cuda_architectures.txt" "") + return() + endif() + + get_property(has_shared GLOBAL PROPERTY MODELING_V2_${family}_HAS_SHARED) + if(has_shared) + foreach(field ARCHS SOURCES DEPS REQUIRES) + get_property(shared_${field} GLOBAL + PROPERTY MODELING_V2_${family}_SHARED_${field}) + endforeach() + # The target filter does not apply: a built target needs its shared code. + _modeling_v2_unmet_requirements(unmet ${shared_REQUIRES}) + _modeling_v2_resolve_archs(shared_archs DECLARED ${shared_ARCHS}) + if(NOT unmet STREQUAL "") + message(STATUS "modeling_v2: ${family}/shared not built: ${unmet}") + elseif(NOT shared_archs) + message( + STATUS + "modeling_v2: ${family}/shared not built: declares ${shared_ARCHS}; " + "this build requests ${CMAKE_CUDA_ARCHITECTURES_ORIG}") + else() + get_property(source_dir GLOBAL PROPERTY MODELING_V2_${family}_SOURCE_DIR) + _modeling_v2_expand_sources(sources "${source_dir}/shared" + ${shared_SOURCES}) + message(STATUS "modeling_v2: ${family}/shared built for ${shared_archs}") + _modeling_v2_add_objects( + th_modeling_v2_${family}_shared + "${family}" + "mv2_${family}" + "${family}::shared" + "${shared_archs}" + "${sources}" + "${shared_DEPS}") + endif() + endif() + + get_property(objects GLOBAL PROPERTY MODELING_V2_${family}_OBJECTS) + get_property(deps GLOBAL PROPERTY MODELING_V2_${family}_DEPS) + get_property(archs GLOBAL PROPERTY MODELING_V2_${family}_ARCHS) + list(REMOVE_DUPLICATES deps) + list(REMOVE_DUPLICATES archs) + + set(library th_modeling_v2_${family}) + _modeling_v2_registration_source(registration "${family}" "${binary_dir}") + set(object_sources "") + foreach(object IN LISTS objects) + list(APPEND object_sources $) + endforeach() + add_library(${library} MODULE "${registration}" ${object_sources}) + _modeling_v2_link_libraries(dep_libraries ${deps}) + # Linked the way th_common is: torch, th_utils and libtensorrt_llm.so, whose + # singletons (logger, mcast registry, driver wrapper) stay single. + target_link_libraries( + ${library} PRIVATE ${TORCH_LIBRARIES} th_utils ${SHARED_TARGET} + ${CUDA_RT_LIB} ${CUDA_DRV_LIB} ${dep_libraries}) + set(version_script + "${PROJECT_SOURCE_DIR}/tensorrt_llm/modeling_v2/common/familyLibrary.version" + ) + set_target_properties( + ${library} + PROPERTIES PREFIX "lib" + LIBRARY_OUTPUT_DIRECTORY "${binary_dir}" + POSITION_INDEPENDENT_CODE ON + LINKER_LANGUAGE CXX + LINK_DEPENDS "${version_script}" + BUILD_RPATH "$ORIGIN;$ORIGIN/../../nvidia/nccl/lib") + target_link_options( + ${library} PRIVATE "-Wl,--version-script=${version_script}" + "-Wl,--no-undefined") + add_dependencies(modeling_v2_libs ${library}) + file(WRITE "${binary_dir}/cuda_architectures.txt" "${archs}") +endfunction() + +# Build every declared family's library. Called by +# cpp/tensorrt_llm/modeling_v2/CMakeLists.txt after it added the families. +function(add_modeling_v2_family_libraries) + get_property(families GLOBAL PROPERTY MODELING_V2_FAMILIES) + foreach(family IN LISTS families) + _modeling_v2_add_family_library("${family}") + endforeach() +endfunction() diff --git a/cpp/tensorrt_llm/CMakeLists.txt b/cpp/tensorrt_llm/CMakeLists.txt index 5b06fc841510..fc84a4fc1942 100644 --- a/cpp/tensorrt_llm/CMakeLists.txt +++ b/cpp/tensorrt_llm/CMakeLists.txt @@ -340,12 +340,8 @@ if(NOT WIN32) endif() if(BUILD_PYT) - # modeling_v2 family libraries (cpp/tensorrt_llm/modeling_v2) are separate - # modules loaded on demand; th_common reports the contract fingerprint they - # are checked against, so it is defined before thop. - include(modelingV2) - trtllm_mv2_define_contract_fingerprint() add_subdirectory(thop) + # modeling_v2 family libraries: torch-op libraries loaded on demand. add_subdirectory(modeling_v2) endif() diff --git a/cpp/tensorrt_llm/modeling_v2/CMakeLists.txt b/cpp/tensorrt_llm/modeling_v2/CMakeLists.txt index a38827529a57..95cb4a75e89b 100644 --- a/cpp/tensorrt_llm/modeling_v2/CMakeLists.txt +++ b/cpp/tensorrt_llm/modeling_v2/CMakeLists.txt @@ -15,27 +15,37 @@ # the License. # -# Native code of modeling_v2 targets: one library per model family, loaded only -# when routing picks one of its targets. Every subdirectory with a +# Native code of modeling_v2 targets: one torch-op library per model family, +# loaded only when routing picks one of its targets. Every subdirectory with a # CMakeLists.txt is a family; see README.md and -# cpp/cmake/modules/modelingV2.cmake. +# cpp/cmake/modules/modeling_v2.cmake. + +include(modeling_v2) add_custom_target(modeling_v2_libs) -if(WIN32) - trtllm_mv2_write_disabled_manifest( - "modeling_v2 libraries are not built on Windows") -elseif(NOT BUILD_MODELING_V2) - trtllm_mv2_write_disabled_manifest("built with BUILD_MODELING_V2=OFF") -else() - file( - GLOB family_lists - LIST_DIRECTORIES false - CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/*/CMakeLists.txt") - list(SORT family_lists) +file( + GLOB family_lists + LIST_DIRECTORIES false + CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/*/CMakeLists.txt") +list(SORT family_lists) + +if(WIN32 OR NOT BUILD_MODELING_V2) + # As deep_ep does on a build without a matching architecture: an empty + # cuda_architectures.txt per family, so build_wheel.py installs nothing, + # including a library left from an earlier configuration. foreach(family_list IN LISTS family_lists) get_filename_component(family_dir "${family_list}" DIRECTORY) - add_subdirectory("${family_dir}") + get_filename_component(family_dir_name "${family_dir}" NAME) + file(WRITE + "${CMAKE_CURRENT_BINARY_DIR}/${family_dir_name}/cuda_architectures.txt" + "") endforeach() - trtllm_mv2_finalize() + return() endif() + +foreach(family_list IN LISTS family_lists) + get_filename_component(family_dir "${family_list}" DIRECTORY) + add_subdirectory("${family_dir}") +endforeach() +add_modeling_v2_family_libraries() diff --git a/cpp/tensorrt_llm/modeling_v2/README.md b/cpp/tensorrt_llm/modeling_v2/README.md index c8e37cb6b836..a45b28e2dfbc 100644 --- a/cpp/tensorrt_llm/modeling_v2/README.md +++ b/cpp/tensorrt_llm/modeling_v2/README.md @@ -6,7 +6,8 @@ C++/CUDA used only by modeling_v2 targets (`tensorrt_llm/_torch/_experimental/modeling_v2/`) lives here, not in `cpp/tensorrt_llm/kernels/` or `thop/`. Each model family builds into one -library, `libtrtllm_mv2_.so`. +torch-op library, `libth_modeling_v2_.so`, installed next to +`libth_common.so`. - **Not linked into stock.** Nothing in the family library is linked into `libtensorrt_llm.so` or `libth_common.so`. @@ -24,8 +25,8 @@ cpp/tensorrt_llm/modeling_v2/ CMakeLists.txt adds every family directory (the only stock hook) contractSurface.txt the stock headers native code may include common/ namespace and torch-registration macros - / one model family -> libtrtllm_mv2_.so - CMakeLists.txt trtllm_mv2_family / trtllm_mv2_shared / trtllm_mv2_target + / one model family -> libth_modeling_v2_.so + CMakeLists.txt add_modeling_v2_family / _shared / _target shared/ code more than one target of the family calls / one target kernels/ *.cu *.cuh *.h -- compiled for the target's ARCHS, no torch @@ -51,9 +52,9 @@ The example below adds an op to target `foo_nvfp4__sm_103__tp4` of family 1. **Declare it.** The declaration goes in `cpp/tensorrt_llm/modeling_v2/foo/CMakeLists.txt`: ```cmake - trtllm_mv2_family(foo) + add_modeling_v2_family(foo) - trtllm_mv2_target( + add_modeling_v2_target( foo foo_nvfp4__sm_103__tp4 ARCHS 103a SOURCES kernels/*.cu ops/*.cpp @@ -123,25 +124,26 @@ No stock file changes along the way. ## CMake API -The functions are defined in `cpp/cmake/modules/modelingV2.cmake`. +The functions are defined in `cpp/cmake/modules/modeling_v2.cmake`, in the style +of `add_th_common_family`. | Function | What it declares | |---|---| -| `trtllm_mv2_family()` | The family; call it first in the family's `CMakeLists.txt`. | -| `trtllm_mv2_target( ARCHS ... SOURCES ... [DEPS ...] [REQUIRES ...])` | One target's code, in `/`. | -| `trtllm_mv2_shared( ARCHS ... SOURCES ... [DEPS ...] [REQUIRES ...])` | Code several targets of the family call, in `shared/`. Its ops are in `torch.ops.mv2_`. | +| `add_modeling_v2_family()` | The family; call it first in the family's `CMakeLists.txt`. | +| `add_modeling_v2_target( ARCHS ... SOURCES ... [DEPS ...] [REQUIRES ...])` | One target's code, in `/`. | +| `add_modeling_v2_shared( ARCHS ... SOURCES ... [DEPS ...] [REQUIRES ...])` | Code several targets of the family call, in `shared/`. Its ops are in `torch.ops.mv2_`. | | Keyword | Meaning | |---|---| -| `ARCHS` | `NN`, `NNa` or `NNf`, intersected with the build's requested architectures: `NN`/`NNa` build when `NN` is requested; `NNf` builds when any member of family `NN` is. A target whose intersection is empty is **not built**: no placeholder kernels, and the manifest records why. The declared architectures must cover the SM in the target's name; when moving existing code, keep the architectures it was gated with. | +| `ARCHS` | `NN`, `NNa` or `NNf`, intersected with the build's requested architectures: `NN`/`NNa` build when `NN` is requested; `NNf` builds when any member of family `NN` is. This is how flash_mla and deep_ep choose their architectures. A target whose intersection is empty is **not built**: no placeholder kernels, and the configure step prints why. The declared architectures must cover the SM in the target's name; when moving existing code, keep the architectures it was gated with. | | `SOURCES` | Paths or globs relative to the unit's directory. `.cu` files are compiled for `ARCHS` without torch; `.cpp` files are the torch bindings. | | `DEPS` | `cublas`, `cublaslt`, `cuda_driver`, `internal_cutlass`, `nccl`, `trtllm_gen_fmha_interface`. Ask for anything else by extending the module. | | `REQUIRES` | `MULTI_DEVICE`, `NVSHMEM`. A target whose requirement the build lacks (e.g. a single-device build) is skipped and recorded. | ### Build options -- **`-DBUILD_MODELING_V2=OFF`** builds no family library. The manifest then - says so, and every target with native code is a `NoTarget`. +- **`-DBUILD_MODELING_V2=OFF`** builds no family library, so every target with + native code is a `NoTarget`. - **`-DTRTLLM_MODELING_V2_TARGETS=/;...`** builds only the matching targets. Globs are allowed, e.g. `fixture/*`; the default is `all`. Pass it through @@ -150,7 +152,7 @@ The functions are defined in `cpp/cmake/modules/modelingV2.cmake`. edit relinks only that library: ```bash - cmake --build cpp/build --target trtllm_mv2_ + cmake --build cpp/build --target th_modeling_v2_ ``` ## Namespaces @@ -158,7 +160,7 @@ The functions are defined in `cpp/cmake/modules/modelingV2.cmake`. | Kind | Name | |---|---| | Target torch ops | `torch.ops.mv2__`, with `__` collapsed to `_` | -| Family shared torch ops | `torch.ops.mv2_`; also holds `contract_fingerprint()` | +| Family shared torch ops | `torch.ops.mv2_`; also holds `built_targets()` | | Target C++ | `tensorrt_llm::modeling_v2::::` (`TRTLLM_MV2_NAMESPACE_BEGIN`) | | Shared C++ | `tensorrt_llm::modeling_v2::::shared` | @@ -198,17 +200,11 @@ reserved and not built yet. It links against `libtensorrt_llm.so` (logger, `TllmException`, mcast registry and other singletons stay there), `th_utils` and torch. -**The contract fingerprint.** `contractSurface.txt` is also the stock side of -the contract. The build hashes those headers, the global compile definitions, -and the torch and CUDA versions into a *contract fingerprint*: - -- `libth_common.so` reports it as `torch.ops.trtllm.mv2_contract_fingerprint()`. -- Each family library reports it as `torch.ops.mv2_.contract_fingerprint()`. -- The loader refuses a family library whose fingerprint differs from the - loaded `libth_common.so`'s. - -Adding a header to the list widens what stock changes can affect targets; the -header's stock owners review it. +All family libraries come from the same build as the stock libraries, as +`flash_mla`, `deep_gemm` and `deep_ep` do; a precompiled wheel used with a +checkout is checked by `setup.py`'s native-input skew detection, which covers +all of `cpp/`. Adding a header to `contractSurface.txt` widens what stock +changes can affect targets; the header's stock owners review it. **When a target needs stock to change**, for example a new option on a stock kernel, that change is a separate stock PR. It must leave stock behavior @@ -216,24 +212,28 @@ unchanged by default, and it is reviewed by the stock owners. ## How it loads -`_native.load_target_ops(__name__)` runs these steps: +`_native.load_target_ops(__name__)` treats a family library the way the stock +code treats its optional native modules (`flash_mla`, `deep_ep`): -1. Read `tensorrt_llm/libs/modeling_v2_manifest.json`. -2. Check that the target was built, and built for this GPU. -3. Open the family library. -4. Compare contract fingerprints. -5. Import the target's `fake_ops.py`. +1. Open `tensorrt_llm/libs/libth_modeling_v2_.so`, once per process. +2. Ask it which targets it carries: `torch.ops.mv2_.built_targets()`, + a query op like `th_common`'s `mhc_fused_hc_mma_enabled()`. +3. Import the target's `fake_ops.py`. | Outcome | Raised as | Meaning | |---|---|---| -| The target was skipped or excluded by this build, or does not run on this GPU | `NoTarget` | `auto` keeps the built-in model; `require` reports why | -| The manifest or library is missing, `dlopen` fails, the fingerprint differs, or an op is missing | `NativeLoadError` | A broken installation; never a fallback | +| The library is not installed, or does not carry the target | `NoTarget` | This build did not compile it; `auto` keeps the built-in model, `require` reports why | +| The library is installed but fails to open, or an op `fake_ops.py` declares is missing | `NativeLoadError` | A broken installation; never a fallback | `route` imports a target only on the branch that returns it, so this runs as -part of routing. Every rank then agrees on the result +part of routing, and routing has already checked the device's compute +capability. Every rank then agrees on the result (`modeling_v2_resolve_on_all_ranks`), or every rank fails. -`_native.available_targets()` lists what the installed wheel supports. +**Packaging.** Like deep_ep and flash_mla, each family writes +`cuda_architectures.txt` into its build directory -- the architectures its +library was built for, empty when it has none -- and `scripts/build_wheel.py` +installs the library only when that list is not empty, asserting it exists. ## CI @@ -243,7 +243,7 @@ trees. What it selects depends on what the PR changes: | PR changes only… | CI runs… | |---|---| | One family (`models//` and/or `cpp/tensorrt_llm/modeling_v2//`) | That family's test-db entries plus the modeling_v2 CPU checks | -| Shared modeling_v2 code (catalog, loader, resolver, `common/`, the CMake module, `contractSurface.txt`) | Every modeling_v2 entry | +| Shared modeling_v2 code (catalog, loader, resolver, `common/`, the CMake module) | Every modeling_v2 entry | | Stock code | The full suite | **Why the narrowing is safe.** It relies on these CPU checks, which run in diff --git a/cpp/tensorrt_llm/modeling_v2/common/familyLibrary.version b/cpp/tensorrt_llm/modeling_v2/common/familyLibrary.version index 9feababdfb1a..59b171adfea3 100644 --- a/cpp/tensorrt_llm/modeling_v2/common/familyLibrary.version +++ b/cpp/tensorrt_llm/modeling_v2/common/familyLibrary.version @@ -1,7 +1,7 @@ /* A modeling_v2 family library exports nothing: its torch ops register from * static initializers when the library is loaded, and keeping every symbol * local lets two family libraries define the same C++ names in one process. - * See cpp/cmake/modules/modelingV2.cmake. */ + * See cpp/cmake/modules/modeling_v2.cmake. */ { local: *; }; diff --git a/cpp/tensorrt_llm/modeling_v2/common/namespace.h b/cpp/tensorrt_llm/modeling_v2/common/namespace.h index 397bccb42317..0bad346d62a4 100644 --- a/cpp/tensorrt_llm/modeling_v2/common/namespace.h +++ b/cpp/tensorrt_llm/modeling_v2/common/namespace.h @@ -19,10 +19,10 @@ #include "tensorrt_llm/common/config.h" -// trtllm_mv2_target() and trtllm_mv2_shared() define TRTLLM_MV2_SCOPE as -// `::` or `::shared` (cpp/cmake/modules/modelingV2.cmake). +// add_modeling_v2_target() and add_modeling_v2_shared() define TRTLLM_MV2_SCOPE +// as `::` or `::shared` (cpp/cmake/modules/modeling_v2.cmake). #ifndef TRTLLM_MV2_SCOPE -#error "modeling_v2 native code is compiled through trtllm_mv2_target() or trtllm_mv2_shared()" +#error "modeling_v2 native code is compiled through add_modeling_v2_target() or add_modeling_v2_shared()" #endif //! Opens tensorrt_llm::modeling_v2:::: (or ::shared) for the unit diff --git a/cpp/tensorrt_llm/modeling_v2/common/torchLibrary.h b/cpp/tensorrt_llm/modeling_v2/common/torchLibrary.h index 64e455da6268..2b5de20fcd2b 100644 --- a/cpp/tensorrt_llm/modeling_v2/common/torchLibrary.h +++ b/cpp/tensorrt_llm/modeling_v2/common/torchLibrary.h @@ -19,12 +19,12 @@ #include -// trtllm_mv2_target() defines TRTLLM_MV2_TORCH_NS as mv2__ and -// trtllm_mv2_shared() as mv2_ (cpp/cmake/modules/modelingV2.cmake). The +// add_modeling_v2_target() defines TRTLLM_MV2_TORCH_NS as mv2__ +// and add_modeling_v2_shared() as mv2_ (cpp/cmake/modules/modeling_v2.cmake). The // family library claims each namespace once with TORCH_LIBRARY, so op files only // ever add fragments to it. #ifndef TRTLLM_MV2_TORCH_NS -#error "modeling_v2 torch bindings are compiled through trtllm_mv2_target() or trtllm_mv2_shared()" +#error "modeling_v2 torch bindings are compiled through add_modeling_v2_target() or add_modeling_v2_shared()" #endif // One level of indirection so that TORCH_LIBRARY_*'s own # and ## operators see diff --git a/cpp/tensorrt_llm/modeling_v2/contractSurface.txt b/cpp/tensorrt_llm/modeling_v2/contractSurface.txt index c55f822396ef..54628fb5df4a 100644 --- a/cpp/tensorrt_llm/modeling_v2/contractSurface.txt +++ b/cpp/tensorrt_llm/modeling_v2/contractSurface.txt @@ -2,14 +2,9 @@ # SPDX-License-Identifier: Apache-2.0 # # The stock headers modeling_v2 native code may include, as written in #include -# (resolved against cpp/include and cpp/). This list is the contract between -# stock and the family libraries: -# -# * cpp/cmake/modules/modelingV2.cmake hashes these headers into the contract -# fingerprint that th_common and every family library report, and the -# loader refuses a family library whose fingerprint differs; -# * tests/unittest/_torch/modeling_v2/test_modeling_v2_native_claims.py -# rejects any other stock include under cpp/tensorrt_llm/modeling_v2. +# (resolved against cpp/include and cpp/): the stock surface target C++ +# depends on. tests/unittest/_torch/modeling_v2/test_modeling_v2_native_claims.py +# rejects any other stock include under cpp/tensorrt_llm/modeling_v2. # # Adding a header widens what stock changes can affect modeling_v2 targets; # it is reviewed by the stock owners of that header. diff --git a/cpp/tensorrt_llm/modeling_v2/fixture/CMakeLists.txt b/cpp/tensorrt_llm/modeling_v2/fixture/CMakeLists.txt index 9e48b10dafc7..47b8fee5bfbc 100644 --- a/cpp/tensorrt_llm/modeling_v2/fixture/CMakeLists.txt +++ b/cpp/tensorrt_llm/modeling_v2/fixture/CMakeLists.txt @@ -16,13 +16,13 @@ # # The infrastructure fixture: one target with one op, so that CI drives the -# whole native path -- build, packaging, load, contract fingerprint, fake op, +# whole native path -- build, packaging, load, built_targets(), fake op, # GPU op -- on every GPU it has. It is also the smallest complete family to copy # (see ../README.md). Its architectures cover every GPU CI runs on; a real # target declares the architectures it was written and gated for. -trtllm_mv2_family(fixture) +add_modeling_v2_family(fixture) -trtllm_mv2_target( +add_modeling_v2_target( fixture scale_add__sm_80__default ARCHS 80 90 100f 120f SOURCES kernels/scaleAdd.cu ops/scaleAddOp.cpp) diff --git a/cpp/tensorrt_llm/thop/CMakeLists.txt b/cpp/tensorrt_llm/thop/CMakeLists.txt index 4108cc8f76d5..fa7c0a458fa2 100644 --- a/cpp/tensorrt_llm/thop/CMakeLists.txt +++ b/cpp/tensorrt_llm/thop/CMakeLists.txt @@ -171,13 +171,6 @@ add_th_common_family( # model. add_subdirectory(glmMoeDsa) -# trtllm::mv2_contract_fingerprint: the stock side of the check the modeling_v2 -# loader makes before it uses a family library (cpp/tensorrt_llm/modeling_v2). -add_th_common_family(th_common_modeling_v2 modelingV2ContractOp.cpp) -add_dependencies(th_common_modeling_v2 trtllm_mv2_contract_fingerprint) -target_include_directories(th_common_modeling_v2 - PRIVATE ${TRTLLM_MV2_GENERATED_DIR}) - add_library( th_common SHARED $ @@ -186,8 +179,7 @@ add_library( $ $ $ - $ - $) + $) set_property(TARGET th_common PROPERTY POSITION_INDEPENDENT_CODE ON) set_target_properties(th_common PROPERTIES CXX_STANDARD 20 CXX_STANDARD_REQUIRED ON) diff --git a/cpp/tensorrt_llm/thop/modelingV2ContractOp.cpp b/cpp/tensorrt_llm/thop/modelingV2ContractOp.cpp deleted file mode 100644 index c8467f34a09c..000000000000 --- a/cpp/tensorrt_llm/thop/modelingV2ContractOp.cpp +++ /dev/null @@ -1,46 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Generated by trtllm_mv2_define_contract_fingerprint (cpp/cmake/modules/modelingV2.cmake). -#include "mv2ContractFingerprint.h" - -#include -#include - -namespace -{ - -//! The contract fingerprint this library was built with: a hash of the stock headers listed in -//! cpp/tensorrt_llm/modeling_v2/contractSurface.txt, the global compile definitions, and the torch and CUDA -//! versions. Every modeling_v2 family library reports its own, and the loader -//! (tensorrt_llm/_torch/_experimental/modeling_v2/_native.py) refuses one that differs from this. -std::string mv2ContractFingerprint() -{ - return TRTLLM_MV2_CONTRACT_FINGERPRINT; -} - -} // namespace - -TORCH_LIBRARY_FRAGMENT(trtllm, m) -{ - m.def("mv2_contract_fingerprint() -> str"); -} - -TORCH_LIBRARY_IMPL(trtllm, CompositeExplicitAutograd, m) -{ - m.impl("mv2_contract_fingerprint", &mv2ContractFingerprint); -} diff --git a/jenkins/scripts/cbts/README.md b/jenkins/scripts/cbts/README.md index c1b5ae6a3886..138ab4589b6d 100644 --- a/jenkins/scripts/cbts/README.md +++ b/jenkins/scripts/cbts/README.md @@ -50,7 +50,7 @@ Nine rules, registered in `cbts/command/main.py::RULE_CLASSES`: | `TestListRule` | `testlistonly` | `tests/integration/test_lists/test-db/*.yml` | | `VisualGenRule` | `visualgenonly` | `examples/visual_gen/**`, `scripts/visualgen_eval/**`, `tensorrt_llm/_torch/visual_gen/**`, `tensorrt_llm/media/**`, `tensorrt_llm/visual_gen/**` (excl. docs; reference images such as `cat_piano.png` ARE test fixtures and stay claimed; outward-facing files force fallback) | | `SpecDecRule` | `specdeconly` | `tensorrt_llm/_torch/speculative/**`, `tensorrt_llm/models/{eagle,medusa,redrafter}/**`, `examples/{eagle,medusa,redrafter,draft_target_model,ngram}/**`, `examples/llm-api/llm_speculative_decoding.py` (excl. docs; other suffixes incl. images kept as potential test fixtures) | -| `ModelingV2Rule` | `modelingv2only` | `tensorrt_llm/_torch/_experimental/modeling_v2/**`, `cpp/tensorrt_llm/modeling_v2/**`, `cpp/cmake/modules/modelingV2*.cmake` (excl. docs); per family when only one family changed | +| `ModelingV2Rule` | `modelingv2only` | `tensorrt_llm/_torch/_experimental/modeling_v2/**`, `cpp/tensorrt_llm/modeling_v2/**`, `cpp/cmake/modules/modeling_v2.cmake` (excl. docs); per family when only one family changed | | `AgentFlowRule` | `agentflowonly` | `agent-flow/**` (excl. docs and out-of-scope metadata) | | `OpenEngineRule` | `openengineonly` | `tensorrt_llm/grpc/openengine/**` (excl. docs) | | `DocsRule` | `docsonly` | Outside noop paths: `docs/**` and verified Python docstring edits → `CPU-Build_Docs` plus complete `l0_cpu`; other `**/*.md` / `**/*.rst` → `CPU-Build_Docs` only | @@ -67,7 +67,7 @@ See `cbts/rules/README.md` for per-rule logic. | `testlistonly` | `TestListRule` fired solo: PR only adds entries under `tests/integration/test_lists/test-db/*.yml`. | | `visualgenonly` | `VisualGenRule` fired solo: PR only touches VisualGen internal source paths (`examples/visual_gen/**`, `scripts/visualgen_eval/**`, `tensorrt_llm/_torch/visual_gen/**`; excl. docs; image fixtures like `cat_piano.png` are claimed). Narrows to blocks containing VG test entries. Outward-facing files under `tensorrt_llm/visual_gen/**` and `tensorrt_llm/media/**` (eagerly imported by `trtllm-serve`) force `null` fallback. | | `specdeconly` | `SpecDecRule` fired solo: PR only touches speculative-decoding source paths (`tensorrt_llm/_torch/speculative/**`, `tensorrt_llm/models/{eagle,medusa,redrafter}/**`, `examples/{eagle,medusa,redrafter,draft_target_model,ngram}/**`, `examples/llm-api/llm_speculative_decoding.py`; excl. docs). Narrows to blocks containing spec-dec test entries (eagle / medusa / redrafter / ngram / draft-target-model / MTP). | -| `modelingv2only` | `ModelingV2Rule` fired solo: PR only touches modeling_v2 code -- the Python subtree (`tensorrt_llm/_torch/_experimental/modeling_v2/**`), its native code (`cpp/tensorrt_llm/modeling_v2/**`, one library per model family) or the native build module (`cpp/cmake/modules/modelingV2*.cmake`); excl. docs, which are a fifth of the Python subtree — every catalog entry carries a contract document. A change confined to one family's directories narrows to that family's entries (`accuracy/modeling_v2/test_.py`, `unittest/_torch/modeling_v2/models/`) plus the modeling_v2 CPU checks; any other modeling_v2 change narrows to every modeling_v2 entry. No outward-facing fallback is needed: nothing beyond the resolver module is imported, and no family library is loaded, unless `LLM(modeling_v2=...)` is on; the CPU checks that keep this true run in every scope. The callers outside the subtree (`_torch/pyexecutor/model_loader.py`, `_torch/models/modeling_auto.py`, the `trtllm::mv2_contract_fingerprint` op in `cpp/tensorrt_llm/thop/`) are left unclaimed, so touching them falls back to baseline. | +| `modelingv2only` | `ModelingV2Rule` fired solo: PR only touches modeling_v2 code -- the Python subtree (`tensorrt_llm/_torch/_experimental/modeling_v2/**`), its native code (`cpp/tensorrt_llm/modeling_v2/**`, one torch-op library per model family) or the native build module (`cpp/cmake/modules/modeling_v2.cmake`); excl. docs, which are a fifth of the Python subtree — every catalog entry carries a contract document. A change confined to one family's directories narrows to that family's entries (`accuracy/modeling_v2/test_.py`, `unittest/_torch/modeling_v2/models/`) plus the modeling_v2 CPU checks; any other modeling_v2 change narrows to every modeling_v2 entry. No outward-facing fallback is needed: nothing beyond the resolver module is imported, and no family library is loaded, unless `LLM(modeling_v2=...)` is on; the CPU checks that keep this true run in every scope. The callers outside the subtree (`_torch/pyexecutor/model_loader.py`, `_torch/models/modeling_auto.py`) are left unclaimed, so touching them falls back to baseline. | | `agentflowonly` | `AgentFlowRule` fired solo: PR only touches `agent-flow/**` source or test files (excl. docs). Runs `CPU-AgentFlow-UnitTest`. | | `openengineonly` | `OpenEngineRule` fired solo: PR only touches `tensorrt_llm/grpc/openengine/**` source files (excl. docs). Narrows to the registered OpenEngine unit tests: the stub-based ones on the always-run `CPU-Generic-*` stages, plus `test_capability_conformance.py` on `A10-PyTorch-*`, which needs a GPU. | | `docsonly` | `DocsRule` fired solo: all documentation changes run `CPU-Build_Docs`; changes under `docs/**` also run the complete `l0_cpu` suite on matching `CPU-Generic-*` stages. Docs-only orchestration skips the SBSA track, so its resolved ARM stage is not launched. | diff --git a/jenkins/scripts/cbts/cbts/rules/README.md b/jenkins/scripts/cbts/cbts/rules/README.md index aba2aa56619c..6a8ac735b8d9 100644 --- a/jenkins/scripts/cbts/cbts/rules/README.md +++ b/jenkins/scripts/cbts/cbts/rules/README.md @@ -16,7 +16,7 @@ for the overall CBTS architecture. | `test_list.py` | `TestListRule` | `testlistonly` | `tests/integration/test_lists/test-db/*.yml` | | `visual_gen.py` | `VisualGenRule` | `visualgenonly` | `examples/visual_gen/**`, `scripts/visualgen_eval/**`, `tensorrt_llm/_torch/visual_gen/**`, `tensorrt_llm/media/**`, `tensorrt_llm/visual_gen/**` (each excl. docs) | | `spec_dec.py` | `SpecDecRule` | `specdeconly` | `tensorrt_llm/_torch/speculative/**`, `tensorrt_llm/models/{eagle,medusa,redrafter}/**`, `examples/{eagle,medusa,redrafter,draft_target_model,ngram}/**`, `examples/llm-api/llm_speculative_decoding.py` (each excl. docs) | -| `modeling_v2.py` | `ModelingV2Rule` | `modelingv2only` | `tensorrt_llm/_torch/_experimental/modeling_v2/**`, `cpp/tensorrt_llm/modeling_v2/**`, `cpp/cmake/modules/modelingV2*.cmake` (excl. docs); per family when only one family changed | +| `modeling_v2.py` | `ModelingV2Rule` | `modelingv2only` | `tensorrt_llm/_torch/_experimental/modeling_v2/**`, `cpp/tensorrt_llm/modeling_v2/**`, `cpp/cmake/modules/modeling_v2.cmake` (excl. docs); per family when only one family changed | | `agent_flow.py` | `AgentFlowRule` | `agentflowonly` | `agent-flow/**` (excl. docs and out-of-scope metadata) → the single `CPU-AgentFlow-UnitTest` stage; not test-db-driven | | `openengine.py` | `OpenEngineRule` | `openengineonly` | `tensorrt_llm/grpc/openengine/**` (excl. docs) → the `l0_cpu` block containing `unittest/grpc/openengine/` | | `docs.py` | `DocsRule` | `docsonly` | Outside noop paths: `docs/**` and verified Python docstring edits → `CPU-Build_Docs` plus complete `l0_cpu`; other `**/*.md` / `**/*.rst` → `CPU-Build_Docs` only | @@ -304,9 +304,9 @@ Path-only rule. Claims non-documentation source changes under `tensorrt_llm/_torch/_experimental/modeling_v2/`, the self-contained second modeling path (one flat forward per checkpoint/arch/parallel triple, assembled from a catalog of op wrappers), under `cpp/tensorrt_llm/modeling_v2/`, its -native code (one library per model family, loaded only when routing picks +native code (one torch-op library per model family, loaded only when routing picks one of the family's targets), and the native build module -`cpp/cmake/modules/modelingV2*.cmake`. +`cpp/cmake/modules/modeling_v2.cmake`. Scope — per family, else all of modeling_v2: a change confined to one family's directories (`models//` in Python, diff --git a/jenkins/scripts/cbts/cbts/rules/modeling_v2.py b/jenkins/scripts/cbts/cbts/rules/modeling_v2.py index 9db8beae3026..e97394a3088a 100644 --- a/jenkins/scripts/cbts/cbts/rules/modeling_v2.py +++ b/jenkins/scripts/cbts/cbts/rules/modeling_v2.py @@ -17,8 +17,8 @@ (checkpoint, GPU arch, parallel topology), assembled from a catalog of op wrappers. Its Python lives under `tensorrt_llm/_torch/_experimental/modeling_v2/` and its native code under `cpp/tensorrt_llm/modeling_v2/`, which builds one -library per model family, loaded only when routing picks one of the family's -targets. +torch-op library per model family, loaded only when routing picks one of the +family's targets. Scope — per family, else all of modeling_v2: A change confined to one family's directories (`models//` in Python, @@ -26,7 +26,7 @@ entries -- `accuracy/modeling_v2/test_.py` and `unittest/_torch/modeling_v2/models//` -- plus the modeling_v2 CPU checks. Any other modeling_v2 change (catalog, resolver, loader, the native -build module and contract surface) selects every modeling_v2 entry. Several +build module and its common headers) selects every modeling_v2 entry. Several families in one PR union their entries. Why no outward fallback is needed: nothing beyond the resolver is imported, and @@ -38,8 +38,8 @@ keep that true (stock never references a target, targets never reach a sibling, `modeling_v2` defaults to `off`); they run in every scope this rule selects. The callers outside the subtree (`_torch/pyexecutor/model_loader.py`, -`_torch/models/modeling_auto.py`, the `trtllm::mv2_contract_fingerprint` op in -`cpp/tensorrt_llm/thop/`) stay unclaimed: a change to them runs the baseline. +`_torch/models/modeling_auto.py`) stay unclaimed: a change to them runs the +baseline. `.md` exclusion matters here: every catalog entry carries a contract document, so a fifth of the Python subtree is Markdown; claiming it would let a @@ -71,12 +71,7 @@ # Directories of the native tree that are not families. _CPP_NON_FAMILY_DIRS = frozenset({"common", "catalog"}) # The native build module: shared by every family library. -_CPP_BUILD_FILES = frozenset( - { - "cpp/cmake/modules/modelingV2.cmake", - "cpp/cmake/modules/modelingV2Fingerprint.cmake", - } -) +_CPP_BUILD_FILES = frozenset({"cpp/cmake/modules/modeling_v2.cmake"}) # Source-path prefixes the rule may claim. Tests under tests/** are left to # TestsDefRule; the two scopes combine via _TESTSONLY_FAMILY. @@ -104,7 +99,7 @@ def _camel(name: str) -> str: """The C++ directory of a family: `minimax_m3` -> `minimaxM3`. - Mirrors `_trtllm_mv2_camel` in cpp/cmake/modules/modelingV2.cmake. + Mirrors `_modeling_v2_camel` in cpp/cmake/modules/modeling_v2.cmake. """ parts = [p for p in name.split("_") if p] return parts[0] + "".join(p[:1].upper() + p[1:] for p in parts[1:]) diff --git a/scripts/build_wheel.py b/scripts/build_wheel.py index d12a33a67e26..a450d4c36133 100755 --- a/scripts/build_wheel.py +++ b/scripts/build_wheel.py @@ -794,34 +794,20 @@ def has_sm90_or_newer(cuda_architectures: str) -> bool: def install_modeling_v2_libs(mv2_build_dir: Path, lib_dir: Path, install_file) -> None: - """Install the modeling_v2 family libraries and their manifest. + """Install the modeling_v2 family libraries the way deep_ep and flash_mla are. - The manifest is what the loader reads - (tensorrt_llm/_torch/_experimental/modeling_v2/_native.py). This is also - the packaging completeness check: the manifest lists, per family, the - library its built targets need, and a listed library that is missing fails - the build here rather than turning into a modeling_v2 fallback at run time. + Each family's build directory holds cuda_architectures.txt, written at + configure time: the architectures its library was built for, empty when + there is none (cpp/cmake/modules/modeling_v2.cmake). """ - manifest_path = mv2_build_dir / "modeling_v2_manifest.json" - if not manifest_path.is_file(): - raise RuntimeError( - f"{manifest_path} is missing; the cmake configure step writes it " - "(cpp/tensorrt_llm/modeling_v2/CMakeLists.txt)") - manifest = json.loads(manifest_path.read_text()) - install_file(manifest_path, lib_dir / manifest_path.name) - missing = [] - for family, entry in sorted(manifest.get("families", {}).items()): - library = entry.get("library") - if library is None: + for arch_file in sorted(mv2_build_dir.glob("*/cuda_architectures.txt")): + if not arch_file.read_text().strip().strip(";"): continue - if not (mv2_build_dir / library).is_file(): - missing.append(f"{family}: {library}") - continue - install_file(mv2_build_dir / library, lib_dir / library) - if missing: - raise RuntimeError( - "modeling_v2 packaging check: the manifest lists libraries the build " - f"did not produce: {', '.join(missing)}") + libraries = list(arch_file.parent.glob("libth_modeling_v2_*.so")) + assert len(libraries) == 1, ( + "Exactly one modeling_v2 library should be present in " + f"{arch_file.parent}: {libraries}") + install_file(libraries[0], lib_dir / libraries[0].name) def stage_nccl_extensions_package(wheel: Path, staging_dir: Path) -> None: diff --git a/setup.py b/setup.py index 8dddac528eac..4d3a518bce79 100644 --- a/setup.py +++ b/setup.py @@ -179,8 +179,7 @@ def has_ext_modules(self): package_data = [ 'libs/libtensorrt_llm.so', 'libs/libth_common.so', - 'libs/libtrtllm_mv2_*.so', - 'libs/modeling_v2_manifest.json', + 'libs/libth_modeling_v2_*.so', 'libs/libtensorrt_llm_ucx_wrapper.so', 'libs/libdecoder_attention_0.so', 'libs/libtensorrt_llm_nixl_wrapper.so', diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/README.md b/tensorrt_llm/_torch/_experimental/modeling_v2/README.md index b7dee6dbd746..8a5b2301b2c1 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/README.md +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/README.md @@ -125,12 +125,14 @@ catalog// entries more than one target calls A target's C++/CUDA lives beside the stock tree, not in it: `cpp/tensorrt_llm/modeling_v2///`, built into one -library per family, `libtrtllm_mv2_.so`. Nothing of it is linked into -`libtensorrt_llm.so` or `libth_common.so`, and nothing loads it until routing -imports one of the family's targets, whose `__init__.py` starts with -`native_ops = load_target_ops(__name__)`. A target this build or this device -cannot run is a `NoTarget`, so `auto` keeps the built-in model; a broken -installation is a `NativeLoadError`, never a fallback. +torch-op library per family, `libth_modeling_v2_.so`, installed next to +`libth_common.so`. Nothing of it is linked into `libtensorrt_llm.so` or +`libth_common.so`, and nothing loads it until routing imports one of the +family's targets, whose `__init__.py` starts with +`native_ops = load_target_ops(__name__)`. As with the stock code's optional +native modules, a library this installation does not carry is a `NoTarget`, so +`auto` keeps the built-in model; one that is installed but broken is a +`NativeLoadError`, never a fallback. `cpp/tensorrt_llm/modeling_v2/README.md` is the guide, and `models/fixture/` is the smallest complete example. diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/_native.py b/tensorrt_llm/_torch/_experimental/modeling_v2/_native.py index ee872fd8bbff..bfdbbd267793 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/_native.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/_native.py @@ -3,30 +3,28 @@ """A target's native (C++/CUDA) code: where it is built, and how the target loads it. Native code of a model family lives in ``cpp/tensorrt_llm/modeling_v2//`` -and builds into one library, ``libtrtllm_mv2_.so``, installed next to -``libth_common.so`` together with ``modeling_v2_manifest.json``. Nothing of it is in -the stock libraries, and nothing loads it until routing picks a target of the -family: a target package that has native code starts with:: +and builds into one torch-op library, ``libth_modeling_v2_.so``, installed +next to ``libth_common.so``. Nothing of it is in the stock libraries, and nothing +loads it until routing picks a target of the family: a target package that has +native code starts with:: from tensorrt_llm._torch._experimental.modeling_v2._native import load_target_ops - ops = load_target_ops(__name__) # torch.ops.mv2__ + native_ops = load_target_ops(__name__) # torch.ops.mv2__ ``route`` imports the target only on the branch that returns it, so loading is part -of routing: a target whose code this wheel did not build raises :class:`NoTarget` -(``auto`` keeps the built-in model, ``require`` reports why), while a library that -is present but unusable raises :class:`NativeLoadError`, which is never a fallback. - -The checks, in order: - -1. the manifest lists the target as built (otherwise ``NoTarget``); -2. the device's compute capability is covered by an architecture the target was - built for (otherwise ``NoTarget``); -3. the family library opens from the fixed package path; -4. its contract fingerprint equals the loaded ``libth_common.so``'s -- the two were - compiled against the same stock headers, definitions, torch and CUDA; -5. the target's ``fake_ops.py`` registers, which fails if an op it declares is - missing from the library. +of routing, and it follows how the stock code treats its optional native modules +(``flash_mla``, ``deep_ep``): + +* a library this installation does not have is a feature that is not there: + :class:`NoTarget`, so ``auto`` keeps the built-in model and ``require`` says why; +* a library that is there but fails to load is an error, :class:`NativeLoadError`, + never a fallback; +* which targets a library carries it answers itself, + ``torch.ops.mv2_.built_targets()``, the way ``th_common`` answers feature + queries; a target it does not carry is :class:`NoTarget`; +* the target's ``fake_ops.py`` registers right after the library opens, which fails + if an op it declares is missing from the library. ``cpp/tensorrt_llm/modeling_v2/README.md`` is the guide for authors. """ @@ -35,37 +33,31 @@ import importlib import importlib.util -import json import sys -from functools import lru_cache from pathlib import Path -from typing import Any, Dict, Iterable, List, Optional, Tuple +from typing import Any, Dict, FrozenSet, Optional, Tuple from ._router_index import NoTarget -MANIFEST_NAME = "modeling_v2_manifest.json" -MANIFEST_SCHEMA = 1 - _PACKAGE = __name__.rpartition(".")[0] _MODELS_PACKAGE = f"{_PACKAGE}.models" -# tensorrt_llm/libs, where build_wheel.py installs the family libraries. +# tensorrt_llm/libs, where build_wheel.py installs libth_common.so and the family libraries. _LIBS_DIR = Path(__file__).resolve().parents[3] / "libs" -_loaded_libraries: set = set() +_built_targets: Dict[str, FrozenSet[str]] = {} class NativeLoadError(RuntimeError): - """A family library the manifest lists as built cannot be used. + """A family library is installed but cannot be used. - Raised for a broken or mismatched installation: a missing library, one that - fails to open, a contract-fingerprint mismatch, an op ``fake_ops.py`` declares - but the library lacks. It is deliberately not a :class:`NoTarget`, so ``auto`` - does not hide it behind the built-in model. + Raised when the library fails to open or an op the target's ``fake_ops.py`` + declares is missing from it. It is deliberately not a :class:`NoTarget`, so + ``auto`` does not hide a broken installation behind the built-in model. """ -# --- Naming (mirrors cpp/cmake/modules/modelingV2.cmake) --------------------- +# --- Naming (mirrors cpp/cmake/modules/modeling_v2.cmake) -------------------- def camel(name: str) -> str: @@ -89,6 +81,11 @@ def torch_namespace(family: str, target: Optional[str] = None) -> str: return f"mv2_{family}_{identifier(target)}" +def library_name(family: str) -> str: + """The file name of a family's library.""" + return f"libth_modeling_v2_{family}.so" + + def split_target_package(package: str) -> Tuple[str, str]: """``(family, target)`` of a target package name.""" prefix = f"{_MODELS_PACKAGE}." @@ -105,200 +102,69 @@ def namespace_of(package: str) -> str: return torch_namespace(*split_target_package(package)) -# --- Architectures --------------------------------------------------------------- - - -def arch_covers(arch: str, capability: Tuple[int, int]) -> bool: - """Whether code built for ``arch`` (a manifest entry such as ``100f-real``) runs on - a device of compute capability ``capability``. - - ``NNa`` runs on exactly NN; ``NN`` on NN and later minor versions of the same - major version; ``NNf`` on every member of family NN, which this build treats the - same way. - """ - name = arch[: -len("-real")] if arch.endswith("-real") else arch - suffix = name[-1] if name[-1] in "af" else "" - number = int(name[:-1] if suffix else name) - major, minor = divmod(number, 10) - if suffix == "a": - return capability == (major, minor) - return capability[0] == major and capability[1] >= minor - - -# --- Manifest ----------------------------------------------------------------- - - -@lru_cache(maxsize=None) -def read_manifest(path: Optional[Path] = None) -> Dict[str, Any]: - """The manifest the build wrote next to the family libraries.""" - path = path or (_LIBS_DIR / MANIFEST_NAME) - try: - manifest = json.loads(path.read_text()) - except FileNotFoundError: - raise NativeLoadError( - f"{path} is missing: this installation of TensorRT-LLM has no modeling_v2 " - "manifest. Rebuild with scripts/build_wheel.py." - ) from None - except (OSError, ValueError) as error: - raise NativeLoadError(f"{path} cannot be read: {error}") from None - if manifest.get("schema") != MANIFEST_SCHEMA: - raise NativeLoadError( - f"{path} has schema {manifest.get('schema')!r}; this loader reads schema " - f"{MANIFEST_SCHEMA}. The Python package and the native libraries are from " - "different builds." - ) - return manifest - - -def available_targets(manifest: Optional[Dict[str, Any]] = None) -> List[Dict[str, Any]]: - """Every target the manifest knows, with its status, architectures and reason -- - what this installation supports.""" - manifest = read_manifest() if manifest is None else manifest - rows = [] - for family, entry in sorted(manifest.get("families", {}).items()): - for target, unit in sorted(entry.get("targets", {}).items()): - rows.append( - { - "family": family, - "target": target, - "status": unit["status"], - "archs": list(unit["archs"]), - "reason": unit["reason"], - "library": entry.get("library"), - } - ) - return rows - - -def _family_entry(manifest: Dict[str, Any], family: str) -> Dict[str, Any]: - if not manifest.get("enabled", False): - raise NoTarget( - f"this TensorRT-LLM build has no modeling_v2 libraries: {manifest.get('reason')}" - ) - entry = manifest.get("families", {}).get(family) - if entry is None: - raise NativeLoadError( - f"modeling_v2 family {family!r} has native code but is not in the manifest; " - "the Python package and the native libraries are from different builds" - ) - return entry - - -def _target_entry(manifest: Dict[str, Any], family: str, target: str) -> Dict[str, Any]: - entry = _family_entry(manifest, family) - unit = entry.get("targets", {}).get(target) - if unit is None: - raise NativeLoadError( - f"modeling_v2 target {family}/{target} has native code but is not in the " - "manifest; the Python package and the native libraries are from different builds" - ) - if unit["status"] != "built": - raise NoTarget( - f"{family}/{target}: native code not built in this TensorRT-LLM build " - f"({unit['status']}: {unit['reason']})" - ) - return unit - - # --- Loading ------------------------------------------------------------------ -def _ops_namespace(namespace: str) -> Any: +def _open_library(path: Path, family: str) -> FrozenSet[str]: + """dlopen a family library and return the targets it carries.""" import torch - return getattr(torch.ops, namespace) + torch.ops.load_library(str(path)) + answer = getattr(torch.ops, torch_namespace(family)).built_targets() + return frozenset(t for t in answer.split(",") if t) -def load_family_library(family: str, manifest: Optional[Dict[str, Any]] = None) -> None: - """Open ``libtrtllm_mv2_.so`` and check its contract fingerprint. +def load_family_library(family: str) -> FrozenSet[str]: + """Open ``libth_modeling_v2_.so`` and return the targets it carries. - Idempotent. Does not look at a device, so it can run anywhere the stock - libraries load; :func:`load_target_ops` adds the target-level checks. + Idempotent, and independent of the device, so it can run anywhere the stock + libraries load. :func:`load_target_ops` adds the target-level checks. """ - manifest = read_manifest() if manifest is None else manifest - entry = _family_entry(manifest, family) - library = entry.get("library") - if library is None: - raise NoTarget(f"modeling_v2 family {family!r}: no target of it is built in this build") - path = _LIBS_DIR / library - if path in _loaded_libraries: - return + if family in _built_targets: + return _built_targets[family] + if sys.platform == "win32": + raise NoTarget("modeling_v2 native code is not built on Windows") + path = _LIBS_DIR / library_name(family) if not path.is_file(): - raise NativeLoadError(f"{path} is listed in the manifest but missing") - - import torch - - try: - torch.ops.load_library(str(path)) - except OSError as error: - raise NativeLoadError(f"{path} cannot be loaded: {error}") from None - - try: - stock = torch.ops.trtllm.mv2_contract_fingerprint() - except (AttributeError, RuntimeError) as error: - raise NativeLoadError( - f"libth_common.so does not report a modeling_v2 contract fingerprint ({error}); " - f"it predates {path.name}" - ) from None - fingerprint = _ops_namespace(torch_namespace(family)).contract_fingerprint() - if fingerprint != stock: - raise NativeLoadError( - f"{path.name} was built against a different stock contract than the loaded " - f"libth_common.so (fingerprint {fingerprint} vs {stock}); install libraries " - "from one build" + raise NoTarget( + f"this installation has no {path.name}: the build compiles a family's library " + "only when its CUDA architectures include one of the family's targets, and " + "only with BUILD_MODELING_V2=ON" ) - _loaded_libraries.add(path) - - -def _device_capability() -> Optional[Tuple[int, int]]: - import torch - - if not torch.cuda.is_available(): - return None - return torch.cuda.get_device_capability() + try: + built = _open_library(path, family) + except (AttributeError, OSError, RuntimeError) as error: + raise NativeLoadError(f"{path} is installed but cannot be loaded: {error}") from None + _built_targets[family] = built + return built -def load_target_ops(package: str, manifest: Optional[Dict[str, Any]] = None) -> Any: +def load_target_ops(package: str) -> Any: """Load the native code of the target package ``package`` and return its ops namespace, ``torch.ops.mv2__``. Call it from the target's ``__init__.py`` with ``__name__``. Raises - :class:`NoTarget` when this build or this device cannot run the target and - :class:`NativeLoadError` when the installation is broken. + :class:`NoTarget` when this installation does not carry the target and + :class:`NativeLoadError` when its library is broken. """ - if sys.platform == "win32": - raise NoTarget("modeling_v2 native code is not built on Windows") family, target = split_target_package(package) - manifest = read_manifest() if manifest is None else manifest - unit = _target_entry(manifest, family, target) - - capability = _device_capability() - if capability is not None and not any(arch_covers(a, capability) for a in unit["archs"]): + built = load_family_library(family) + if target not in built: + carried = ", ".join(sorted(built)) or "none" raise NoTarget( - f"{family}/{target}: built for {', '.join(unit['archs'])}, which does not run " - f"on this sm_{capability[0]}{capability[1]} device" + f"{library_name(family)} does not carry {target}: this build did not compile " + f"it (its ARCHS miss the build's, a REQUIRES is unmet, or " + f"TRTLLM_MODELING_V2_TARGETS excluded it); it carries {carried}" ) - load_family_library(family, manifest) - fake_ops = f"{package}.fake_ops" if importlib.util.find_spec(fake_ops) is not None: try: importlib.import_module(fake_ops) except (AttributeError, RuntimeError, ValueError) as error: raise NativeLoadError( - f"{fake_ops} does not match the ops in the family library: {error}" + f"{fake_ops} does not match the ops in {library_name(family)}: {error}" ) from None - return _ops_namespace(unit["namespace"]) - - -def built_families(manifest: Optional[Dict[str, Any]] = None) -> Iterable[str]: - """Families whose library this build produced.""" - manifest = read_manifest() if manifest is None else manifest - if not manifest.get("enabled", False): - return [] - return sorted( - family - for family, entry in manifest.get("families", {}).items() - if entry.get("library") is not None - ) + import torch + + return getattr(torch.ops, torch_namespace(family, target)) diff --git a/tests/unittest/_torch/modeling_v2/models/fixture/scale_add__sm_80__default/elementwise/test_modeling_v2_scale_add.py b/tests/unittest/_torch/modeling_v2/models/fixture/scale_add__sm_80__default/elementwise/test_modeling_v2_scale_add.py index 15b357cbc7e5..22ed2bea193f 100644 --- a/tests/unittest/_torch/modeling_v2/models/fixture/scale_add__sm_80__default/elementwise/test_modeling_v2_scale_add.py +++ b/tests/unittest/_torch/modeling_v2/models/fixture/scale_add__sm_80__default/elementwise/test_modeling_v2_scale_add.py @@ -4,8 +4,8 @@ Importing the entry imports its target package, which loads the fixture family library through ``_native.load_target_ops``: a pass here means the library was -built for this GPU, packaged, opened, matched the stock contract fingerprint and -registered its fake op -- before the first cell runs. +built for this GPU, packaged, opened, listed the target in its built_targets() +and registered its fake op -- before the first cell runs. """ import zlib diff --git a/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_claims.py b/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_claims.py index b57113f35de4..cfb69dd77a36 100644 --- a/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_claims.py +++ b/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_claims.py @@ -25,18 +25,13 @@ import pytest -from tensorrt_llm._torch._experimental.modeling_v2._native import ( - arch_covers, - camel, - identifier, - torch_namespace, -) +from tensorrt_llm._torch._experimental.modeling_v2._native import camel, identifier, torch_namespace _REPO = Path(__file__).resolve().parents[4] _NATIVE = _REPO / "cpp/tensorrt_llm/modeling_v2" _PY = _REPO / "tensorrt_llm/_torch/_experimental/modeling_v2" _MODELS = _PY / "models" -_CMAKE_MODULE = _REPO / "cpp/cmake/modules/modelingV2.cmake" +_CMAKE_MODULE = _REPO / "cpp/cmake/modules/modeling_v2.cmake" _TEST_DB = _REPO / "tests/integration/test_lists/test-db" pytestmark = [ @@ -51,6 +46,19 @@ _SM_GROUPS = {"10x": ((10, 0), (10, 3))} +def arch_covers(arch: str, capability: tuple[int, int]) -> bool: + """Whether code built for ``arch`` (``NN``, ``NNa`` or ``NNf``) runs on ``capability``. + + ``NNa`` runs on exactly NN; ``NN`` on NN and later minor versions of the same + major version; ``NNf`` on every member of family NN, the same way. + """ + suffix = arch[-1] if arch[-1] in "af" else "" + major, minor = divmod(int(arch[:-1] if suffix else arch), 10) + if suffix == "a": + return capability == (major, minor) + return capability[0] == major and capability[1] >= minor + + # --- Reading the CMake declarations ------------------------------------------ @@ -75,19 +83,19 @@ def _families() -> dict[str, Path]: """family -> native family directory, from every family CMakeLists.txt.""" families = {} for cmake in sorted(_NATIVE.glob("*/CMakeLists.txt")): - calls = _calls(cmake.read_text(), "trtllm_mv2_family") + calls = _calls(cmake.read_text(), "add_modeling_v2_family") assert len(calls) == 1 and len(calls[0]) == 1, ( - f"{cmake.relative_to(_REPO)} must call trtllm_mv2_family() exactly once" + f"{cmake.relative_to(_REPO)} must call add_modeling_v2_family() exactly once" ) families[calls[0][0]] = cmake.parent return families def _targets() -> list[tuple[str, str, list[str]]]: - """(family, target, tokens) of every trtllm_mv2_target call.""" + """(family, target, tokens) of every add_modeling_v2_target call.""" out = [] for family_dir in _families().values(): - for tokens in _calls((family_dir / "CMakeLists.txt").read_text(), "trtllm_mv2_target"): + for tokens in _calls((family_dir / "CMakeLists.txt").read_text(), "add_modeling_v2_target"): out.append((tokens[0], tokens[1], tokens[2:])) return out @@ -129,7 +137,7 @@ def test_every_native_target_is_a_python_target_that_loads_it(): if sub.name == "shared": continue assert any(camel(t) == sub.name for f, t in declared if f == family), ( - f"{sub.relative_to(_REPO)} is not the directory of any trtllm_mv2_target " + f"{sub.relative_to(_REPO)} is not the directory of any add_modeling_v2_target " f"of family {family!r}" ) @@ -142,7 +150,7 @@ def test_python_targets_that_load_native_code_have_it(): family, target = package.parent.parent.name, package.parent.name assert (family, target) in declared, ( f"{package.relative_to(_REPO)} loads native code, but no " - f"trtllm_mv2_target({family} {target} ...) declares it" + f"add_modeling_v2_target({family} {target} ...) declares it" ) @@ -386,18 +394,18 @@ def test_accuracy_entries_name_a_family(): @pytest.mark.skipif(shutil.which("cmake") is None, reason="needs cmake") def test_cmake_and_python_agree_on_names_and_architectures(tmp_path): """The loader and CBTS compute directory, namespace and architecture - coverage in Python; the build computes them in modelingV2.cmake.""" + coverage in Python; the build computes them in modeling_v2.cmake.""" names = ["fixture", "minimax_m3", "scale_add__sm_80__default", "r1_0528_nvfp4__sm_10x__default"] script = tmp_path / "probe.cmake" lines = [f'include("{_CMAKE_MODULE}")'] for name in names: lines += [ - f'_trtllm_mv2_camel(c "{name}")', - f'_trtllm_mv2_ident(i "{name}")', + f'_modeling_v2_camel(c "{name}")', + f'_modeling_v2_ident(i "{name}")', f'message(STATUS "name {name} ${{c}} ${{i}}")', ] lines += [ - "_trtllm_mv2_resolve_archs(r DECLARED 80 90a 100f 103a 120f REQUESTED 86 90 103 107)", + "_modeling_v2_resolve_archs(r DECLARED 80 90a 100f 103a 120f REQUESTED 86 90 103 107)", 'message(STATUS "archs ${r}")', ] script.write_text("\n".join(lines) + "\n") diff --git a/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_loader.py b/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_loader.py index 927f78289104..0c05961357f1 100644 --- a/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_loader.py +++ b/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_loader.py @@ -1,17 +1,17 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""What ``_native`` does with a manifest, and that the installed libraries load. +"""What ``_native`` decides, and that the installed family libraries load. -The first half drives the loader's decisions with synthetic manifests: which -outcomes are ``NoTarget`` (this build or this device cannot run the target, so -``auto`` keeps the built-in model) and which are ``NativeLoadError`` (the -installation is broken, which is never a fallback). Nothing is opened. +The first half drives the loader's decisions without opening anything: which +outcomes are ``NoTarget`` (this installation does not carry the target, so +``auto`` keeps the built-in model) and which are ``NativeLoadError`` (an +installed library is broken, which is never a fallback). The second half is the load smoke test of the installed wheel: every family -library the manifest lists opens in a fresh process, reports the stock contract -fingerprint, and defines no op its target's fake_ops.py does not fake. A family -library aborts the process from a static initializer if two libraries claim one -torch namespace, so each one is opened in a child process. +library in ``tensorrt_llm/libs`` opens in a fresh process, names the targets it +carries, and defines no op the target's fake_ops.py does not fake. A library +aborts its process from a static initializer if two libraries claim one torch +namespace, so each one is opened in a child process. """ from __future__ import annotations @@ -27,13 +27,12 @@ from tensorrt_llm._torch._experimental.modeling_v2 import _native from tensorrt_llm._torch._experimental.modeling_v2._native import ( NativeLoadError, - arch_covers, - available_targets, camel, identifier, + library_name, + load_family_library, load_target_ops, namespace_of, - read_manifest, split_target_package, torch_namespace, ) @@ -44,28 +43,18 @@ _FIXTURE = "tensorrt_llm._torch._experimental.modeling_v2.models.fixture.scale_add__sm_80__default" -def _manifest(status="built", archs=("100f-real",), library="libtrtllm_mv2_fixture.so", **extra): - return { - "schema": _native.MANIFEST_SCHEMA, - "enabled": True, - "requested_archs": ["100", "103"], - "families": { - "fixture": { - "library": library, - "namespace": "mv2_fixture", - "shared": None, - "targets": { - "scale_add__sm_80__default": { - "status": status, - "namespace": "mv2_fixture_scale_add_sm_80_default", - "declared_archs": ["100f"], - "archs": list(archs), - "reason": extra.get("reason", ""), - } - }, - } - }, - } +@pytest.fixture +def libs(monkeypatch, tmp_path): + """An empty libs directory and a cold library cache.""" + monkeypatch.setattr(_native, "_LIBS_DIR", tmp_path) + monkeypatch.setattr(_native, "_built_targets", {}) + return tmp_path + + +def _install(libs: Path, monkeypatch, opens) -> None: + """Put a fixture library in `libs`; `opens` stands in for dlopen.""" + (libs / library_name("fixture")).write_bytes(b"") + monkeypatch.setattr(_native, "_open_library", lambda path, family: opens()) # --- Naming ---------------------------------------------------------------------- @@ -85,7 +74,8 @@ def test_camel_is_the_native_directory(name, expected): assert camel(name) == expected -def test_namespaces_collapse_double_underscores(): +def test_names_of_libraries_and_namespaces(): + assert library_name("minimax_m3") == "libth_modeling_v2_minimax_m3.so" assert identifier("scale_add__sm_80__default") == "scale_add_sm_80_default" assert torch_namespace("fixture") == "mv2_fixture" assert namespace_of(_FIXTURE) == "mv2_fixture_scale_add_sm_80_default" @@ -94,92 +84,46 @@ def test_namespaces_collapse_double_underscores(): split_target_package("tensorrt_llm._torch.models") -@pytest.mark.parametrize( - "arch, capability, runs", - [ - ("100f-real", (10, 0), True), - ("100f-real", (10, 3), True), - ("100f-real", (10, 7), True), - ("100f-real", (12, 0), False), - ("103a-real", (10, 3), True), - ("103a-real", (10, 0), False), - ("80-real", (8, 6), True), - ("80-real", (9, 0), False), - ("90a-real", (9, 0), True), - ("120f", (12, 1), True), - ], -) -def test_arch_coverage(arch, capability, runs): - assert arch_covers(arch, capability) is runs - - # --- Decisions ----------------------------------------------------------------- -def test_a_missing_manifest_is_a_broken_installation(tmp_path): - with pytest.raises(NativeLoadError, match="no modeling_v2"): - read_manifest(tmp_path / "absent.json") - +def test_a_library_this_installation_lacks_is_no_target(libs): + with pytest.raises(NoTarget, match="has no libth_modeling_v2_fixture.so"): + load_target_ops(_FIXTURE) -def test_a_manifest_from_another_schema_is_refused(tmp_path): - path = tmp_path / "manifest.json" - path.write_text(json.dumps({**_manifest(), "schema": 999})) - with pytest.raises(NativeLoadError, match="schema"): - read_manifest(path) +def test_a_library_that_fails_to_open_is_a_broken_installation(libs, monkeypatch): + def fail(): + raise OSError("undefined symbol: _ZN12tensorrt_llm6common") -def test_a_build_without_modeling_v2_is_no_target(): - manifest = { - "schema": _native.MANIFEST_SCHEMA, - "enabled": False, - "reason": "off", - "families": {}, - } - with pytest.raises(NoTarget, match="no modeling_v2 libraries"): - load_target_ops(_FIXTURE, manifest) + _install(libs, monkeypatch, fail) + with pytest.raises(NativeLoadError, match="undefined symbol"): + load_target_ops(_FIXTURE) -@pytest.mark.parametrize("status", ["skipped", "excluded"]) -def test_a_target_this_build_skipped_is_no_target(status): - manifest = _manifest(status=status, archs=(), reason="needs sm_107") - with pytest.raises(NoTarget, match="needs sm_107"): - load_target_ops(_FIXTURE, manifest) +def test_a_target_the_library_does_not_carry_is_no_target(libs, monkeypatch): + _install(libs, monkeypatch, lambda: frozenset({"other__sm_90__default"})) + with pytest.raises(NoTarget, match="does not carry scale_add__sm_80__default"): + load_target_ops(_FIXTURE) -def test_a_target_the_manifest_does_not_know_is_a_broken_installation(): - manifest = _manifest() - manifest["families"]["fixture"]["targets"] = {} - with pytest.raises(NativeLoadError, match="different builds"): - load_target_ops(_FIXTURE, manifest) - with pytest.raises(NativeLoadError, match="different builds"): - load_target_ops(_FIXTURE, {**manifest, "families": {}}) +def test_a_library_opens_once(libs, monkeypatch): + opened = [] + def opens(): + opened.append(1) + return frozenset({"scale_add__sm_80__default"}) -def test_a_device_the_target_was_not_built_for_is_no_target(monkeypatch): - monkeypatch.setattr(_native, "_device_capability", lambda: (8, 0)) - with pytest.raises(NoTarget, match="does not run on this sm_80"): - load_target_ops(_FIXTURE, _manifest(archs=("100f-real",))) + _install(libs, monkeypatch, opens) + assert load_family_library("fixture") == {"scale_add__sm_80__default"} + assert load_family_library("fixture") == {"scale_add__sm_80__default"} + assert len(opened) == 1 -def test_a_listed_library_that_is_missing_is_a_broken_installation(monkeypatch, tmp_path): - monkeypatch.setattr(_native, "_device_capability", lambda: None) - monkeypatch.setattr(_native, "_LIBS_DIR", tmp_path) - with pytest.raises(NativeLoadError, match="missing"): - load_target_ops(_FIXTURE, _manifest()) - - -def test_available_targets_lists_every_target(): - rows = available_targets(_manifest(status="skipped", archs=(), reason="needs sm_107")) - assert rows == [ - { - "family": "fixture", - "target": "scale_add__sm_80__default", - "status": "skipped", - "archs": [], - "reason": "needs sm_107", - "library": "libtrtllm_mv2_fixture.so", - } - ] +def test_windows_has_no_native_targets(libs, monkeypatch): + monkeypatch.setattr(sys, "platform", "win32") + with pytest.raises(NoTarget, match="Windows"): + load_family_library("fixture") # --- The installed wheel --------------------------------------------------------- @@ -191,43 +135,36 @@ def test_available_targets_lists_every_target(): import tensorrt_llm # noqa: F401 -- loads libth_common.so from tensorrt_llm._torch._experimental.modeling_v2 import _native - family = sys.argv[1] - _native.load_family_library(family) - schemas = [s.name for s in torch._C._jit_get_all_schemas()] - print(json.dumps(sorted(n for n in schemas if n.startswith("mv2_")))) + built = sorted(_native.load_family_library(sys.argv[1])) + schemas = sorted(s.name for s in torch._C._jit_get_all_schemas() if s.name.startswith("mv2_")) + print(json.dumps({"built": built, "ops": schemas})) """ ) -def _installed_manifest(): - path = _native._LIBS_DIR / _native.MANIFEST_NAME - assert path.is_file(), ( - f"{path} is missing: scripts/build_wheel.py installs it with the family libraries" - ) - return read_manifest(path) - - -def test_every_built_family_library_loads(): - manifest = _installed_manifest() - families = list(_native.built_families(manifest)) - if not families: - pytest.skip(f"this build has no modeling_v2 family library: {manifest.get('reason')}") +def test_every_installed_family_library_loads(): + libraries = sorted(_native._LIBS_DIR.glob("libth_modeling_v2_*.so")) + if not libraries: + pytest.skip("this installation has no modeling_v2 family library") models = Path(_native.__file__).resolve().parent / "models" - for family in families: + for library in libraries: + family = library.name[len("libth_modeling_v2_") : -len(".so")] result = subprocess.run( [sys.executable, "-c", _LOAD_ONE_FAMILY, family], capture_output=True, text=True, ) assert result.returncode == 0, ( - f"libtrtllm_mv2_{family}.so failed to load:\n{result.stdout}\n{result.stderr}" + f"{library.name} failed to load:\n{result.stdout}\n{result.stderr}" ) - ops = json.loads(result.stdout.strip().splitlines()[-1]) - for target, unit in manifest["families"][family]["targets"].items(): - if unit["status"] != "built": - continue - prefix = f"{unit['namespace']}::" - target_ops = [op[len(prefix) :] for op in ops if op.startswith(prefix)] + report = json.loads(result.stdout.strip().splitlines()[-1]) + assert report["built"], f"{library.name} carries no target" + for target in report["built"]: + assert (models / family / target / "__init__.py").is_file(), ( + f"{library.name} carries {target}, which has no Python target package" + ) + prefix = f"{torch_namespace(family, target)}::" + target_ops = [op[len(prefix) :] for op in report["ops"] if op.startswith(prefix)] fake_ops = models / family / target / "fake_ops.py" text = fake_ops.read_text() if fake_ops.is_file() else "" unfaked = [op for op in target_ops if f'::{op}"' not in text] diff --git a/tests/unittest/_torch/modeling_v2/test_modeling_v2_rank_agreement.py b/tests/unittest/_torch/modeling_v2/test_modeling_v2_rank_agreement.py index c5cda015a399..27db451ec8fe 100644 --- a/tests/unittest/_torch/modeling_v2/test_modeling_v2_rank_agreement.py +++ b/tests/unittest/_torch/modeling_v2/test_modeling_v2_rank_agreement.py @@ -98,7 +98,7 @@ def test_disagreeing_ranks_all_fail(monkeypatch, dist): def test_another_ranks_error_fails_this_rank(monkeypatch, dist): - dist.others = [(None, None, "NativeLoadError: libtrtllm_mv2_x.so cannot be loaded")] + dist.others = [(None, None, "NativeLoadError: libth_modeling_v2_x.so cannot be loaded")] _route_to(monkeypatch, target=Target) with pytest.raises(RuntimeError, match="rank 1: NativeLoadError"): modeling_v2_resolve_on_all_ranks(_config(), _args("auto")) diff --git a/tests/unittest/scripts/cbts/test_modeling_v2_rule.py b/tests/unittest/scripts/cbts/test_modeling_v2_rule.py index 0c031fb83536..32f5030eba43 100644 --- a/tests/unittest/scripts/cbts/test_modeling_v2_rule.py +++ b/tests/unittest/scripts/cbts/test_modeling_v2_rule.py @@ -91,9 +91,9 @@ def test_claims_python_and_native_sources_but_not_docs_or_stock(): assert _is_mv2_claim(f"{_PY}/models/fixture/scale_add__sm_80__default/__init__.py") assert _is_mv2_claim(f"{_CPP}/fixture/scaleAddSm80Default/kernels/scaleAdd.cu") assert _is_mv2_claim(f"{_CPP}/contractSurface.txt") - assert _is_mv2_claim("cpp/cmake/modules/modelingV2.cmake") + assert _is_mv2_claim("cpp/cmake/modules/modeling_v2.cmake") assert not _is_mv2_claim(f"{_CPP}/README.md") - assert not _is_mv2_claim("cpp/tensorrt_llm/thop/modelingV2ContractOp.cpp") + assert not _is_mv2_claim("cpp/tensorrt_llm/thop/attentionOp.cpp") assert not _is_mv2_claim("tensorrt_llm/_torch/pyexecutor/model_loader.py") @@ -143,7 +143,7 @@ def test_two_families_union(rule): f"{_PY}/_native.py", f"{_CPP}/common/torchLibrary.h", f"{_CPP}/contractSurface.txt", - "cpp/cmake/modules/modelingV2.cmake", + "cpp/cmake/modules/modeling_v2.cmake", ], ) def test_shared_modeling_v2_code_selects_everything_of_modeling_v2(rule, changed): From 4195583c59d7010556dcbbc027251203b4694293 Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Sun, 25 Oct 2026 22:43:40 -0700 Subject: [PATCH 03/20] [None][fix] modeling_v2: remove a family library an earlier build installed As build_wheel.py does for deep_ep's package, remove an installed libth_modeling_v2_.so that the current build did not produce, so a library left by an earlier configuration is neither loaded nor shipped. Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- scripts/build_wheel.py | 9 ++- .../scripts/test_build_wheel_modeling_v2.py | 71 +++++++++++++++++++ 2 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 tests/unittest/scripts/test_build_wheel_modeling_v2.py diff --git a/scripts/build_wheel.py b/scripts/build_wheel.py index a450d4c36133..b858578c77ac 100755 --- a/scripts/build_wheel.py +++ b/scripts/build_wheel.py @@ -798,8 +798,11 @@ def install_modeling_v2_libs(mv2_build_dir: Path, lib_dir: Path, Each family's build directory holds cuda_architectures.txt, written at configure time: the architectures its library was built for, empty when - there is none (cpp/cmake/modules/modeling_v2.cmake). + there is none (cpp/cmake/modules/modeling_v2.cmake). As deep_ep's package + is, a family library installed by an earlier build that this build did not + produce is removed, so it is neither loaded nor shipped. """ + installed = set() for arch_file in sorted(mv2_build_dir.glob("*/cuda_architectures.txt")): if not arch_file.read_text().strip().strip(";"): continue @@ -808,6 +811,10 @@ def install_modeling_v2_libs(mv2_build_dir: Path, lib_dir: Path, "Exactly one modeling_v2 library should be present in " f"{arch_file.parent}: {libraries}") install_file(libraries[0], lib_dir / libraries[0].name) + installed.add(libraries[0].name) + for stale in lib_dir.glob("libth_modeling_v2_*.so"): + if stale.name not in installed: + stale.unlink() def stage_nccl_extensions_package(wheel: Path, staging_dir: Path) -> None: diff --git a/tests/unittest/scripts/test_build_wheel_modeling_v2.py b/tests/unittest/scripts/test_build_wheel_modeling_v2.py new file mode 100644 index 000000000000..8a527f0b6410 --- /dev/null +++ b/tests/unittest/scripts/test_build_wheel_modeling_v2.py @@ -0,0 +1,71 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""CPU-only checks of how build_wheel.py installs the modeling_v2 family libraries.""" + +import importlib.util +import shutil +from pathlib import Path +from types import ModuleType + +import pytest + +pytestmark = pytest.mark.cpu_only + + +@pytest.fixture(scope="module") +def build_wheel() -> ModuleType: + script = Path(__file__).resolve().parents[3] / "scripts" / "build_wheel.py" + spec = importlib.util.spec_from_file_location("build_wheel", script) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _family(build_dir: Path, directory: str, archs: str, libraries=()) -> None: + family_dir = build_dir / directory + family_dir.mkdir(parents=True) + (family_dir / "cuda_architectures.txt").write_text(archs) + for library in libraries: + (family_dir / library).write_bytes(b"\x7fELF") + + +@pytest.fixture +def dirs(tmp_path: Path): + build_dir = tmp_path / "build" / "tensorrt_llm" / "modeling_v2" + lib_dir = tmp_path / "tensorrt_llm" / "libs" + build_dir.mkdir(parents=True) + lib_dir.mkdir(parents=True) + return build_dir, lib_dir + + +def test_installs_only_families_with_architectures(build_wheel, dirs) -> None: + build_dir, lib_dir = dirs + _family(build_dir, "fixture", "103a-real", ["libth_modeling_v2_fixture.so"]) + _family(build_dir, "glmMoeDsa", "") + (lib_dir / "libth_common.so").write_bytes(b"stock") + + build_wheel.install_modeling_v2_libs(build_dir, lib_dir, shutil.copy) + + assert sorted(p.name for p in lib_dir.iterdir()) == [ + "libth_common.so", + "libth_modeling_v2_fixture.so", + ] + + +def test_removes_a_library_this_build_did_not_produce(build_wheel, dirs) -> None: + build_dir, lib_dir = dirs + _family(build_dir, "fixture", "", ["libth_modeling_v2_fixture.so"]) + (lib_dir / "libth_modeling_v2_fixture.so").write_bytes(b"earlier build") + (lib_dir / "libth_modeling_v2_removed_family.so").symlink_to(lib_dir / "gone.so") + + build_wheel.install_modeling_v2_libs(build_dir, lib_dir, shutil.copy) + + assert list(lib_dir.iterdir()) == [] + + +def test_a_family_with_architectures_must_have_one_library(build_wheel, dirs) -> None: + build_dir, lib_dir = dirs + _family(build_dir, "fixture", "103a-real") + + with pytest.raises(AssertionError, match="Exactly one modeling_v2 library"): + build_wheel.install_modeling_v2_libs(build_dir, lib_dir, shutil.copy) From 97636135ac21a41d16c20d1ad97235979644db83 Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Sun, 25 Oct 2026 23:42:27 -0700 Subject: [PATCH 04/20] [None][chore] modeling_v2: keep the native-code fixture as temporary infrastructure, built for 100f The fixture family is an infrastructure test, not a model: it is what drives the family-library build, packaging and loading path in CI while no family has native code and the other modeling_v2 B300 unit-test entries are waived. - Mark it temporary in its CMakeLists.txt, package docstring, READMEs and test list, with the condition for removing it. - Build it for 100f only, the B300 its GPU test runs on, instead of 80/90/100f/120f, so the library it adds to every wheel stays small. The target is renamed scale_add__sm_80__default -> scale_add__sm_10x__default to match the architectures it is built for. Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- cpp/tensorrt_llm/modeling_v2/README.md | 2 ++ .../modeling_v2/fixture/CMakeLists.txt | 15 ++++++++------- .../kernels/scaleAdd.cu | 2 +- .../kernels/scaleAdd.h | 0 .../ops/scaleAddOp.cpp | 2 +- .../_torch/_experimental/modeling_v2/README.md | 3 ++- .../_torch/_experimental/modeling_v2/_native.py | 4 ++-- .../modeling_v2/models/fixture/__init__.py | 15 ++++++++------- .../__init__.py | 6 +++--- .../fake_ops.py | 0 .../ops/__init__.py | 0 .../ops/elementwise/__init__.py | 0 .../ops/elementwise/scale_add.py | 2 +- tests/integration/test_lists/test-db/l0_b300.yml | 1 + .../elementwise/test_modeling_v2_scale_add.py | 4 ++-- .../test_modeling_v2_native_claims.py | 6 +++--- .../test_modeling_v2_native_loader.py | 16 ++++++++-------- .../scripts/cbts/test_modeling_v2_rule.py | 10 +++++----- 18 files changed, 47 insertions(+), 41 deletions(-) rename cpp/tensorrt_llm/modeling_v2/fixture/{scaleAddSm80Default => scaleAddSm10xDefault}/kernels/scaleAdd.cu (97%) rename cpp/tensorrt_llm/modeling_v2/fixture/{scaleAddSm80Default => scaleAddSm10xDefault}/kernels/scaleAdd.h (100%) rename cpp/tensorrt_llm/modeling_v2/fixture/{scaleAddSm80Default => scaleAddSm10xDefault}/ops/scaleAddOp.cpp (96%) rename tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/{scale_add__sm_80__default => scale_add__sm_10x__default}/__init__.py (71%) rename tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/{scale_add__sm_80__default => scale_add__sm_10x__default}/fake_ops.py (100%) rename tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/{scale_add__sm_80__default => scale_add__sm_10x__default}/ops/__init__.py (100%) rename tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/{scale_add__sm_80__default => scale_add__sm_10x__default}/ops/elementwise/__init__.py (100%) rename tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/{scale_add__sm_80__default => scale_add__sm_10x__default}/ops/elementwise/scale_add.py (95%) rename tests/unittest/_torch/modeling_v2/models/fixture/{scale_add__sm_80__default => scale_add__sm_10x__default}/elementwise/test_modeling_v2_scale_add.py (95%) diff --git a/cpp/tensorrt_llm/modeling_v2/README.md b/cpp/tensorrt_llm/modeling_v2/README.md index a45b28e2dfbc..c42e00b04a73 100644 --- a/cpp/tensorrt_llm/modeling_v2/README.md +++ b/cpp/tensorrt_llm/modeling_v2/README.md @@ -17,6 +17,8 @@ torch-op library, `libth_modeling_v2_.so`, installed next to This document is the guide for adding native code to a target. The family `fixture/` is the smallest complete example: one target with one op. Copy it. +It is a temporary infrastructure fixture, not a model, and is removed once a +family with native code has GPU unit tests that run in CI. ## Layout diff --git a/cpp/tensorrt_llm/modeling_v2/fixture/CMakeLists.txt b/cpp/tensorrt_llm/modeling_v2/fixture/CMakeLists.txt index 47b8fee5bfbc..e9ee4ebe453a 100644 --- a/cpp/tensorrt_llm/modeling_v2/fixture/CMakeLists.txt +++ b/cpp/tensorrt_llm/modeling_v2/fixture/CMakeLists.txt @@ -15,14 +15,15 @@ # the License. # -# The infrastructure fixture: one target with one op, so that CI drives the -# whole native path -- build, packaging, load, built_targets(), fake op, -# GPU op -- on every GPU it has. It is also the smallest complete family to copy -# (see ../README.md). Its architectures cover every GPU CI runs on; a real -# target declares the architectures it was written and gated for. +# Temporary infrastructure fixture, not a model: one target with one op, so +# that CI drives the whole native path -- build, packaging, load, +# built_targets(), fake op, GPU op. Remove this family (with its Python package +# and tests) once a family with native code has GPU unit tests that run in CI. +# It is built for 100f only, the B300 its GPU test runs on, to keep the +# library it adds to every wheel small. add_modeling_v2_family(fixture) add_modeling_v2_target( - fixture scale_add__sm_80__default - ARCHS 80 90 100f 120f + fixture scale_add__sm_10x__default + ARCHS 100f SOURCES kernels/scaleAdd.cu ops/scaleAddOp.cpp) diff --git a/cpp/tensorrt_llm/modeling_v2/fixture/scaleAddSm80Default/kernels/scaleAdd.cu b/cpp/tensorrt_llm/modeling_v2/fixture/scaleAddSm10xDefault/kernels/scaleAdd.cu similarity index 97% rename from cpp/tensorrt_llm/modeling_v2/fixture/scaleAddSm80Default/kernels/scaleAdd.cu rename to cpp/tensorrt_llm/modeling_v2/fixture/scaleAddSm10xDefault/kernels/scaleAdd.cu index 6c5404426ad8..360cd682753a 100644 --- a/cpp/tensorrt_llm/modeling_v2/fixture/scaleAddSm80Default/kernels/scaleAdd.cu +++ b/cpp/tensorrt_llm/modeling_v2/fixture/scaleAddSm10xDefault/kernels/scaleAdd.cu @@ -15,7 +15,7 @@ * limitations under the License. */ -#include "tensorrt_llm/modeling_v2/fixture/scaleAddSm80Default/kernels/scaleAdd.h" +#include "tensorrt_llm/modeling_v2/fixture/scaleAddSm10xDefault/kernels/scaleAdd.h" #include #include diff --git a/cpp/tensorrt_llm/modeling_v2/fixture/scaleAddSm80Default/kernels/scaleAdd.h b/cpp/tensorrt_llm/modeling_v2/fixture/scaleAddSm10xDefault/kernels/scaleAdd.h similarity index 100% rename from cpp/tensorrt_llm/modeling_v2/fixture/scaleAddSm80Default/kernels/scaleAdd.h rename to cpp/tensorrt_llm/modeling_v2/fixture/scaleAddSm10xDefault/kernels/scaleAdd.h diff --git a/cpp/tensorrt_llm/modeling_v2/fixture/scaleAddSm80Default/ops/scaleAddOp.cpp b/cpp/tensorrt_llm/modeling_v2/fixture/scaleAddSm10xDefault/ops/scaleAddOp.cpp similarity index 96% rename from cpp/tensorrt_llm/modeling_v2/fixture/scaleAddSm80Default/ops/scaleAddOp.cpp rename to cpp/tensorrt_llm/modeling_v2/fixture/scaleAddSm10xDefault/ops/scaleAddOp.cpp index 1abf21f025da..e4a6bc553e99 100644 --- a/cpp/tensorrt_llm/modeling_v2/fixture/scaleAddSm80Default/ops/scaleAddOp.cpp +++ b/cpp/tensorrt_llm/modeling_v2/fixture/scaleAddSm10xDefault/ops/scaleAddOp.cpp @@ -18,7 +18,7 @@ #include "tensorrt_llm/common/assert.h" #include "tensorrt_llm/modeling_v2/common/namespace.h" #include "tensorrt_llm/modeling_v2/common/torchLibrary.h" -#include "tensorrt_llm/modeling_v2/fixture/scaleAddSm80Default/kernels/scaleAdd.h" +#include "tensorrt_llm/modeling_v2/fixture/scaleAddSm10xDefault/kernels/scaleAdd.h" #include #include diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/README.md b/tensorrt_llm/_torch/_experimental/modeling_v2/README.md index 8a5b2301b2c1..eae8cbb8fcc6 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/README.md +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/README.md @@ -134,7 +134,8 @@ native modules, a library this installation does not carry is a `NoTarget`, so `auto` keeps the built-in model; one that is installed but broken is a `NativeLoadError`, never a fallback. `cpp/tensorrt_llm/modeling_v2/README.md` is the guide, and -`models/fixture/` is the smallest complete example. +`models/fixture/`, a temporary infrastructure fixture rather than a model, is +the smallest complete example. An entry, shared or owned, is one `OpWrapper` subclass: `__call__` is the op call a target makes, `reference` is what it is supposed to compute in plain diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/_native.py b/tensorrt_llm/_torch/_experimental/modeling_v2/_native.py index bfdbbd267793..d16ca4b62222 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/_native.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/_native.py @@ -61,8 +61,8 @@ class NativeLoadError(RuntimeError): def camel(name: str) -> str: - """The C++ directory of a family or target: ``scale_add__sm_80__default`` -> - ``scaleAddSm80Default``.""" + """The C++ directory of a family or target: ``scale_add__sm_10x__default`` -> + ``scaleAddSm10xDefault``.""" parts = [p for p in name.split("_") if p] return parts[0] + "".join(p[:1].upper() + p[1:] for p in parts[1:]) diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/__init__.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/__init__.py index 22de297f5244..79047741cdd5 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/__init__.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/__init__.py @@ -1,13 +1,14 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""The native-code fixture: a family with no model behind it. +"""Temporary infrastructure fixture for native code: a family with no model behind it. -Its one target, ``scale_add__sm_80__default``, carries one native op so that CI +Its one target, ``scale_add__sm_10x__default``, carries one native op so that CI drives the whole native path -- the family library's build and packaging, the -loader's checks, the fake op, the GPU op -- on every GPU it has. No routing -module names it; tests import the target directly. +loader's checks, the fake op, the GPU op -- on B300. No routing module names it; +tests import the target directly. -It is also the smallest complete example of a target with native code; its C++ -half is ``cpp/tensorrt_llm/modeling_v2/fixture/`` and -``cpp/tensorrt_llm/modeling_v2/README.md`` walks through both. +Remove the family, its C++ half ``cpp/tensorrt_llm/modeling_v2/fixture/`` and +its tests once a family with native code has GPU unit tests that run in CI. +Until then it is also the smallest complete example of a target with native +code, which ``cpp/tensorrt_llm/modeling_v2/README.md`` walks through. """ diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_80__default/__init__.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_10x__default/__init__.py similarity index 71% rename from tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_80__default/__init__.py rename to tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_10x__default/__init__.py index 4a04b0fb8089..1f6251103cc9 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_80__default/__init__.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_10x__default/__init__.py @@ -1,10 +1,10 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Fixture / sm_80 / default: the native-code fixture's only target. +"""Fixture / sm_10x / default: the native-code fixture's only target. Loading the package loads its native code: ``native_ops`` is -``torch.ops.mv2_fixture_scale_add_sm_80_default``, built from -``cpp/tensorrt_llm/modeling_v2/fixture/scaleAddSm80Default/``. A target with +``torch.ops.mv2_fixture_scale_add_sm_10x_default``, built from +``cpp/tensorrt_llm/modeling_v2/fixture/scaleAddSm10xDefault/``. A target with native code starts this way, so that ``route`` importing the target is what loads -- and checks -- its library. (Not ``ops``: that name is the target's ``ops/`` package, which would replace it on import.) diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_80__default/fake_ops.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_10x__default/fake_ops.py similarity index 100% rename from tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_80__default/fake_ops.py rename to tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_10x__default/fake_ops.py diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_80__default/ops/__init__.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_10x__default/ops/__init__.py similarity index 100% rename from tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_80__default/ops/__init__.py rename to tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_10x__default/ops/__init__.py diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_80__default/ops/elementwise/__init__.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_10x__default/ops/elementwise/__init__.py similarity index 100% rename from tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_80__default/ops/elementwise/__init__.py rename to tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_10x__default/ops/elementwise/__init__.py diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_80__default/ops/elementwise/scale_add.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_10x__default/ops/elementwise/scale_add.py similarity index 95% rename from tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_80__default/ops/elementwise/scale_add.py rename to tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_10x__default/ops/elementwise/scale_add.py index 456801e1d9b0..0546d58f1c57 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_80__default/ops/elementwise/scale_add.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_10x__default/ops/elementwise/scale_add.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 """`x * alpha + beta`: the native-code fixture's op, an entry like any other. -The kernel is ``cpp/tensorrt_llm/modeling_v2/fixture/scaleAddSm80Default/``; the +The kernel is ``cpp/tensorrt_llm/modeling_v2/fixture/scaleAddSm10xDefault/``; the op is reached through the target's ``native_ops`` namespace, which loading the target package filled. Nothing here is specific to native code beyond that one name. """ diff --git a/tests/integration/test_lists/test-db/l0_b300.yml b/tests/integration/test_lists/test-db/l0_b300.yml index 621c6cd9d79d..ae2096685c87 100644 --- a/tests/integration/test_lists/test-db/l0_b300.yml +++ b/tests/integration/test_lists/test-db/l0_b300.yml @@ -142,6 +142,7 @@ l0_b300: # cpu_only files run on the CPU stage instead), so 30 per entry is at least 2.2x headroom. - unittest/_torch/modeling_v2 --ignore=unittest/_torch/modeling_v2/comm --ignore=unittest/_torch/modeling_v2/models TIMEOUT (30) - unittest/_torch/modeling_v2/models/deepseek_v3 TIMEOUT (30) + # Temporary native-code infrastructure fixture (no model); built for 100f only. - unittest/_torch/modeling_v2/models/fixture TIMEOUT (30) - unittest/_torch/modeling_v2/models/glm_moe_dsa TIMEOUT (30) - unittest/_torch/modeling_v2/models/qwen4_exp TIMEOUT (30) diff --git a/tests/unittest/_torch/modeling_v2/models/fixture/scale_add__sm_80__default/elementwise/test_modeling_v2_scale_add.py b/tests/unittest/_torch/modeling_v2/models/fixture/scale_add__sm_10x__default/elementwise/test_modeling_v2_scale_add.py similarity index 95% rename from tests/unittest/_torch/modeling_v2/models/fixture/scale_add__sm_80__default/elementwise/test_modeling_v2_scale_add.py rename to tests/unittest/_torch/modeling_v2/models/fixture/scale_add__sm_10x__default/elementwise/test_modeling_v2_scale_add.py index 22ed2bea193f..c6168e989aa5 100644 --- a/tests/unittest/_torch/modeling_v2/models/fixture/scale_add__sm_80__default/elementwise/test_modeling_v2_scale_add.py +++ b/tests/unittest/_torch/modeling_v2/models/fixture/scale_add__sm_10x__default/elementwise/test_modeling_v2_scale_add.py @@ -13,10 +13,10 @@ import pytest import torch -from tensorrt_llm._torch._experimental.modeling_v2.models.fixture.scale_add__sm_80__default import ( # noqa: E501 +from tensorrt_llm._torch._experimental.modeling_v2.models.fixture.scale_add__sm_10x__default import ( # noqa: E501 native_ops, ) -from tensorrt_llm._torch._experimental.modeling_v2.models.fixture.scale_add__sm_80__default.ops.elementwise.scale_add import ( # noqa: E501 +from tensorrt_llm._torch._experimental.modeling_v2.models.fixture.scale_add__sm_10x__default.ops.elementwise.scale_add import ( # noqa: E501 scale_add as op, ) diff --git a/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_claims.py b/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_claims.py index cfb69dd77a36..7c9986e937ff 100644 --- a/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_claims.py +++ b/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_claims.py @@ -395,7 +395,7 @@ def test_accuracy_entries_name_a_family(): def test_cmake_and_python_agree_on_names_and_architectures(tmp_path): """The loader and CBTS compute directory, namespace and architecture coverage in Python; the build computes them in modeling_v2.cmake.""" - names = ["fixture", "minimax_m3", "scale_add__sm_80__default", "r1_0528_nvfp4__sm_10x__default"] + names = ["fixture", "minimax_m3", "scale_add__sm_10x__default", "r1_0528_nvfp4__sm_10x__default"] script = tmp_path / "probe.cmake" lines = [f'include("{_CMAKE_MODULE}")'] for name in names: @@ -416,6 +416,6 @@ def test_cmake_and_python_agree_on_names_and_architectures(tmp_path): assert f"name {name} {camel(name)} {identifier(name)}" in output, output assert "archs 90a-real;100f-real;103a-real" in output, output # The family namespace the build registers is the loader's. - assert torch_namespace("fixture", "scale_add__sm_80__default") == ( - "mv2_fixture_scale_add_sm_80_default" + assert torch_namespace("fixture", "scale_add__sm_10x__default") == ( + "mv2_fixture_scale_add_sm_10x_default" ) diff --git a/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_loader.py b/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_loader.py index 0c05961357f1..2c2185040370 100644 --- a/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_loader.py +++ b/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_loader.py @@ -40,7 +40,7 @@ pytestmark = pytest.mark.cpu_only -_FIXTURE = "tensorrt_llm._torch._experimental.modeling_v2.models.fixture.scale_add__sm_80__default" +_FIXTURE = "tensorrt_llm._torch._experimental.modeling_v2.models.fixture.scale_add__sm_10x__default" @pytest.fixture @@ -76,10 +76,10 @@ def test_camel_is_the_native_directory(name, expected): def test_names_of_libraries_and_namespaces(): assert library_name("minimax_m3") == "libth_modeling_v2_minimax_m3.so" - assert identifier("scale_add__sm_80__default") == "scale_add_sm_80_default" + assert identifier("scale_add__sm_10x__default") == "scale_add_sm_10x_default" assert torch_namespace("fixture") == "mv2_fixture" - assert namespace_of(_FIXTURE) == "mv2_fixture_scale_add_sm_80_default" - assert split_target_package(_FIXTURE) == ("fixture", "scale_add__sm_80__default") + assert namespace_of(_FIXTURE) == "mv2_fixture_scale_add_sm_10x_default" + assert split_target_package(_FIXTURE) == ("fixture", "scale_add__sm_10x__default") with pytest.raises(ValueError): split_target_package("tensorrt_llm._torch.models") @@ -103,7 +103,7 @@ def fail(): def test_a_target_the_library_does_not_carry_is_no_target(libs, monkeypatch): _install(libs, monkeypatch, lambda: frozenset({"other__sm_90__default"})) - with pytest.raises(NoTarget, match="does not carry scale_add__sm_80__default"): + with pytest.raises(NoTarget, match="does not carry scale_add__sm_10x__default"): load_target_ops(_FIXTURE) @@ -112,11 +112,11 @@ def test_a_library_opens_once(libs, monkeypatch): def opens(): opened.append(1) - return frozenset({"scale_add__sm_80__default"}) + return frozenset({"scale_add__sm_10x__default"}) _install(libs, monkeypatch, opens) - assert load_family_library("fixture") == {"scale_add__sm_80__default"} - assert load_family_library("fixture") == {"scale_add__sm_80__default"} + assert load_family_library("fixture") == {"scale_add__sm_10x__default"} + assert load_family_library("fixture") == {"scale_add__sm_10x__default"} assert len(opened) == 1 diff --git a/tests/unittest/scripts/cbts/test_modeling_v2_rule.py b/tests/unittest/scripts/cbts/test_modeling_v2_rule.py index 32f5030eba43..2be9e76788c4 100644 --- a/tests/unittest/scripts/cbts/test_modeling_v2_rule.py +++ b/tests/unittest/scripts/cbts/test_modeling_v2_rule.py @@ -88,8 +88,8 @@ def _selected(result) -> set[str]: def test_claims_python_and_native_sources_but_not_docs_or_stock(): - assert _is_mv2_claim(f"{_PY}/models/fixture/scale_add__sm_80__default/__init__.py") - assert _is_mv2_claim(f"{_CPP}/fixture/scaleAddSm80Default/kernels/scaleAdd.cu") + assert _is_mv2_claim(f"{_PY}/models/fixture/scale_add__sm_10x__default/__init__.py") + assert _is_mv2_claim(f"{_CPP}/fixture/scaleAddSm10xDefault/kernels/scaleAdd.cu") assert _is_mv2_claim(f"{_CPP}/contractSurface.txt") assert _is_mv2_claim("cpp/cmake/modules/modeling_v2.cmake") assert not _is_mv2_claim(f"{_CPP}/README.md") @@ -115,8 +115,8 @@ def test_snake_inverts_camel_without_a_repo_root(): @pytest.mark.parametrize( "changed", [ - f"{_PY}/models/fixture/scale_add__sm_80__default/ops/elementwise/scale_add.py", - f"{_CPP}/fixture/scaleAddSm80Default/kernels/scaleAdd.cu", + f"{_PY}/models/fixture/scale_add__sm_10x__default/ops/elementwise/scale_add.py", + f"{_CPP}/fixture/scaleAddSm10xDefault/kernels/scaleAdd.cu", f"{_CPP}/fixture/CMakeLists.txt", ], ) @@ -130,7 +130,7 @@ def test_a_family_change_selects_that_family_and_the_checks(rule, changed): def test_two_families_union(rule): result = _apply( rule, - f"{_PY}/models/fixture/scale_add__sm_80__default/__init__.py", + f"{_PY}/models/fixture/scale_add__sm_10x__default/__init__.py", f"{_PY}/models/gpt_oss/routing.py", ) assert _selected(result) == {_FIXTURE_UNIT, _GPT_OSS_UNIT, _GPT_OSS_ACCURACY, _CLAIMS} From e33fc687929e05e060c0e7537d99c218d1367c4b Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Mon, 26 Oct 2026 00:54:26 -0700 Subject: [PATCH 05/20] [None][chore] modeling_v2: check side effects of routes registered with @routes models/__init__.py now imports every family's routing module, which registers its route with @routes. The native-code check that family packages have no side effects now allows exactly that: models/__init__.py may import the families' routing modules, a family package holds a docstring only, and a routing module imports a target only inside route, so a target's native code still loads only when route returns the target. Also point the CBTS rule README at modeling_v2_resolve_on_all_ranks, and apply the formatters to the fixture's CMakeLists.txt and the claims test. Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- .../modeling_v2/fixture/CMakeLists.txt | 12 ++--- jenkins/scripts/cbts/cbts/rules/README.md | 6 +-- .../test_modeling_v2_native_claims.py | 49 +++++++++++++++---- 3 files changed, 49 insertions(+), 18 deletions(-) diff --git a/cpp/tensorrt_llm/modeling_v2/fixture/CMakeLists.txt b/cpp/tensorrt_llm/modeling_v2/fixture/CMakeLists.txt index e9ee4ebe453a..07cb27deb2c7 100644 --- a/cpp/tensorrt_llm/modeling_v2/fixture/CMakeLists.txt +++ b/cpp/tensorrt_llm/modeling_v2/fixture/CMakeLists.txt @@ -15,12 +15,12 @@ # the License. # -# Temporary infrastructure fixture, not a model: one target with one op, so -# that CI drives the whole native path -- build, packaging, load, -# built_targets(), fake op, GPU op. Remove this family (with its Python package -# and tests) once a family with native code has GPU unit tests that run in CI. -# It is built for 100f only, the B300 its GPU test runs on, to keep the -# library it adds to every wheel small. +# Temporary infrastructure fixture, not a model: one target with one op, so that +# CI drives the whole native path -- build, packaging, load, built_targets(), +# fake op, GPU op. Remove this family (with its Python package and tests) once a +# family with native code has GPU unit tests that run in CI. It is built for +# 100f only, the B300 its GPU test runs on, to keep the library it adds to every +# wheel small. add_modeling_v2_family(fixture) add_modeling_v2_target( diff --git a/jenkins/scripts/cbts/cbts/rules/README.md b/jenkins/scripts/cbts/cbts/rules/README.md index 6a8ac735b8d9..270aa6977537 100644 --- a/jenkins/scripts/cbts/cbts/rules/README.md +++ b/jenkins/scripts/cbts/cbts/rules/README.md @@ -345,9 +345,9 @@ carve-out is needed. Outward fallback: not needed, and by design rather than by accident. Nothing beyond the resolver module is imported unless `LLM(modeling_v2=...)` -is on — the model loader calls `modeling_v2_resolve`, which returns -immediately when the switch is off, and the routing modules are imported -lazily behind it. The callers outside the subtree, +is on — the model loader calls `modeling_v2_resolve_on_all_ranks`, which +returns immediately when the switch is off, and the routing modules are +imported lazily behind it. The callers outside the subtree, `tensorrt_llm/_torch/pyexecutor/model_loader.py` (decides) and `tensorrt_llm/_torch/models/modeling_auto.py` (reads the decision off `ModelConfig`), are deliberately left unclaimed: a change to the shared diff --git a/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_claims.py b/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_claims.py index 7c9986e937ff..b81c82f2db77 100644 --- a/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_claims.py +++ b/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_claims.py @@ -309,17 +309,43 @@ def test_no_op_probing(): assert not offenders, f"op probing in {offenders}" +def _is_docstring_or_future(node: ast.stmt) -> bool: + docstring = isinstance(node, ast.Expr) and isinstance(node.value, ast.Constant) + future = isinstance(node, ast.ImportFrom) and node.module == "__future__" + return docstring or future + + def test_family_packages_have_no_side_effects(): - """Native code loads per target; a family package that imported anything - would load it for every target of the family, or for none of them.""" - for init in [_MODELS / "__init__.py", *sorted(_MODELS.glob("*/__init__.py"))]: - body = ast.parse(init.read_text()).body - for node in body: - docstring = isinstance(node, ast.Expr) and isinstance(node.value, ast.Constant) - future = isinstance(node, ast.ImportFrom) and node.module == "__future__" - assert docstring or future, ( + """Native code loads per target, when ``route`` imports the target. + + ``models/__init__.py`` imports only the families' routing modules, a family + package holds a docstring only, and a routing module imports a target only + inside ``route``. Anything else would load a target's native code on every + resolve, or for every target of the family. + """ + for node in ast.parse((_MODELS / "__init__.py").read_text()).body: + routing_import = ( + isinstance(node, ast.ImportFrom) + and node.level == 1 + and node.module is not None + and "." not in node.module + and [alias.name for alias in node.names] == ["routing"] + ) + assert _is_docstring_or_future(node) or routing_import, ( + f"{(_MODELS / '__init__.py').relative_to(_REPO)}:{node.lineno} -- " + "models/__init__.py imports the families' routing modules and nothing else" + ) + for init in sorted(_MODELS.glob("*/__init__.py")): + for node in ast.parse(init.read_text()).body: + assert _is_docstring_or_future(node), ( f"{init.relative_to(_REPO)}:{node.lineno} -- a family package holds a docstring only" ) + for routing in sorted(_MODELS.glob("*/routing.py")): + for node in ast.parse(routing.read_text()).body: + assert not (isinstance(node, ast.ImportFrom) and node.level == 1), ( + f"{routing.relative_to(_REPO)}:{node.lineno} -- imports .{node.module} at " + "module level; a routing module imports a target only inside route" + ) def test_modeling_v2_defaults_to_off(): @@ -395,7 +421,12 @@ def test_accuracy_entries_name_a_family(): def test_cmake_and_python_agree_on_names_and_architectures(tmp_path): """The loader and CBTS compute directory, namespace and architecture coverage in Python; the build computes them in modeling_v2.cmake.""" - names = ["fixture", "minimax_m3", "scale_add__sm_10x__default", "r1_0528_nvfp4__sm_10x__default"] + names = [ + "fixture", + "minimax_m3", + "scale_add__sm_10x__default", + "r1_0528_nvfp4__sm_10x__default", + ] script = tmp_path / "probe.cmake" lines = [f'include("{_CMAKE_MODULE}")'] for name in names: From 9382cdc6f08f3be35e2550cfe32f4306f81bb306 Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Mon, 26 Oct 2026 01:50:23 -0700 Subject: [PATCH 06/20] [None][feat] modeling_v2: family-shared native ops and targets without code of their own Completes the family-shared side of the family libraries, which a family whose targets all call the same native ops needs (Kimi K3 is the first): - add_modeling_v2_target takes SOURCES optionally. A target without them has no directory and runs its family's shared code; the library carries it only when the shared code is built too. - Source properties a family sets in its own CMakeLists.txt (COMPILE_OPTIONS, COMPILE_DEFINITIONS) also apply to its shared code, which is assembled from the modeling_v2 directory. - The loader imports the family's models//fake_ops.py when it opens the library, and load_family_ops(__name__) returns the shared ops, torch.ops.mv2_, to code anywhere in the family. - The claims test requires a fake for every shared op in the family's fake_ops.py. Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- cpp/cmake/modules/modeling_v2.cmake | 82 +++++++++++++++---- cpp/tensorrt_llm/modeling_v2/README.md | 37 ++++++++- .../_experimental/modeling_v2/README.md | 5 +- .../_experimental/modeling_v2/_native.py | 60 +++++++++++--- .../test_modeling_v2_native_claims.py | 24 ++++-- .../test_modeling_v2_native_loader.py | 48 ++++++++++- 6 files changed, 218 insertions(+), 38 deletions(-) diff --git a/cpp/cmake/modules/modeling_v2.cmake b/cpp/cmake/modules/modeling_v2.cmake index 937f9f402b24..81ba91ec7d36 100644 --- a/cpp/cmake/modules/modeling_v2.cmake +++ b/cpp/cmake/modules/modeling_v2.cmake @@ -31,7 +31,7 @@ names (snake_case); directories are their camelCase form: add_modeling_v2_family() add_modeling_v2_shared( ARCHS ... SOURCES ... [DEPS ...] [REQUIRES ...]) - add_modeling_v2_target( ARCHS ... SOURCES ... [DEPS ...] [REQUIRES ...]) + add_modeling_v2_target( ARCHS ... [SOURCES ...] [DEPS ...] [REQUIRES ...]) ARCHS NN, NNa or NNf. Each is intersected with the build's requested architectures, the way flash_mla and deep_ep choose theirs; a target @@ -39,7 +39,12 @@ ARCHS NN, NNa or NNf. Each is intersected with the build's requested does not carry it. SOURCES Paths or globs relative to the target directory (shared/ for add_modeling_v2_shared). .cu files are compiled for ARCHS without - torch; .cpp files are the torch bindings. + torch; .cpp files are the torch bindings. A target without SOURCES + has no directory and no code of its own: it runs its family's + shared code, and declaring it is what makes the library carry it. + Source properties the family's CMakeLists.txt sets with + set_source_files_properties (e.g. COMPILE_OPTIONS) apply to shared + and target sources alike. DEPS From a closed list: cublas, cublaslt, cuda_driver, internal_cutlass, nccl, trtllm_gen_fmha_interface. REQUIRES Build features: MULTI_DEVICE, NVSHMEM. Unmet requirements skip the @@ -370,6 +375,20 @@ function( endif() endforeach() + # Shared code is assembled from the modeling_v2 directory, where source + # properties the family set in its own directory are not visible. + get_property(family_dir GLOBAL PROPERTY MODELING_V2_${family}_SOURCE_DIR) + foreach(source IN LISTS cu_sources cpp_sources) + foreach(property COMPILE_OPTIONS COMPILE_DEFINITIONS) + get_source_file_property(value "${source}" DIRECTORY "${family_dir}" + ${property}) + if(value) + set_source_files_properties("${source}" PROPERTIES ${property} + "${value}") + endif() + endforeach() + endforeach() + set(definitions TRTLLM_MV2_TORCH_NS=${torch_ns} TRTLLM_MV2_FAMILY_TORCH_NS=mv2_${family} TRTLLM_MV2_SCOPE=${scope}) @@ -454,20 +473,29 @@ function(add_modeling_v2_target family target) cmake_parse_arguments(PARSE_ARGV 2 arg "" "" "ARCHS;SOURCES;DEPS;REQUIRES") _modeling_v2_require_family("${family}") _modeling_v2_check_target_name("${target}") - if(NOT arg_ARCHS OR NOT arg_SOURCES) + if(NOT arg_ARCHS) message( FATAL_ERROR - "modeling_v2: add_modeling_v2_target(${family} ${target}) needs ARCHS and SOURCES" - ) + "modeling_v2: add_modeling_v2_target(${family} ${target}) needs ARCHS") endif() _modeling_v2_check_deps(${arg_DEPS}) _modeling_v2_camel(target_dir_name "${target}") set(target_dir "${CMAKE_CURRENT_SOURCE_DIR}/${target_dir_name}") - if(NOT IS_DIRECTORY "${target_dir}") + if(arg_SOURCES AND NOT IS_DIRECTORY "${target_dir}") message( FATAL_ERROR "modeling_v2: target '${target}' must live in directory ${target_dir}") endif() + if(NOT arg_SOURCES) + get_property(has_shared GLOBAL PROPERTY MODELING_V2_${family}_HAS_SHARED) + if(NOT has_shared) + message( + FATAL_ERROR + "modeling_v2: target '${family}/${target}' has no SOURCES, so it runs " + "its family's shared code; declare add_modeling_v2_shared(${family} ...) " + "before it") + endif() + endif() get_property(targets GLOBAL PROPERTY MODELING_V2_${family}_TARGETS) if(target IN_LIST targets) @@ -489,21 +517,43 @@ function(add_modeling_v2_target family target) _modeling_v2_skip_reason(reason "${family}" "${target}" "${arg_ARCHS}" "${arg_REQUIRES}") + if(reason STREQUAL "" AND NOT arg_SOURCES) + # A target without code of its own runs only where its shared code builds. + get_property(shared_archs_declared GLOBAL + PROPERTY MODELING_V2_${family}_SHARED_ARCHS) + get_property(shared_requires GLOBAL + PROPERTY MODELING_V2_${family}_SHARED_REQUIRES) + _modeling_v2_unmet_requirements(unmet ${shared_requires}) + _modeling_v2_resolve_archs(shared_archs DECLARED ${shared_archs_declared}) + if(NOT unmet STREQUAL "") + set(reason "its family's shared code is not built: ${unmet}") + elseif(NOT shared_archs) + set(reason "its family's shared code declares ${shared_archs_declared}") + endif() + endif() if(NOT reason STREQUAL "") message(STATUS "modeling_v2: ${family}/${target} not built: ${reason}") return() endif() _modeling_v2_resolve_archs(archs DECLARED ${arg_ARCHS}) - message(STATUS "modeling_v2: ${family}/${target} built for ${archs}") - _modeling_v2_expand_sources(sources "${target_dir}" ${arg_SOURCES}) - _modeling_v2_add_objects( - th_modeling_v2_${family}_${target_ident} - "${family}" - "${torch_ns}" - "${family}::${target_ident}" - "${archs}" - "${sources}" - "${arg_DEPS}") + if(arg_SOURCES) + message(STATUS "modeling_v2: ${family}/${target} built for ${archs}") + _modeling_v2_expand_sources(sources "${target_dir}" ${arg_SOURCES}) + _modeling_v2_add_objects( + th_modeling_v2_${family}_${target_ident} + "${family}" + "${torch_ns}" + "${family}::${target_ident}" + "${archs}" + "${sources}" + "${arg_DEPS}") + else() + message( + STATUS + "modeling_v2: ${family}/${target} carried for ${archs} (family shared code only)" + ) + set_property(GLOBAL APPEND PROPERTY MODELING_V2_${family}_ARCHS ${archs}) + endif() set_property(GLOBAL APPEND PROPERTY MODELING_V2_${family}_BUILT "${target}") endfunction() diff --git a/cpp/tensorrt_llm/modeling_v2/README.md b/cpp/tensorrt_llm/modeling_v2/README.md index c42e00b04a73..6e5060e652fd 100644 --- a/cpp/tensorrt_llm/modeling_v2/README.md +++ b/cpp/tensorrt_llm/modeling_v2/README.md @@ -124,6 +124,39 @@ The example below adds an op to target `foo_nvfp4__sm_103__tp4` of family No stock file changes along the way. +### Code the family's targets share + +When several targets of a family call the same op, its code goes in the +family's `shared/` once, and its ops are in the family's torch namespace, +`torch.ops.mv2_`: + +```cmake +add_modeling_v2_family(foo) + +add_modeling_v2_shared( + foo + ARCHS 100f + SOURCES kernels/*.cu ops/*.cpp + REQUIRES MULTI_DEVICE) + +# Targets with no code of their own declare no SOURCES and have no directory; +# declaring them is what makes the family library carry them. +add_modeling_v2_target(foo foo_nvfp4__sm_100__tp8 ARCHS 100f) +add_modeling_v2_target(foo foo_nvfp4__sm_100__tp16 ARCHS 100f) +``` + +- **Fakes** of the shared ops go in the family's `models/foo/fake_ops.py`; the + loader imports it right after it opens the library. Use + `torch_namespace(family_of(__name__))` for the namespace. +- **Callers** anywhere in the family reach the ops through + `load_family_ops(__name__).my_op(...)`; after the first call it is a + dictionary lookup. +- **Target packages** still start with `load_target_ops(__name__)`, which is + where routing finds out whether this build carries the target. +- **A target without code of its own** is skipped whenever the shared code is + (an unmet `REQUIRES`, or no matching architecture), so the library never + carries a target whose ops it lacks. + ## CMake API The functions are defined in `cpp/cmake/modules/modeling_v2.cmake`, in the style @@ -132,13 +165,13 @@ of `add_th_common_family`. | Function | What it declares | |---|---| | `add_modeling_v2_family()` | The family; call it first in the family's `CMakeLists.txt`. | -| `add_modeling_v2_target( ARCHS ... SOURCES ... [DEPS ...] [REQUIRES ...])` | One target's code, in `/`. | +| `add_modeling_v2_target( ARCHS ... [SOURCES ...] [DEPS ...] [REQUIRES ...])` | One target's code, in `/`; without `SOURCES`, a target that runs only its family's shared code. | | `add_modeling_v2_shared( ARCHS ... SOURCES ... [DEPS ...] [REQUIRES ...])` | Code several targets of the family call, in `shared/`. Its ops are in `torch.ops.mv2_`. | | Keyword | Meaning | |---|---| | `ARCHS` | `NN`, `NNa` or `NNf`, intersected with the build's requested architectures: `NN`/`NNa` build when `NN` is requested; `NNf` builds when any member of family `NN` is. This is how flash_mla and deep_ep choose their architectures. A target whose intersection is empty is **not built**: no placeholder kernels, and the configure step prints why. The declared architectures must cover the SM in the target's name; when moving existing code, keep the architectures it was gated with. | -| `SOURCES` | Paths or globs relative to the unit's directory. `.cu` files are compiled for `ARCHS` without torch; `.cpp` files are the torch bindings. | +| `SOURCES` | Paths or globs relative to the unit's directory. `.cu` files are compiled for `ARCHS` without torch; `.cpp` files are the torch bindings. A per-file option goes on the file itself, in the family's `CMakeLists.txt`: `set_source_files_properties(shared/kernels/x.cu PROPERTIES COMPILE_OPTIONS --use_fast_math)`. | | `DEPS` | `cublas`, `cublaslt`, `cuda_driver`, `internal_cutlass`, `nccl`, `trtllm_gen_fmha_interface`. Ask for anything else by extending the module. | | `REQUIRES` | `MULTI_DEVICE`, `NVSHMEM`. A target whose requirement the build lacks (e.g. a single-device build) is skipped and recorded. | diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/README.md b/tensorrt_llm/_torch/_experimental/modeling_v2/README.md index eae8cbb8fcc6..9b896406d3fb 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/README.md +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/README.md @@ -114,6 +114,7 @@ _router_index.py architectures[0] -> routing module. Small, stable, test-guar _native.py loads a target's native code (see "Native code" below) models// routing.py one forward-reading decision tree per architecture family + fake_ops.py fakes of the family's shared native ops, if it has any ____/ modeling.py weights.py ops// entries only this target calls @@ -132,7 +133,9 @@ family's targets, whose `__init__.py` starts with `native_ops = load_target_ops(__name__)`. As with the stock code's optional native modules, a library this installation does not carry is a `NoTarget`, so `auto` keeps the built-in model; one that is installed but broken is a -`NativeLoadError`, never a fallback. +`NativeLoadError`, never a fallback. Code several targets of a family share is +built once, in the family's `shared/`; its ops are `torch.ops.mv2_`, +which family code reaches with `load_family_ops(__name__)`. `cpp/tensorrt_llm/modeling_v2/README.md` is the guide, and `models/fixture/`, a temporary infrastructure fixture rather than a model, is the smallest complete example. diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/_native.py b/tensorrt_llm/_torch/_experimental/modeling_v2/_native.py index d16ca4b62222..f72ae866d880 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/_native.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/_native.py @@ -23,8 +23,12 @@ * which targets a library carries it answers itself, ``torch.ops.mv2_.built_targets()``, the way ``th_common`` answers feature queries; a target it does not carry is :class:`NoTarget`; -* the target's ``fake_ops.py`` registers right after the library opens, which fails - if an op it declares is missing from the library. +* the family's ``fake_ops.py`` (its shared ops) registers right after the library + opens, and the target's ``fake_ops.py`` right after the target is found in it; + either fails if an op it declares is missing from the library. + +Ops of a family's shared code are in ``torch.ops.mv2_``; code of the family +reaches them through :func:`load_family_ops`. ``cpp/tensorrt_llm/modeling_v2/README.md`` is the guide for authors. """ @@ -46,6 +50,7 @@ _LIBS_DIR = Path(__file__).resolve().parents[3] / "libs" _built_targets: Dict[str, FrozenSet[str]] = {} +_family_ops: Dict[str, Any] = {} class NativeLoadError(RuntimeError): @@ -97,6 +102,15 @@ def split_target_package(package: str) -> Tuple[str, str]: return parts[0], parts[1] +def family_of(module: str) -> str: + """The family of a module anywhere under ``models.``.""" + prefix = f"{_MODELS_PACKAGE}." + family = module[len(prefix) :].partition(".")[0] if module.startswith(prefix) else "" + if not family: + raise ValueError(f"{module!r} is not a module of a modeling_v2 family") + return family + + def namespace_of(package: str) -> str: """The torch namespace of the target package ``package``, e.g. for its fake_ops.py.""" return torch_namespace(*split_target_package(package)) @@ -105,6 +119,19 @@ def namespace_of(package: str) -> str: # --- Loading ------------------------------------------------------------------ +def _import_fakes(module: str, family: str) -> None: + """Import a ``fake_ops`` module, if there is one; it registers fakes of ops the + family library must define.""" + if importlib.util.find_spec(module) is None: + return + try: + importlib.import_module(module) + except (AttributeError, RuntimeError, ValueError) as error: + raise NativeLoadError( + f"{module} does not match the ops in {library_name(family)}: {error}" + ) from None + + def _open_library(path: Path, family: str) -> FrozenSet[str]: """dlopen a family library and return the targets it carries.""" import torch @@ -135,6 +162,7 @@ def load_family_library(family: str) -> FrozenSet[str]: built = _open_library(path, family) except (AttributeError, OSError, RuntimeError) as error: raise NativeLoadError(f"{path} is installed but cannot be loaded: {error}") from None + _import_fakes(f"{_MODELS_PACKAGE}.{family}.fake_ops", family) _built_targets[family] = built return built @@ -157,14 +185,26 @@ def load_target_ops(package: str) -> Any: f"TRTLLM_MODELING_V2_TARGETS excluded it); it carries {carried}" ) - fake_ops = f"{package}.fake_ops" - if importlib.util.find_spec(fake_ops) is not None: - try: - importlib.import_module(fake_ops) - except (AttributeError, RuntimeError, ValueError) as error: - raise NativeLoadError( - f"{fake_ops} does not match the ops in {library_name(family)}: {error}" - ) from None + _import_fakes(f"{package}.fake_ops", family) import torch return getattr(torch.ops, torch_namespace(family, target)) + + +def load_family_ops(module: str) -> Any: + """Load the native code of the family ``module`` belongs to and return the ops of + its shared code, ``torch.ops.mv2_``. + + Call it with ``__name__`` from any module of the family, where the op is used; + after the first call it is a dictionary lookup. Raises like + :func:`load_family_library`. Whether a target is carried is + :func:`load_target_ops`'s check, which the target's ``__init__.py`` runs first. + """ + ops = _family_ops.get(module) + if ops is None: + family = family_of(module) + load_family_library(family) + import torch + + ops = _family_ops[module] = getattr(torch.ops, torch_namespace(family)) + return ops diff --git a/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_claims.py b/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_claims.py index b81c82f2db77..9f2a3f7b4637 100644 --- a/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_claims.py +++ b/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_claims.py @@ -228,18 +228,26 @@ def test_native_code_registers_only_into_its_own_namespace(): def test_every_native_op_has_a_fake(): - """fake_ops.py doubles as the Python side's list of a target's ops: the - loader imports it, so an op without a fake is one torch.compile and fake - tracing cannot see, and a fake without an op fails the load.""" - for family, target, _ in _targets(): - target_dir = _families()[family] / camel(target) + """fake_ops.py doubles as the Python side's list of a unit's ops: the loader + imports it, so an op without a fake is one torch.compile and fake tracing + cannot see, and a fake without an op fails the load. A target's ops have + their fakes in the target's fake_ops.py, the family's shared ops in the + family's.""" + units = [(_families()[f] / camel(t), _MODELS / f / t) for f, t, _ in _targets()] + units += [(d / "shared", _MODELS / f) for f, d in _families().items()] + for native_dir, python_dir in units: + if not native_dir.is_dir(): + continue ops = sorted( - {op for p in _native_files(target_dir) for op in _OP_DEF.findall(p.read_text())} + {op for p in _native_files(native_dir) for op in _OP_DEF.findall(p.read_text())} ) if not ops: continue - fake_ops = _MODELS / family / target / "fake_ops.py" - assert fake_ops.is_file(), f"{family}/{target} defines {ops} but has no fake_ops.py" + fake_ops = python_dir / "fake_ops.py" + assert fake_ops.is_file(), ( + f"{native_dir.relative_to(_REPO)} defines {ops}, " + f"but {fake_ops.relative_to(_REPO)} does not exist" + ) text = fake_ops.read_text() missing = [op for op in ops if f'::{op}"' not in text] assert not missing, f"{fake_ops.relative_to(_REPO)} has no fake for {missing}" diff --git a/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_loader.py b/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_loader.py index 2c2185040370..3bc88df07087 100644 --- a/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_loader.py +++ b/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_loader.py @@ -20,6 +20,7 @@ import subprocess import sys import textwrap +import types from pathlib import Path import pytest @@ -28,9 +29,11 @@ from tensorrt_llm._torch._experimental.modeling_v2._native import ( NativeLoadError, camel, + family_of, identifier, library_name, load_family_library, + load_family_ops, load_target_ops, namespace_of, split_target_package, @@ -48,13 +51,16 @@ def libs(monkeypatch, tmp_path): """An empty libs directory and a cold library cache.""" monkeypatch.setattr(_native, "_LIBS_DIR", tmp_path) monkeypatch.setattr(_native, "_built_targets", {}) + monkeypatch.setattr(_native, "_family_ops", {}) return tmp_path def _install(libs: Path, monkeypatch, opens) -> None: - """Put a fixture library in `libs`; `opens` stands in for dlopen.""" + """Put a fixture library in `libs`; `opens` stands in for dlopen, and no + fake_ops module is imported, since the stand-in defines no op.""" (libs / library_name("fixture")).write_bytes(b"") monkeypatch.setattr(_native, "_open_library", lambda path, family: opens()) + monkeypatch.setattr(_native, "_import_fakes", lambda module, family: None) # --- Naming ---------------------------------------------------------------------- @@ -82,6 +88,10 @@ def test_names_of_libraries_and_namespaces(): assert split_target_package(_FIXTURE) == ("fixture", "scale_add__sm_10x__default") with pytest.raises(ValueError): split_target_package("tensorrt_llm._torch.models") + assert family_of(f"{_FIXTURE}.ops.elementwise.scale_add") == "fixture" + assert family_of(_FIXTURE.rpartition(".")[0] + ".fake_ops") == "fixture" + with pytest.raises(ValueError): + family_of("tensorrt_llm._torch._experimental.modeling_v2.models") # --- Decisions ----------------------------------------------------------------- @@ -120,6 +130,42 @@ def opens(): assert len(opened) == 1 +@pytest.fixture +def torch_ops(monkeypatch): + """A stand-in torch.ops holding the fixture's namespaces.""" + ops = types.SimpleNamespace(mv2_fixture=object(), mv2_fixture_scale_add_sm_10x_default=object()) + monkeypatch.setitem(sys.modules, "torch", types.SimpleNamespace(ops=ops)) + return ops + + +def test_opening_a_library_registers_the_family_fakes_first(libs, monkeypatch, torch_ops): + _install(libs, monkeypatch, lambda: frozenset({"scale_add__sm_10x__default"})) + imported = [] + monkeypatch.setattr(_native, "_import_fakes", lambda module, family: imported.append(module)) + assert load_target_ops(_FIXTURE) is torch_ops.mv2_fixture_scale_add_sm_10x_default + family_package = _FIXTURE.rpartition(".")[0] + assert imported == [f"{family_package}.fake_ops", f"{_FIXTURE}.fake_ops"] + + +def test_family_ops_are_the_family_namespace_and_load_once(libs, monkeypatch, torch_ops): + opened = [] + + def opens(): + opened.append(1) + return frozenset({"scale_add__sm_10x__default"}) + + _install(libs, monkeypatch, opens) + module = f"{_FIXTURE}.ops.elementwise.scale_add" + assert load_family_ops(module) is torch_ops.mv2_fixture + assert load_family_ops(module) is torch_ops.mv2_fixture + assert len(opened) == 1 + + +def test_family_ops_of_a_family_this_installation_lacks_are_no_target(libs): + with pytest.raises(NoTarget, match="has no libth_modeling_v2_fixture.so"): + load_family_ops(f"{_FIXTURE}.ops.elementwise.scale_add") + + def test_windows_has_no_native_targets(libs, monkeypatch): monkeypatch.setattr(sys, "platform", "win32") with pytest.raises(NoTarget, match="Windows"): From 65b5ec27f2c1e5c401a488b382095d96d5c3ae5e Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Mon, 26 Oct 2026 02:36:28 -0700 Subject: [PATCH 07/20] [None][feat] modeling_v2: keep a family's native code with its Python code; gpt_oss as the example A family's C++/CUDA now lives in its own directory of the modeling_v2 tree, so adding a model with kernels edits nothing under cpp/: - Layout: models//csrc/ holds the family's CMakeLists.txt and the code its targets share; models///csrc/ a target's own kernels and torch bindings. Includes of the family's code are written from models/ ("//csrc/x.h"). - cpp/tensorrt_llm/modeling_v2/ keeps only what every family library shares: the discovery, which now finds every models/*/csrc/CMakeLists.txt, the common/ macros, the version script and contractSurface.txt. The camelCase directory mapping is gone, from the build, the loader and the CBTS rule. - One-time hooks for the new location: setup.py's precompiled-wheel skew detection covers models/**/csrc/, CODEOWNERS adds the kernels owners on it, and the CBTS rule scopes a csrc/ change to its family. - Checks: native code only in a family's or target's csrc/ (CMakeLists.txt only in the family's), no stock CMake, C++ or packaging file names a family, every csrc/ file carries the SPDX header (the C++ license checker scans cpp/ only), and every target csrc/ is declared. - The temporary fixture family is replaced by a real example: gpt_oss's gpt_oss_120b__sm_10x__default carries swizzle_block_scales, the MXFP4 block-scale swizzle its weight loader ran in torch, in one kernel pass. It only moves bytes; its unit test checks it bit for bit against the torch swizzle, at gpt-oss-120b's FC1 and FC2 shapes, and its rejections and fake. B300's family entry is now unittest/_torch/modeling_v2/models/gpt_oss. - NATIVE.md, next to the code, is the author guide: the directories and files a new model adds, step by step with gpt_oss, the CMake API, namespaces, includes, loading and CI. Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- .github/CODEOWNERS | 9 +- cpp/cmake/modules/modeling_v2.cmake | 91 +++--- cpp/tensorrt_llm/modeling_v2/CMakeLists.txt | 28 +- cpp/tensorrt_llm/modeling_v2/README.md | 308 +----------------- .../scaleAddSm10xDefault/kernels/scaleAdd.cu | 97 ------ .../scaleAddSm10xDefault/ops/scaleAddOp.cpp | 61 ---- jenkins/scripts/cbts/README.md | 2 +- jenkins/scripts/cbts/cbts/command/main.py | 2 +- jenkins/scripts/cbts/cbts/rules/README.md | 13 +- .../scripts/cbts/cbts/rules/modeling_v2.py | 59 +--- setup.py | 9 +- .../_experimental/modeling_v2/NATIVE.md | 248 ++++++++++++++ .../_experimental/modeling_v2/README.md | 30 +- .../_experimental/modeling_v2/_native.py | 20 +- .../modeling_v2/models/fixture/__init__.py | 14 - .../scale_add__sm_10x__default/__init__.py | 15 - .../scale_add__sm_10x__default/fake_ops.py | 20 -- .../ops/__init__.py | 3 - .../ops/elementwise/__init__.py | 2 - .../ops/elementwise/scale_add.py | 56 ---- .../models/gpt_oss/csrc}/CMakeLists.txt | 14 +- .../gpt_oss_120b__sm_10x__default/__init__.py | 4 + .../csrc/swizzleBlockScales.cu | 67 ++++ .../csrc/swizzleBlockScales.h | 19 +- .../csrc/swizzleBlockScalesOp.cpp | 68 ++++ .../gpt_oss_120b__sm_10x__default/fake_ops.py | 18 + .../gpt_oss_120b__sm_10x__default/weights.py | 10 +- .../test_lists/test-db/l0_b300.yml | 3 +- .../elementwise/test_modeling_v2_scale_add.py | 67 ---- ...odeling_v2_gpt_oss_swizzle_block_scales.py | 79 +++++ .../test_modeling_v2_native_claims.py | 196 ++++++----- .../test_modeling_v2_native_loader.py | 79 ++--- .../scripts/cbts/test_modeling_v2_rule.py | 62 ++-- 33 files changed, 814 insertions(+), 959 deletions(-) delete mode 100644 cpp/tensorrt_llm/modeling_v2/fixture/scaleAddSm10xDefault/kernels/scaleAdd.cu delete mode 100644 cpp/tensorrt_llm/modeling_v2/fixture/scaleAddSm10xDefault/ops/scaleAddOp.cpp create mode 100644 tensorrt_llm/_torch/_experimental/modeling_v2/NATIVE.md delete mode 100644 tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/__init__.py delete mode 100644 tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_10x__default/__init__.py delete mode 100644 tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_10x__default/fake_ops.py delete mode 100644 tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_10x__default/ops/__init__.py delete mode 100644 tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_10x__default/ops/elementwise/__init__.py delete mode 100644 tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_10x__default/ops/elementwise/scale_add.py rename {cpp/tensorrt_llm/modeling_v2/fixture => tensorrt_llm/_torch/_experimental/modeling_v2/models/gpt_oss/csrc}/CMakeLists.txt (57%) create mode 100644 tensorrt_llm/_torch/_experimental/modeling_v2/models/gpt_oss/gpt_oss_120b__sm_10x__default/csrc/swizzleBlockScales.cu rename cpp/tensorrt_llm/modeling_v2/fixture/scaleAddSm10xDefault/kernels/scaleAdd.h => tensorrt_llm/_torch/_experimental/modeling_v2/models/gpt_oss/gpt_oss_120b__sm_10x__default/csrc/swizzleBlockScales.h (63%) create mode 100644 tensorrt_llm/_torch/_experimental/modeling_v2/models/gpt_oss/gpt_oss_120b__sm_10x__default/csrc/swizzleBlockScalesOp.cpp create mode 100644 tensorrt_llm/_torch/_experimental/modeling_v2/models/gpt_oss/gpt_oss_120b__sm_10x__default/fake_ops.py delete mode 100644 tests/unittest/_torch/modeling_v2/models/fixture/scale_add__sm_10x__default/elementwise/test_modeling_v2_scale_add.py create mode 100644 tests/unittest/_torch/modeling_v2/models/gpt_oss/gpt_oss_120b__sm_10x__default/test_modeling_v2_gpt_oss_swizzle_block_scales.py diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index aef368040f09..1392d91ea5e3 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -387,8 +387,9 @@ # Individual handles rather than a team, like SCAFFOLDING: this is one bounded # experiment with named owners, not a standing domain. /tensorrt_llm/_torch/_experimental/modeling_v2 @tianyuxbear @Wanli-Jiang @WeiHaocheng @litaotju -# modeling_v2 native code: the modeling_v2 owners for the targets, the kernels -# owners because it is CUDA and links against the stock libraries. +# What every modeling_v2 family library shares (discovery, common/ macros, the +# contract surface): the modeling_v2 owners, and the kernels owners because it is +# CUDA and links against the stock libraries. /cpp/tensorrt_llm/modeling_v2 @tianyuxbear @Wanli-Jiang @WeiHaocheng @litaotju @NVIDIA/trt-llm-kernels-devs /cpp/cmake/modules/modeling_v2.cmake @tianyuxbear @Wanli-Jiang @WeiHaocheng @litaotju @NVIDIA/trt-llm-kernels-devs /tests/unittest/_torch/modeling_v2 @tianyuxbear @Wanli-Jiang @WeiHaocheng @litaotju @@ -403,6 +404,10 @@ /tests/unittest/_torch/modeling_v2/models/glm_moe_dsa @tianyuxbear @Wanli-Jiang @WeiHaocheng @litaotju @dongfengy @Tabrizian @xwang233 /cpp/tensorrt_llm/kernels/glmMoeDsa @NVIDIA/trt-llm-kernels-devs @dongfengy @Tabrizian @xwang233 /cpp/tensorrt_llm/thop/glmMoeDsa/ @NVIDIA/trt-llm-kernels-devs @dongfengy @Tabrizian @xwang233 +# A family's native code lives in its csrc/ directories: the kernels owners +# co-own it. Last, so it applies under the family rules above; a family with its +# own owners adds them here. +/tensorrt_llm/_torch/_experimental/modeling_v2/models/**/csrc/ @tianyuxbear @Wanli-Jiang @WeiHaocheng @litaotju @NVIDIA/trt-llm-kernels-devs ## TensorRT-LLM LLM Disaggregated /examples/disaggregated @NVIDIA/trt-llm-disagg-devs @NVIDIA/trt-llm-doc-owners diff --git a/cpp/cmake/modules/modeling_v2.cmake b/cpp/cmake/modules/modeling_v2.cmake index 81ba91ec7d36..2a105a72b1e8 100644 --- a/cpp/cmake/modules/modeling_v2.cmake +++ b/cpp/cmake/modules/modeling_v2.cmake @@ -18,16 +18,23 @@ #[[ Native (C++/CUDA) code of modeling_v2 targets. -Each model family under cpp/tensorrt_llm/modeling_v2// builds into -one torch-op library, libth_modeling_v2_.so, installed next to -libth_common.so. Nothing in it is linked into libtensorrt_llm.so or -libth_common.so; the Python loader +A model family's native code lives with its Python code, under +tensorrt_llm/_torch/_experimental/modeling_v2/models//: + + csrc/CMakeLists.txt declares the family, its shared code and its targets + csrc/*.cu|*.h|*.cpp code several targets of the family share + /csrc/... one target's own kernels (.cu) and torch bindings (.cpp) + +Each family builds into one torch-op library, libth_modeling_v2_.so, +installed next to libth_common.so. Nothing in it is linked into +libtensorrt_llm.so or libth_common.so; the Python loader (tensorrt_llm/_torch/_experimental/modeling_v2/_native.py) opens it only when -routing picks one of the family's targets. cpp/tensorrt_llm/modeling_v2/README.md -is the guide for authors. +routing picks one of the family's targets. modeling_v2/NATIVE.md is the guide +for authors; cpp/tensorrt_llm/modeling_v2/ holds only what every family shares +(this discovery, common/ macros, contractSurface.txt). -A family's CMakeLists.txt uses three functions. Names are the Python package -names (snake_case); directories are their camelCase form: +A family's csrc/CMakeLists.txt uses three functions; names are the Python +package names: add_modeling_v2_family() add_modeling_v2_shared( ARCHS ... SOURCES ... [DEPS ...] [REQUIRES ...]) @@ -37,11 +44,14 @@ ARCHS NN, NNa or NNf. Each is intersected with the build's requested architectures, the way flash_mla and deep_ep choose theirs; a target with an empty intersection is not built and its family library does not carry it. -SOURCES Paths or globs relative to the target directory (shared/ for - add_modeling_v2_shared). .cu files are compiled for ARCHS without - torch; .cpp files are the torch bindings. A target without SOURCES - has no directory and no code of its own: it runs its family's - shared code, and declaring it is what makes the library carry it. +SOURCES Paths or globs relative to the unit's csrc/ directory: the + family's for add_modeling_v2_shared, /csrc/ for a target. + .cu files are compiled for ARCHS without torch; .cpp files are the + torch bindings. Includes of the family's own code are written from + the models/ directory: "/csrc/x.h", + "//csrc/x.h". A target without SOURCES has no csrc/ + and no code of its own: it runs its family's shared code, and + declaring it is what makes the library carry it. Source properties the family's CMakeLists.txt sets with set_source_files_properties (e.g. COMPILE_OPTIONS) apply to shared and target sources alike. @@ -64,29 +74,6 @@ set(_MODELING_V2_KNOWN_REQUIRES MULTI_DEVICE NVSHMEM) # --- Naming ------------------------------------------------------------------ -# camelCase form of a snake_case name: minimax_m3_nvfp4__sm_107__tp2 -> -# minimaxM3Nvfp4Sm107Tp2. The same mapping as camel() in the Python checks. -function(_modeling_v2_camel out name) - string(REPLACE "_" ";" parts "${name}") - set(result "") - foreach(part IN LISTS parts) - if(part STREQUAL "") - continue() - endif() - if(result STREQUAL "") - set(result "${part}") - else() - string(SUBSTRING "${part}" 0 1 head) - string(SUBSTRING "${part}" 1 -1 tail) - string(TOUPPER "${head}" head) - string(APPEND result "${head}${tail}") - endif() - endforeach() - set(${out} - "${result}" - PARENT_SCOPE) -endfunction() - # A name usable as a C++ or torch identifier: "__" collapsed to "_", since C++ # reserves identifiers that contain a double underscore. function(_modeling_v2_ident out name) @@ -234,13 +221,14 @@ endfunction() function(add_modeling_v2_family family) _modeling_v2_check_family_name("${family}") - _modeling_v2_camel(camel "${family}") get_filename_component(dir_name "${CMAKE_CURRENT_SOURCE_DIR}" NAME) - if(NOT dir_name STREQUAL camel) + get_filename_component(package_dir "${CMAKE_CURRENT_SOURCE_DIR}" DIRECTORY) + get_filename_component(package_name "${package_dir}" NAME) + if(NOT dir_name STREQUAL "csrc" OR NOT package_name STREQUAL family) message( FATAL_ERROR - "modeling_v2: family '${family}' must live in directory '${camel}', not '${dir_name}'" - ) + "modeling_v2: family '${family}' is declared in models/${family}/csrc/CMakeLists.txt, " + "not in ${CMAKE_CURRENT_SOURCE_DIR}") endif() get_property(families GLOBAL PROPERTY MODELING_V2_FAMILIES) if(family IN_LIST families) @@ -249,6 +237,8 @@ function(add_modeling_v2_family family) set_property(GLOBAL APPEND PROPERTY MODELING_V2_FAMILIES "${family}") set_property(GLOBAL PROPERTY MODELING_V2_${family}_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}") + set_property(GLOBAL PROPERTY MODELING_V2_${family}_PACKAGE_DIR + "${package_dir}") set_property(GLOBAL PROPERTY MODELING_V2_${family}_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}") endfunction() @@ -392,6 +382,10 @@ function( set(definitions TRTLLM_MV2_TORCH_NS=${torch_ns} TRTLLM_MV2_FAMILY_TORCH_NS=mv2_${family} TRTLLM_MV2_SCOPE=${scope}) + # Family code includes its own headers from the models/ directory. + get_filename_component(models_dir "${family_dir}" DIRECTORY) + get_filename_component(models_dir "${models_dir}" DIRECTORY) + set(objects "") if(cu_sources) add_library(${name}_kernels OBJECT ${cu_sources}) @@ -399,6 +393,7 @@ function( ${name}_kernels PROPERTIES POSITION_INDEPENDENT_CODE ON CUDA_ARCHITECTURES "${archs}") target_compile_definitions(${name}_kernels PRIVATE ${definitions}) + target_include_directories(${name}_kernels PRIVATE "${models_dir}") _modeling_v2_apply_compile_deps(${name}_kernels ${deps}) list(APPEND objects ${name}_kernels) endif() @@ -406,6 +401,7 @@ function( add_library(${name}_ops OBJECT ${cpp_sources}) set_target_properties(${name}_ops PROPERTIES POSITION_INDEPENDENT_CODE ON) target_compile_definitions(${name}_ops PRIVATE ${definitions}) + target_include_directories(${name}_ops PRIVATE "${models_dir}") target_link_libraries(${name}_ops PRIVATE ${TORCH_LIBRARIES}) _modeling_v2_apply_compile_deps(${name}_ops ${deps}) list(APPEND objects ${name}_ops) @@ -455,12 +451,6 @@ function(add_modeling_v2_shared family) ) endif() _modeling_v2_check_deps(${arg_DEPS}) - if(NOT IS_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/shared") - message( - FATAL_ERROR - "modeling_v2: family '${family}' shared code lives in ${CMAKE_CURRENT_SOURCE_DIR}/shared" - ) - endif() # Recorded now, built by _modeling_v2_add_family_library only when a target of # the family builds: shared code with no target has nothing to serve. foreach(field ARCHS SOURCES DEPS REQUIRES) @@ -479,12 +469,12 @@ function(add_modeling_v2_target family target) "modeling_v2: add_modeling_v2_target(${family} ${target}) needs ARCHS") endif() _modeling_v2_check_deps(${arg_DEPS}) - _modeling_v2_camel(target_dir_name "${target}") - set(target_dir "${CMAKE_CURRENT_SOURCE_DIR}/${target_dir_name}") + get_property(package_dir GLOBAL PROPERTY MODELING_V2_${family}_PACKAGE_DIR) + set(target_dir "${package_dir}/${target}/csrc") if(arg_SOURCES AND NOT IS_DIRECTORY "${target_dir}") message( FATAL_ERROR - "modeling_v2: target '${target}' must live in directory ${target_dir}") + "modeling_v2: target '${target}' keeps its code in ${target_dir}") endif() if(NOT arg_SOURCES) get_property(has_shared GLOBAL PROPERTY MODELING_V2_${family}_HAS_SHARED) @@ -627,8 +617,7 @@ function(_modeling_v2_add_family_library family) "this build requests ${CMAKE_CUDA_ARCHITECTURES_ORIG}") else() get_property(source_dir GLOBAL PROPERTY MODELING_V2_${family}_SOURCE_DIR) - _modeling_v2_expand_sources(sources "${source_dir}/shared" - ${shared_SOURCES}) + _modeling_v2_expand_sources(sources "${source_dir}" ${shared_SOURCES}) message(STATUS "modeling_v2: ${family}/shared built for ${shared_archs}") _modeling_v2_add_objects( th_modeling_v2_${family}_shared diff --git a/cpp/tensorrt_llm/modeling_v2/CMakeLists.txt b/cpp/tensorrt_llm/modeling_v2/CMakeLists.txt index 95cb4a75e89b..86d75a63f630 100644 --- a/cpp/tensorrt_llm/modeling_v2/CMakeLists.txt +++ b/cpp/tensorrt_llm/modeling_v2/CMakeLists.txt @@ -16,18 +16,24 @@ # # Native code of modeling_v2 targets: one torch-op library per model family, -# loaded only when routing picks one of its targets. Every subdirectory with a -# CMakeLists.txt is a family; see README.md and -# cpp/cmake/modules/modeling_v2.cmake. +# loaded only when routing picks one of its targets. A family keeps its native +# code with its Python code, in +# tensorrt_llm/_torch/_experimental/modeling_v2/models//csrc/ (declared +# by the CMakeLists.txt there) and //csrc/; this directory holds +# only what every family shares. See cpp/cmake/modules/modeling_v2.cmake. include(modeling_v2) add_custom_target(modeling_v2_libs) +get_filename_component( + modeling_v2_models_dir + "${PROJECT_SOURCE_DIR}/../tensorrt_llm/_torch/_experimental/modeling_v2/models" + ABSOLUTE) file( GLOB family_lists LIST_DIRECTORIES false - CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/*/CMakeLists.txt") + CONFIGURE_DEPENDS "${modeling_v2_models_dir}/*/csrc/CMakeLists.txt") list(SORT family_lists) if(WIN32 OR NOT BUILD_MODELING_V2) @@ -35,17 +41,19 @@ if(WIN32 OR NOT BUILD_MODELING_V2) # cuda_architectures.txt per family, so build_wheel.py installs nothing, # including a library left from an earlier configuration. foreach(family_list IN LISTS family_lists) - get_filename_component(family_dir "${family_list}" DIRECTORY) - get_filename_component(family_dir_name "${family_dir}" NAME) - file(WRITE - "${CMAKE_CURRENT_BINARY_DIR}/${family_dir_name}/cuda_architectures.txt" + get_filename_component(csrc_dir "${family_list}" DIRECTORY) + get_filename_component(family_dir "${csrc_dir}" DIRECTORY) + get_filename_component(family "${family_dir}" NAME) + file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/${family}/cuda_architectures.txt" "") endforeach() return() endif() foreach(family_list IN LISTS family_lists) - get_filename_component(family_dir "${family_list}" DIRECTORY) - add_subdirectory("${family_dir}") + get_filename_component(csrc_dir "${family_list}" DIRECTORY) + get_filename_component(family_dir "${csrc_dir}" DIRECTORY) + get_filename_component(family "${family_dir}" NAME) + add_subdirectory("${csrc_dir}" "${CMAKE_CURRENT_BINARY_DIR}/${family}") endforeach() add_modeling_v2_family_libraries() diff --git a/cpp/tensorrt_llm/modeling_v2/README.md b/cpp/tensorrt_llm/modeling_v2/README.md index 6e5060e652fd..613f77cd92da 100644 --- a/cpp/tensorrt_llm/modeling_v2/README.md +++ b/cpp/tensorrt_llm/modeling_v2/README.md @@ -1,302 +1,20 @@ -# modeling_v2 native code +# modeling_v2 family libraries: what they share -C++/CUDA used only by modeling_v2 targets -(`tensorrt_llm/_torch/_experimental/modeling_v2/`) lives here, not in -`cpp/tensorrt_llm/kernels/` or `thop/`. Each model family builds into one -torch-op library, `libth_modeling_v2_.so`, installed next to -`libth_common.so`. +Each modeling_v2 model family keeps its C++/CUDA with its Python code, in +`tensorrt_llm/_torch/_experimental/modeling_v2/models//csrc/` and +`//csrc/`, and builds into its own torch-op library, +`libth_modeling_v2_.so`. The guide for authors is +`tensorrt_llm/_torch/_experimental/modeling_v2/NATIVE.md`. -- **Not linked into stock.** Nothing in the family library is linked into - `libtensorrt_llm.so` or `libth_common.so`. -- **Loaded on demand.** Python loads the family library only when routing - picks one of the family's targets. With `modeling_v2=off` (the default), - nothing from here is loaded. +This directory holds only what every family library shares; adding a model +changes nothing here: -This document is the guide for adding native code to a target. The family -`fixture/` is the smallest complete example: one target with one op. Copy it. -It is a temporary infrastructure fixture, not a model, and is removed once a -family with native code has GPU unit tests that run in CI. - -## Layout - -``` -cpp/tensorrt_llm/modeling_v2/ - CMakeLists.txt adds every family directory (the only stock hook) - contractSurface.txt the stock headers native code may include - common/ namespace and torch-registration macros - / one model family -> libth_modeling_v2_.so - CMakeLists.txt add_modeling_v2_family / _shared / _target - shared/ code more than one target of the family calls - / one target - kernels/ *.cu *.cuh *.h -- compiled for the target's ARCHS, no torch - ops/ *.cpp torch bindings -``` - -### Naming - -**Names are the Python package names.** A family is `minimax_m3`; a target is -`minimax_m3_nvfp4__sm_107__tp2`. - -**Directories are their camelCase form**, following `CODING_GUIDELINES.md`: -`minimaxM3/minimaxM3Nvfp4Sm107Tp2/`. - -The build, the loader and CI compute the same mapping, and -`test_modeling_v2_native_claims.py` checks that it holds in both directions. - -## Adding native code to a target - -The example below adds an op to target `foo_nvfp4__sm_103__tp4` of family -`foo`. - -1. **Declare it.** The declaration goes in `cpp/tensorrt_llm/modeling_v2/foo/CMakeLists.txt`: - - ```cmake - add_modeling_v2_family(foo) - - add_modeling_v2_target( - foo foo_nvfp4__sm_103__tp4 - ARCHS 103a - SOURCES kernels/*.cu ops/*.cpp - REQUIRES MULTI_DEVICE) - ``` - -2. **Write the kernel.** It goes in `fooNvfp4Sm103Tp4/kernels/`, wrapped in the - target's namespace: - - ```cpp - #include "tensorrt_llm/modeling_v2/common/namespace.h" - - TRTLLM_MV2_NAMESPACE_BEGIN // tensorrt_llm::modeling_v2::foo::foo_nvfp4_sm_103_tp4 - namespace kernels - { - cudaError_t invokeMyOp(/* ... */, cudaStream_t stream); - } - TRTLLM_MV2_NAMESPACE_END - ``` - -3. **Bind it to torch** in `fooNvfp4Sm103Tp4/ops/`, through the target's own torch namespace: - - ```cpp - #include "tensorrt_llm/modeling_v2/common/torchLibrary.h" - - TRTLLM_MV2_LIBRARY_FRAGMENT(m) { m.def("my_op(Tensor x) -> Tensor"); } - TRTLLM_MV2_LIBRARY_IMPL(CUDA, m) { m.impl("my_op", &TRTLLM_MV2_NS::myOp); } - ``` - -4. **Load it** from the Python target package, - `models/foo/foo_nvfp4__sm_103__tp4/__init__.py`: - - ```python - from tensorrt_llm._torch._experimental.modeling_v2._native import load_target_ops - - native_ops = load_target_ops(__name__) # torch.ops.mv2_foo_foo_nvfp4_sm_103_tp4 - ``` - - The name is not `ops`: that is the target's `ops/` package, which would - replace the attribute when it is imported. - -5. **Fake every op** in the target's `fake_ops.py`. The loader imports it right - after opening the library: - - ```python - from tensorrt_llm._torch._experimental.modeling_v2._native import namespace_of - - _NS = namespace_of(__package__) - - @torch.library.register_fake(f"{_NS}::my_op") - def _(x): - return torch.empty_like(x) - ``` - -6. **Wrap it in an entry**, `ops//my_op.py`, which calls - `native_ops.my_op(...)`. Test the entry like any other catalog entry, under - `tests/unittest/_torch/modeling_v2/models/foo//`. - -7. **List the tests.** - - The family's unit tests go in a test-db entry of its own: - `unittest/_torch/modeling_v2/models/foo`. - - Its accuracy gates go in `accuracy/modeling_v2/test_foo.py`. - - CBTS selects a family's tests by these two names (see [CI](#ci)). - -No stock file changes along the way. - -### Code the family's targets share - -When several targets of a family call the same op, its code goes in the -family's `shared/` once, and its ops are in the family's torch namespace, -`torch.ops.mv2_`: - -```cmake -add_modeling_v2_family(foo) - -add_modeling_v2_shared( - foo - ARCHS 100f - SOURCES kernels/*.cu ops/*.cpp - REQUIRES MULTI_DEVICE) - -# Targets with no code of their own declare no SOURCES and have no directory; -# declaring them is what makes the family library carry them. -add_modeling_v2_target(foo foo_nvfp4__sm_100__tp8 ARCHS 100f) -add_modeling_v2_target(foo foo_nvfp4__sm_100__tp16 ARCHS 100f) -``` - -- **Fakes** of the shared ops go in the family's `models/foo/fake_ops.py`; the - loader imports it right after it opens the library. Use - `torch_namespace(family_of(__name__))` for the namespace. -- **Callers** anywhere in the family reach the ops through - `load_family_ops(__name__).my_op(...)`; after the first call it is a - dictionary lookup. -- **Target packages** still start with `load_target_ops(__name__)`, which is - where routing finds out whether this build carries the target. -- **A target without code of its own** is skipped whenever the shared code is - (an unmet `REQUIRES`, or no matching architecture), so the library never - carries a target whose ops it lacks. - -## CMake API - -The functions are defined in `cpp/cmake/modules/modeling_v2.cmake`, in the style -of `add_th_common_family`. - -| Function | What it declares | -|---|---| -| `add_modeling_v2_family()` | The family; call it first in the family's `CMakeLists.txt`. | -| `add_modeling_v2_target( ARCHS ... [SOURCES ...] [DEPS ...] [REQUIRES ...])` | One target's code, in `/`; without `SOURCES`, a target that runs only its family's shared code. | -| `add_modeling_v2_shared( ARCHS ... SOURCES ... [DEPS ...] [REQUIRES ...])` | Code several targets of the family call, in `shared/`. Its ops are in `torch.ops.mv2_`. | - -| Keyword | Meaning | -|---|---| -| `ARCHS` | `NN`, `NNa` or `NNf`, intersected with the build's requested architectures: `NN`/`NNa` build when `NN` is requested; `NNf` builds when any member of family `NN` is. This is how flash_mla and deep_ep choose their architectures. A target whose intersection is empty is **not built**: no placeholder kernels, and the configure step prints why. The declared architectures must cover the SM in the target's name; when moving existing code, keep the architectures it was gated with. | -| `SOURCES` | Paths or globs relative to the unit's directory. `.cu` files are compiled for `ARCHS` without torch; `.cpp` files are the torch bindings. A per-file option goes on the file itself, in the family's `CMakeLists.txt`: `set_source_files_properties(shared/kernels/x.cu PROPERTIES COMPILE_OPTIONS --use_fast_math)`. | -| `DEPS` | `cublas`, `cublaslt`, `cuda_driver`, `internal_cutlass`, `nccl`, `trtllm_gen_fmha_interface`. Ask for anything else by extending the module. | -| `REQUIRES` | `MULTI_DEVICE`, `NVSHMEM`. A target whose requirement the build lacks (e.g. a single-device build) is skipped and recorded. | - -### Build options - -- **`-DBUILD_MODELING_V2=OFF`** builds no family library, so every target with - native code is a `NoTarget`. -- **`-DTRTLLM_MODELING_V2_TARGETS=/;...`** builds only the - matching targets. Globs are allowed, e.g. `fixture/*`; the default is - `all`. Pass it through - `scripts/build_wheel.py --extra-cmake-vars TRTLLM_MODELING_V2_TARGETS=...`. -- **Incremental builds.** A family library is its own build target, so an - edit relinks only that library: - - ```bash - cmake --build cpp/build --target th_modeling_v2_ - ``` - -## Namespaces - -| Kind | Name | +| Path | Role | |---|---| -| Target torch ops | `torch.ops.mv2__`, with `__` collapsed to `_` | -| Family shared torch ops | `torch.ops.mv2_`; also holds `built_targets()` | -| Target C++ | `tensorrt_llm::modeling_v2::::` (`TRTLLM_MV2_NAMESPACE_BEGIN`) | -| Shared C++ | `tensorrt_llm::modeling_v2::::shared` | - -**Each torch namespace is claimed exactly once.** The family library's -generated registration unit claims it with `TORCH_LIBRARY`, so a second -library claiming the same namespace fails at load. Op files only add fragments, -through the `TRTLLM_MV2_*` macros; raw `TORCH_LIBRARY*` is rejected by the -claims test. - -**Sibling targets can share file and function names.** Every target has its -own C++ namespace, and the family library exports no symbol -(`common/familyLibrary.version`). - -## Sharing code - -**Sharing moves code up, never sideways:** - -1. target -2. family `shared/` -3. stock `cpp/tensorrt_llm/kernels` + `thop`, once a built-in model calls it - (normal stock review) - -A target never includes or calls a sibling target's code. When two targets -need the same kernel, move it to `shared/`. A cross-family `catalog/` is -reserved and not built yet. - -## What native code may include - -- Its own target's files. -- Its family's `shared/`. -- `common/`. -- The stock headers listed in `contractSurface.txt`, such as - `tensorrt_llm/common/*` and selected kernel device helpers like - `kernels/quantization.cuh`. -- System, CUDA, torch and cutlass headers. - -It links against `libtensorrt_llm.so` (logger, `TllmException`, mcast -registry and other singletons stay there), `th_utils` and torch. - -All family libraries come from the same build as the stock libraries, as -`flash_mla`, `deep_gemm` and `deep_ep` do; a precompiled wheel used with a -checkout is checked by `setup.py`'s native-input skew detection, which covers -all of `cpp/`. Adding a header to `contractSurface.txt` widens what stock -changes can affect targets; the header's stock owners review it. - -**When a target needs stock to change**, for example a new option on a stock -kernel, that change is a separate stock PR. It must leave stock behavior -unchanged by default, and it is reviewed by the stock owners. - -## How it loads - -`_native.load_target_ops(__name__)` treats a family library the way the stock -code treats its optional native modules (`flash_mla`, `deep_ep`): - -1. Open `tensorrt_llm/libs/libth_modeling_v2_.so`, once per process. -2. Ask it which targets it carries: `torch.ops.mv2_.built_targets()`, - a query op like `th_common`'s `mhc_fused_hc_mma_enabled()`. -3. Import the target's `fake_ops.py`. - -| Outcome | Raised as | Meaning | -|---|---|---| -| The library is not installed, or does not carry the target | `NoTarget` | This build did not compile it; `auto` keeps the built-in model, `require` reports why | -| The library is installed but fails to open, or an op `fake_ops.py` declares is missing | `NativeLoadError` | A broken installation; never a fallback | - -`route` imports a target only on the branch that returns it, so this runs as -part of routing, and routing has already checked the device's compute -capability. Every rank then agrees on the result -(`modeling_v2_resolve_on_all_ranks`), or every rank fails. - -**Packaging.** Like deep_ep and flash_mla, each family writes -`cuda_architectures.txt` into its build directory -- the architectures its -library was built for, empty when it has none -- and `scripts/build_wheel.py` -installs the library only when that list is not empty, asserting it exists. - -## CI - -**CBTS rule.** `jenkins/scripts/cbts/cbts/rules/modeling_v2.py` claims both -trees. What it selects depends on what the PR changes: - -| PR changes only… | CI runs… | -|---|---| -| One family (`models//` and/or `cpp/tensorrt_llm/modeling_v2//`) | That family's test-db entries plus the modeling_v2 CPU checks | -| Shared modeling_v2 code (catalog, loader, resolver, `common/`, the CMake module) | Every modeling_v2 entry | -| Stock code | The full suite | - -**Why the narrowing is safe.** It relies on these CPU checks, which run in -every one of those scopes: - -- `test_modeling_v2_native_claims.py`: - - stock never references a target; - - targets reach only themselves, their family's `shared/`, `common/` and - the contract surface; - - every op has a fake; - - `modeling_v2` defaults to `off`; - - families keep their own test-db entries. -- `test_modeling_v2_native_loader.py`: the loader's decisions, and a load - smoke test of every family library the wheel ships. -- `test_modeling_v2_rank_agreement.py`: all ranks resolve to the same target, - or all of them fail. - -**Pre-merge entries.** Every pre-merge modeling_v2 entry also runs on every -stock PR that falls back to the full suite. Keep a family's pre-merge entry -to one short smoke test on hardware matching its SM, and put the accuracy -matrix post-merge. +| `CMakeLists.txt` | Finds every `models//csrc/CMakeLists.txt` and builds the family libraries (`cpp/cmake/modules/modeling_v2.cmake`). | +| `common/namespace.h`, `common/torchLibrary.h` | The per-unit C++ and torch namespace macros native code registers through. | +| `common/familyLibrary.version` | The version script: a family library exports no symbol. | +| `contractSurface.txt` | The stock headers native code may include. Adding one is reviewed by the header's stock owners. | diff --git a/cpp/tensorrt_llm/modeling_v2/fixture/scaleAddSm10xDefault/kernels/scaleAdd.cu b/cpp/tensorrt_llm/modeling_v2/fixture/scaleAddSm10xDefault/kernels/scaleAdd.cu deleted file mode 100644 index 360cd682753a..000000000000 --- a/cpp/tensorrt_llm/modeling_v2/fixture/scaleAddSm10xDefault/kernels/scaleAdd.cu +++ /dev/null @@ -1,97 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "tensorrt_llm/modeling_v2/fixture/scaleAddSm10xDefault/kernels/scaleAdd.h" - -#include -#include - -TRTLLM_MV2_NAMESPACE_BEGIN - -namespace kernels -{ - -namespace -{ - -constexpr int kThreads = 256; -constexpr int64_t kMaxBlocks = 65535; - -__device__ __forceinline__ float toFloat(float value) -{ - return value; -} - -__device__ __forceinline__ float toFloat(__nv_bfloat16 value) -{ - return __bfloat162float(value); -} - -template -__device__ __forceinline__ T fromFloat(float value); - -template <> -__device__ __forceinline__ float fromFloat(float value) -{ - return value; -} - -template <> -__device__ __forceinline__ __nv_bfloat16 fromFloat<__nv_bfloat16>(float value) -{ - return __float2bfloat16_rn(value); -} - -template -__global__ void scaleAddKernel(T const* x, T* out, int64_t numel, float alpha, float beta) -{ - int64_t const stride = static_cast(gridDim.x) * blockDim.x; - for (int64_t i = static_cast(blockIdx.x) * blockDim.x + threadIdx.x; i < numel; i += stride) - { - out[i] = fromFloat(__fadd_rn(__fmul_rn(toFloat(x[i]), alpha), beta)); - } -} - -template -cudaError_t launchScaleAdd(void const* x, void* out, int64_t numel, float alpha, float beta, cudaStream_t stream) -{ - int64_t const blocks = std::min((numel + kThreads - 1) / kThreads, kMaxBlocks); - scaleAddKernel<<(blocks), kThreads, 0, stream>>>( - static_cast(x), static_cast(out), numel, alpha, beta); - return cudaGetLastError(); -} - -} // namespace - -cudaError_t invokeScaleAdd( - void const* x, void* out, int64_t numel, ScaleAddDtype dtype, float alpha, float beta, cudaStream_t stream) -{ - if (numel == 0) - { - return cudaSuccess; - } - switch (dtype) - { - case ScaleAddDtype::kFloat32: return launchScaleAdd(x, out, numel, alpha, beta, stream); - case ScaleAddDtype::kBfloat16: return launchScaleAdd<__nv_bfloat16>(x, out, numel, alpha, beta, stream); - } - return cudaErrorInvalidValue; -} - -} // namespace kernels - -TRTLLM_MV2_NAMESPACE_END diff --git a/cpp/tensorrt_llm/modeling_v2/fixture/scaleAddSm10xDefault/ops/scaleAddOp.cpp b/cpp/tensorrt_llm/modeling_v2/fixture/scaleAddSm10xDefault/ops/scaleAddOp.cpp deleted file mode 100644 index e4a6bc553e99..000000000000 --- a/cpp/tensorrt_llm/modeling_v2/fixture/scaleAddSm10xDefault/ops/scaleAddOp.cpp +++ /dev/null @@ -1,61 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "tensorrt_llm/common/assert.h" -#include "tensorrt_llm/modeling_v2/common/namespace.h" -#include "tensorrt_llm/modeling_v2/common/torchLibrary.h" -#include "tensorrt_llm/modeling_v2/fixture/scaleAddSm10xDefault/kernels/scaleAdd.h" - -#include -#include -#include - -TRTLLM_MV2_NAMESPACE_BEGIN - -namespace -{ - -at::Tensor scaleAdd(at::Tensor const& x, double alpha, double beta) -{ - TORCH_CHECK(x.is_cuda() && x.is_contiguous(), "scale_add: x must be a contiguous CUDA tensor"); - TORCH_CHECK(x.scalar_type() == at::kFloat || x.scalar_type() == at::kBFloat16, - "scale_add: x must be float32 or bfloat16, got ", x.scalar_type()); - c10::cuda::CUDAGuard const deviceGuard{x.device()}; - at::Tensor out = at::empty_like(x); - auto const dtype - = x.scalar_type() == at::kFloat ? kernels::ScaleAddDtype::kFloat32 : kernels::ScaleAddDtype::kBfloat16; - cudaError_t const status = kernels::invokeScaleAdd(x.data_ptr(), out.data_ptr(), x.numel(), dtype, - static_cast(alpha), static_cast(beta), at::cuda::getCurrentCUDAStream(x.get_device())); - // TLLM_CHECK_WITH_INFO throws a TllmException, whose code lives in libtensorrt_llm.so: the fixture keeps the - // dynamic link to the stock library exercised. - TLLM_CHECK_WITH_INFO(status == cudaSuccess, "scale_add launch failed: %s", cudaGetErrorString(status)); - return out; -} - -} // namespace - -TRTLLM_MV2_NAMESPACE_END - -TRTLLM_MV2_LIBRARY_FRAGMENT(m) -{ - m.def("scale_add(Tensor x, float alpha, float beta) -> Tensor"); -} - -TRTLLM_MV2_LIBRARY_IMPL(CUDA, m) -{ - m.impl("scale_add", &TRTLLM_MV2_NS::scaleAdd); -} diff --git a/jenkins/scripts/cbts/README.md b/jenkins/scripts/cbts/README.md index 138ab4589b6d..94e30eb33dc1 100644 --- a/jenkins/scripts/cbts/README.md +++ b/jenkins/scripts/cbts/README.md @@ -67,7 +67,7 @@ See `cbts/rules/README.md` for per-rule logic. | `testlistonly` | `TestListRule` fired solo: PR only adds entries under `tests/integration/test_lists/test-db/*.yml`. | | `visualgenonly` | `VisualGenRule` fired solo: PR only touches VisualGen internal source paths (`examples/visual_gen/**`, `scripts/visualgen_eval/**`, `tensorrt_llm/_torch/visual_gen/**`; excl. docs; image fixtures like `cat_piano.png` are claimed). Narrows to blocks containing VG test entries. Outward-facing files under `tensorrt_llm/visual_gen/**` and `tensorrt_llm/media/**` (eagerly imported by `trtllm-serve`) force `null` fallback. | | `specdeconly` | `SpecDecRule` fired solo: PR only touches speculative-decoding source paths (`tensorrt_llm/_torch/speculative/**`, `tensorrt_llm/models/{eagle,medusa,redrafter}/**`, `examples/{eagle,medusa,redrafter,draft_target_model,ngram}/**`, `examples/llm-api/llm_speculative_decoding.py`; excl. docs). Narrows to blocks containing spec-dec test entries (eagle / medusa / redrafter / ngram / draft-target-model / MTP). | -| `modelingv2only` | `ModelingV2Rule` fired solo: PR only touches modeling_v2 code -- the Python subtree (`tensorrt_llm/_torch/_experimental/modeling_v2/**`), its native code (`cpp/tensorrt_llm/modeling_v2/**`, one torch-op library per model family) or the native build module (`cpp/cmake/modules/modeling_v2.cmake`); excl. docs, which are a fifth of the Python subtree — every catalog entry carries a contract document. A change confined to one family's directories narrows to that family's entries (`accuracy/modeling_v2/test_.py`, `unittest/_torch/modeling_v2/models/`) plus the modeling_v2 CPU checks; any other modeling_v2 change narrows to every modeling_v2 entry. No outward-facing fallback is needed: nothing beyond the resolver module is imported, and no family library is loaded, unless `LLM(modeling_v2=...)` is on; the CPU checks that keep this true run in every scope. The callers outside the subtree (`_torch/pyexecutor/model_loader.py`, `_torch/models/modeling_auto.py`) are left unclaimed, so touching them falls back to baseline. | +| `modelingv2only` | `ModelingV2Rule` fired solo: PR only touches modeling_v2 code -- the subtree (`tensorrt_llm/_torch/_experimental/modeling_v2/**`, each family's native code in its `csrc/` included: one torch-op library per model family) or what the family libraries share (`cpp/tensorrt_llm/modeling_v2/**`, `cpp/cmake/modules/modeling_v2.cmake`); excl. docs, which are a fifth of the Python subtree — every catalog entry carries a contract document. A change confined to one family's directory narrows to that family's entries (`accuracy/modeling_v2/test_.py`, `unittest/_torch/modeling_v2/models/`) plus the modeling_v2 CPU checks; any other modeling_v2 change narrows to every modeling_v2 entry. No outward-facing fallback is needed: nothing beyond the resolver module is imported, and no family library is loaded, unless `LLM(modeling_v2=...)` is on; the CPU checks that keep this true run in every scope. The callers outside the subtree (`_torch/pyexecutor/model_loader.py`, `_torch/models/modeling_auto.py`) are left unclaimed, so touching them falls back to baseline. | | `agentflowonly` | `AgentFlowRule` fired solo: PR only touches `agent-flow/**` source or test files (excl. docs). Runs `CPU-AgentFlow-UnitTest`. | | `openengineonly` | `OpenEngineRule` fired solo: PR only touches `tensorrt_llm/grpc/openengine/**` source files (excl. docs). Narrows to the registered OpenEngine unit tests: the stub-based ones on the always-run `CPU-Generic-*` stages, plus `test_capability_conformance.py` on `A10-PyTorch-*`, which needs a GPU. | | `docsonly` | `DocsRule` fired solo: all documentation changes run `CPU-Build_Docs`; changes under `docs/**` also run the complete `l0_cpu` suite on matching `CPU-Generic-*` stages. Docs-only orchestration skips the SBSA track, so its resolved ARM stage is not launched. | diff --git a/jenkins/scripts/cbts/cbts/command/main.py b/jenkins/scripts/cbts/cbts/command/main.py index 2c1b045a4c23..17ca3e11079c 100644 --- a/jenkins/scripts/cbts/cbts/command/main.py +++ b/jenkins/scripts/cbts/cbts/command/main.py @@ -101,7 +101,7 @@ def build_rules( TestListRule(yaml_index, stages, repo_root=repo_root), VisualGenRule(yaml_index, stages), SpecDecRule(yaml_index, stages), - ModelingV2Rule(yaml_index, stages, repo_root=repo_root), + ModelingV2Rule(yaml_index, stages), AgentFlowRule(yaml_index, stages), OpenEngineRule(yaml_index, stages), DocsRule(yaml_index, stages), diff --git a/jenkins/scripts/cbts/cbts/rules/README.md b/jenkins/scripts/cbts/cbts/rules/README.md index 270aa6977537..0b4cff97d632 100644 --- a/jenkins/scripts/cbts/cbts/rules/README.md +++ b/jenkins/scripts/cbts/cbts/rules/README.md @@ -303,15 +303,14 @@ Outcomes: Path-only rule. Claims non-documentation source changes under `tensorrt_llm/_torch/_experimental/modeling_v2/`, the self-contained second modeling path (one flat forward per checkpoint/arch/parallel triple, assembled -from a catalog of op wrappers), under `cpp/tensorrt_llm/modeling_v2/`, its -native code (one torch-op library per model family, loaded only when routing picks -one of the family's targets), and the native build module -`cpp/cmake/modules/modeling_v2.cmake`. +from a catalog of op wrappers), including each family's native code +(`models//csrc/`, `models///csrc/`: one torch-op library +per model family, loaded only when routing picks one of the family's targets), +plus what every family library shares: `cpp/tensorrt_llm/modeling_v2/` and the +native build module `cpp/cmake/modules/modeling_v2.cmake`. Scope — per family, else all of modeling_v2: a change confined to one -family's directories (`models//` in Python, -`cpp/tensorrt_llm/modeling_v2//` in C++, the family named by the -Python package whose camelCase form the C++ directory is) keeps that family's +family's directory (`models//`, its native code included) keeps that family's entries -- `accuracy/modeling_v2/test_.py` and `unittest/_torch/modeling_v2/models/` -- plus the modeling_v2 CPU checks (`unittest/_torch/modeling_v2/test_modeling_v2_*.py`). Several diff --git a/jenkins/scripts/cbts/cbts/rules/modeling_v2.py b/jenkins/scripts/cbts/cbts/rules/modeling_v2.py index e97394a3088a..8f5e6dbe7f44 100644 --- a/jenkins/scripts/cbts/cbts/rules/modeling_v2.py +++ b/jenkins/scripts/cbts/cbts/rules/modeling_v2.py @@ -15,18 +15,20 @@ modeling_v2 is a second modeling path: one self-contained forward per (checkpoint, GPU arch, parallel topology), assembled from a catalog of op -wrappers. Its Python lives under `tensorrt_llm/_torch/_experimental/modeling_v2/` -and its native code under `cpp/tensorrt_llm/modeling_v2/`, which builds one -torch-op library per model family, loaded only when routing picks one of the -family's targets. +wrappers. It lives under `tensorrt_llm/_torch/_experimental/modeling_v2/`, a +family's native code included (`models//csrc/`, +`models///csrc/`), which builds one torch-op library per model +family, loaded only when routing picks one of the family's targets. What every +family library shares is in `cpp/tensorrt_llm/modeling_v2/` and +`cpp/cmake/modules/modeling_v2.cmake`. Scope — per family, else all of modeling_v2: -A change confined to one family's directories (`models//` in Python, -`cpp/tensorrt_llm/modeling_v2//` in C++) selects that family's test +A change confined to one family's directory (`models//`, its native +code included) selects that family's test entries -- `accuracy/modeling_v2/test_.py` and `unittest/_torch/modeling_v2/models//` -- plus the modeling_v2 CPU checks. Any other modeling_v2 change (catalog, resolver, loader, the native -build module and its common headers) selects every modeling_v2 entry. Several +build module, its discovery and common headers) selects every modeling_v2 entry. Several families in one PR union their entries. Why no outward fallback is needed: nothing beyond the resolver is imported, and @@ -56,7 +58,6 @@ from __future__ import annotations import re -from pathlib import Path from typing import Optional from cbts.blocks import Stage, YAMLIndex, _entry_target, _target_in_filter_subtree @@ -67,9 +68,8 @@ _PY_ROOT = "tensorrt_llm/_torch/_experimental/modeling_v2/" _PY_MODELS = _PY_ROOT + "models/" +# What every family library shares: discovery, common/ macros, contractSurface.txt. _CPP_ROOT = "cpp/tensorrt_llm/modeling_v2/" -# Directories of the native tree that are not families. -_CPP_NON_FAMILY_DIRS = frozenset({"common", "catalog"}) # The native build module: shared by every family library. _CPP_BUILD_FILES = frozenset({"cpp/cmake/modules/modeling_v2.cmake"}) @@ -96,20 +96,6 @@ _MV2_CHECK_ENTRY = re.compile(r"^unittest/_torch/modeling_v2/test_modeling_v2_[\w]+\.py$") -def _camel(name: str) -> str: - """The C++ directory of a family: `minimax_m3` -> `minimaxM3`. - - Mirrors `_modeling_v2_camel` in cpp/cmake/modules/modeling_v2.cmake. - """ - parts = [p for p in name.split("_") if p] - return parts[0] + "".join(p[:1].upper() + p[1:] for p in parts[1:]) - - -def _snake(camel_name: str) -> str: - """Best-effort inverse of `_camel`, used only without a repo root.""" - return re.sub(r"(?<=[a-z0-9])([A-Z])", lambda m: "_" + m.group(1).lower(), camel_name) - - def _is_mv2_claim(path: str) -> bool: """Decide whether ModelingV2Rule claims `path`. @@ -157,33 +143,16 @@ class ModelingV2Rule(Rule): name = "modelingv2" needs_diff_for: tuple[str, ...] = () - def __init__( - self, - yaml_index: YAMLIndex, - stages: dict[str, Stage], - repo_root: Optional[Path] = None, - ) -> None: + def __init__(self, yaml_index: YAMLIndex, stages: dict[str, Stage]) -> None: self.yaml_index = yaml_index self._stages_by_yaml = stages_by_yaml_stem(stages) - # camelCase native directory -> family, from the Python family packages. - self._family_of_cpp_dir: dict[str, str] = {} - if repo_root is not None: - models = Path(repo_root) / _PY_MODELS - if models.is_dir(): - for family_dir in models.iterdir(): - if family_dir.is_dir() and (family_dir / "__init__.py").is_file(): - self._family_of_cpp_dir[_camel(family_dir.name)] = family_dir.name - - def family_of(self, path: str) -> Optional[str]: + + @staticmethod + def family_of(path: str) -> Optional[str]: """The family a claimed path belongs to, or None for shared modeling_v2 code.""" if path.startswith(_PY_MODELS): parts = path[len(_PY_MODELS) :].split("/") return parts[0] if len(parts) > 1 else None - if path.startswith(_CPP_ROOT): - parts = path[len(_CPP_ROOT) :].split("/") - if len(parts) < 2 or parts[0] in _CPP_NON_FAMILY_DIRS: - return None - return self._family_of_cpp_dir.get(parts[0], _snake(parts[0])) return None @staticmethod diff --git a/setup.py b/setup.py index 4d3a518bce79..2b49833f7e47 100644 --- a/setup.py +++ b/setup.py @@ -270,8 +270,13 @@ def warn_on_build_skew(precompiled_location: str) -> None: import subprocess NATIVE_BUILD_INPUTS = [ - "cpp/", "3rdparty/", "setup.py", "scripts/build_wheel.py", - "requirements.txt" + "cpp/", + "3rdparty/", + "setup.py", + "scripts/build_wheel.py", + "requirements.txt", + # modeling_v2 families keep their native code with their Python code. + ":(glob)tensorrt_llm/_torch/_experimental/modeling_v2/models/**/csrc/**" ] def head_of(checkout: str) -> str | None: diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/NATIVE.md b/tensorrt_llm/_torch/_experimental/modeling_v2/NATIVE.md new file mode 100644 index 000000000000..91d8c7d8e290 --- /dev/null +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/NATIVE.md @@ -0,0 +1,248 @@ + + + +# Native code of modeling_v2 targets + +C++/CUDA that only modeling_v2 targets use lives with the Python code of its +model family, in this directory tree, not in `cpp/tensorrt_llm/kernels/` or +`thop/`. Each family builds into one torch-op library, +`libth_modeling_v2_.so`, installed next to `libth_common.so`. + +- **Not linked into stock.** Nothing in a family library is linked into + `libtensorrt_llm.so` or `libth_common.so`. +- **Loaded on demand.** The library opens only when routing picks one of the + family's targets. With `modeling_v2=off` (the default), nothing loads. +- **No stock file per model.** Adding a model, its kernels included, edits + nothing under `cpp/`, nor `setup.py` or `scripts/build_wheel.py`: the build + finds every `models//csrc/CMakeLists.txt` by itself. + +`models/gpt_oss/` is the worked example: its target +`gpt_oss_120b__sm_10x__default` carries one kernel, the MXFP4 block-scale +swizzle its weight loader runs. + +## Onboarding a model with native code + +The files a new family `foo` with one target adds. Everything except the last +three rows is inside its own directories: + +``` +tensorrt_llm/_torch/_experimental/modeling_v2/models/foo/ + __init__.py docstring only + routing.py @routes("FooForCausalLM") def route(...) + fake_ops.py fakes of the family's shared ops, if it has any + csrc/ + CMakeLists.txt add_modeling_v2_family / _shared / _target + *.cu *.cuh *.h *.cpp code several targets of the family share, if any + foo_nvfp4__sm_103__tp4/ ____ + __init__.py native_ops = load_target_ops(__name__) + modeling.py weights.py + fake_ops.py a fake for every op of csrc/ + ops//.py entries only this target calls + csrc/ + *.cu *.cuh *.h kernels, compiled for ARCHS without torch + *Op.cpp torch bindings +tests/unittest/_torch/modeling_v2/models/foo/foo_nvfp4__sm_103__tp4/test_modeling_v2_*.py +tests/integration/defs/accuracy/modeling_v2/test_foo.py + +models/__init__.py + from .foo import routing as foo_routing +tests/integration/test_lists/test-db/l0_.yml + + unittest/_torch/modeling_v2/models/foo +.github/CODEOWNERS optional: the family's owners +``` + +Naming is the Python package names throughout: family `foo`, target +`foo_nvfp4__sm_103__tp4`, its torch namespace `mv2_foo_foo_nvfp4_sm_103_tp4`. +`tests/unittest/_torch/modeling_v2/test_modeling_v2_native_claims.py` checks +every rule below on CPU; run it before sending the PR. + +## Step by step + +The gpt_oss files are the example for each step. + +1. **Declare the family and the target** in `models/foo/csrc/CMakeLists.txt` + (`models/gpt_oss/csrc/CMakeLists.txt`): + + ```cmake + add_modeling_v2_family(foo) + + add_modeling_v2_target( + foo foo_nvfp4__sm_103__tp4 + ARCHS 103a + SOURCES *.cu *.cpp + REQUIRES MULTI_DEVICE) + ``` + +2. **Write the kernel** in `foo_nvfp4__sm_103__tp4/csrc/`, in the target's C++ + namespace (`gpt_oss_120b__sm_10x__default/csrc/swizzleBlockScales.{h,cu}`): + + ```cpp + #include "tensorrt_llm/modeling_v2/common/namespace.h" + + TRTLLM_MV2_NAMESPACE_BEGIN // tensorrt_llm::modeling_v2::foo::foo_nvfp4_sm_103_tp4 + namespace kernels + { + cudaError_t invokeMyOp(/* ... */, cudaStream_t stream); + } + TRTLLM_MV2_NAMESPACE_END + ``` + +3. **Bind it to torch** in the same `csrc/`, through the target's torch + namespace (`swizzleBlockScalesOp.cpp`). Include the family's own headers + from `models/`: + + ```cpp + #include "foo/foo_nvfp4__sm_103__tp4/csrc/myOp.h" + #include "tensorrt_llm/modeling_v2/common/torchLibrary.h" + + TRTLLM_MV2_LIBRARY_FRAGMENT(m) { m.def("my_op(Tensor x) -> Tensor"); } + TRTLLM_MV2_LIBRARY_IMPL(CUDA, m) { m.impl("my_op", &TRTLLM_MV2_NS::myOp); } + ``` + +4. **Load it** from the target package's `__init__.py`: + + ```python + from tensorrt_llm._torch._experimental.modeling_v2._native import load_target_ops + + native_ops = load_target_ops(__name__) # torch.ops.mv2_foo_foo_nvfp4_sm_103_tp4 + ``` + + The target's modules use it as `from . import native_ops` + (`gpt_oss_120b__sm_10x__default/weights.py`). The name is not `ops`: that + is the target's `ops/` package. + +5. **Fake every op** in the target's `fake_ops.py`; the loader imports it right + after it finds the target in the library: + + ```python + from tensorrt_llm._torch._experimental.modeling_v2._native import namespace_of + + _NS = namespace_of(__package__) + + @torch.library.register_fake(f"{_NS}::my_op") + def _(x): + return torch.empty_like(x) + ``` + +6. **Test it** under `tests/unittest/_torch/modeling_v2/models/foo/`, and list + the family's directory in the test list of the GPU its SM names; accuracy + gates go in `accuracy/modeling_v2/test_foo.py` + (`test_modeling_v2_gpt_oss_swizzle_block_scales.py`, and + `unittest/_torch/modeling_v2/models/gpt_oss` in `l0_b300.yml`). CBTS + selects a family's tests by these two names (see [CI](#ci)). + +## Code the family's targets share + +When several targets call the same op, its code goes once in the family's +`csrc/`, next to its `CMakeLists.txt`, and its ops are in the family's torch +namespace, `torch.ops.mv2_`: + +```cmake +add_modeling_v2_shared( + foo + ARCHS 100f + SOURCES *.cu *.cpp + REQUIRES MULTI_DEVICE) + +# Targets with no code of their own declare no SOURCES and have no csrc/; +# declaring them is what makes the family library carry them. +add_modeling_v2_target(foo foo_nvfp4__sm_100__tp8 ARCHS 100f) +``` + +- **Fakes** of the shared ops go in the family's `models/foo/fake_ops.py`, + with `torch_namespace(family_of(__name__))`; the loader imports it when it + opens the library. +- **Callers** anywhere in the family use + `load_family_ops(__name__).my_op(...)`. +- **A target without code of its own** is skipped whenever the shared code is, + so the library never carries a target whose ops it lacks. + +Sharing moves code up, never sideways: target `csrc/`, then the family's +`csrc/`, then stock (`cpp/tensorrt_llm/kernels` + `thop`) once a built-in +model calls it, through normal stock review. A target never includes or calls +a sibling target's code. + +## CMake API + +Defined in `cpp/cmake/modules/modeling_v2.cmake`, in the style of +`add_th_common_family`. + +| Function | What it declares | +|---|---| +| `add_modeling_v2_family()` | The family; first call in `models//csrc/CMakeLists.txt`. | +| `add_modeling_v2_target( ARCHS ... [SOURCES ...] [DEPS ...] [REQUIRES ...])` | One target's code, in `/csrc/`; without `SOURCES`, a target that runs only its family's shared code. | +| `add_modeling_v2_shared( ARCHS ... SOURCES ... [DEPS ...] [REQUIRES ...])` | Code several targets call, in the family's `csrc/`; its ops are in `torch.ops.mv2_`. | + +| Keyword | Meaning | +|---|---| +| `ARCHS` | `NN`, `NNa` or `NNf`, intersected with the build's requested architectures, as flash_mla and deep_ep choose theirs. A target whose intersection is empty is **not built** (the configure step prints why) and is a `NoTarget` at run time. The architectures must cover the SM in the target's name. | +| `SOURCES` | Paths or globs relative to the unit's `csrc/`. `.cu` files compile for `ARCHS` without torch; `.cpp` files are the torch bindings. A per-file option goes on the file in the family's `CMakeLists.txt`: `set_source_files_properties(x.cu PROPERTIES COMPILE_OPTIONS --use_fast_math)`. | +| `DEPS` | `cublas`, `cublaslt`, `cuda_driver`, `internal_cutlass`, `nccl`, `trtllm_gen_fmha_interface`. Anything else is an addition to the module. | +| `REQUIRES` | `MULTI_DEVICE`, `NVSHMEM`. A target the build cannot satisfy is skipped. | + +**Build options.** `-DBUILD_MODELING_V2=OFF` builds no family library. +`-DTRTLLM_MODELING_V2_TARGETS=/;...` (globs allowed, e.g. +`gpt_oss/*`) builds only those; pass it with +`scripts/build_wheel.py -D "TRTLLM_MODELING_V2_TARGETS=..."`. A family library +is its own build target, so a kernel edit relinks only it: +`cmake --build cpp/build --target th_modeling_v2_`. + +## Namespaces + +| Kind | Name | +|---|---| +| Target torch ops | `torch.ops.mv2__`, with `__` collapsed to `_` | +| Family shared torch ops | `torch.ops.mv2_`; also holds `built_targets()` | +| Target C++ | `tensorrt_llm::modeling_v2::::` (`TRTLLM_MV2_NAMESPACE_BEGIN`) | +| Shared C++ | `tensorrt_llm::modeling_v2::::shared` | + +Each torch namespace is claimed exactly once, by the family library's generated +registration, so a second library claiming it fails at load. Op files only add +fragments through the `TRTLLM_MV2_*` macros; raw `TORCH_LIBRARY*` is rejected. +Every target has its own C++ namespace and the library exports no symbol, so +sibling targets can share file and function names. + +## What native code may include + +- Its own target's `csrc/` and its family's `csrc/`, written from `models/`: + `"foo/foo_nvfp4__sm_103__tp4/csrc/x.h"`, `"foo/csrc/y.h"`. +- `tensorrt_llm/modeling_v2/common/` (the macros). +- The stock headers listed in `cpp/tensorrt_llm/modeling_v2/contractSurface.txt`. + Adding one widens what stock changes can affect targets, so the header's stock + owners review it. +- System, CUDA, torch and cutlass headers. + +It links against `libtensorrt_llm.so` (logger, `TllmException`, mcast registry +and other singletons stay there), `th_utils` and torch. **When a target needs +stock to change**, e.g. a new option on a stock kernel, that is a separate +stock PR that leaves stock behavior unchanged by default. + +## How it loads + +`load_target_ops(__name__)` treats a family library the way stock code treats +its optional native modules (`flash_mla`, `deep_ep`): + +| Outcome | Raised as | Meaning | +|---|---|---| +| The library is not installed, or does not carry the target | `NoTarget` | This build did not compile it; `auto` keeps the built-in model, `require` reports why | +| The library is installed but fails to open, or an op a `fake_ops.py` declares is missing | `NativeLoadError` | A broken installation; never a fallback | + +Packaging follows `flash_mla` and `deep_ep` too: each family writes +`cuda_architectures.txt` into its build directory, and `scripts/build_wheel.py` +installs the library only when that list is not empty. + +## CI + +| PR changes only… | CI runs… | +|---|---| +| One family (`models//`, its `csrc/` included) | That family's test-db entries plus the modeling_v2 CPU checks | +| Shared modeling_v2 code (catalog, loader, resolver, `models/__init__.py`, `cpp/tensorrt_llm/modeling_v2/`, the CMake module) | Every modeling_v2 entry | +| Stock code | The full suite | + +The narrowing is safe because family libraries are never in the stock +libraries, and the CPU checks that keep it so run in every scope: +`test_modeling_v2_native_claims.py` (native code only in `csrc/`, stock never +names a family, includes stay within the target, its family, `common/` and the +contract surface, every op has a fake, license headers), +`test_modeling_v2_native_loader.py` and `test_modeling_v2_rank_agreement.py`. +Keep a family's pre-merge entry to one short test on hardware matching its SM; +the accuracy matrix goes post-merge. diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/README.md b/tensorrt_llm/_torch/_experimental/modeling_v2/README.md index 9b896406d3fb..8aff69b93dd8 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/README.md +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/README.md @@ -112,33 +112,33 @@ checkpoint of the same shape is ungated. ``` _router_index.py architectures[0] -> routing module. Small, stable, test-guarded. _native.py loads a target's native code (see "Native code" below) +NATIVE.md the guide to a target's native (C++/CUDA) code models// routing.py one forward-reading decision tree per architecture family fake_ops.py fakes of the family's shared native ops, if it has any + csrc/ the family's CMakeLists.txt and the native code its targets share ____/ modeling.py weights.py ops// entries only this target calls fake_ops.py fakes of the target's native ops, if it has any + csrc/ the target's own kernels and torch bindings catalog// entries more than one target calls ``` ### Native code -A target's C++/CUDA lives beside the stock tree, not in it: -`cpp/tensorrt_llm/modeling_v2///`, built into one -torch-op library per family, `libth_modeling_v2_.so`, installed next to -`libth_common.so`. Nothing of it is linked into `libtensorrt_llm.so` or -`libth_common.so`, and nothing loads it until routing imports one of the -family's targets, whose `__init__.py` starts with -`native_ops = load_target_ops(__name__)`. As with the stock code's optional -native modules, a library this installation does not carry is a `NoTarget`, so -`auto` keeps the built-in model; one that is installed but broken is a -`NativeLoadError`, never a fallback. Code several targets of a family share is -built once, in the family's `shared/`; its ops are `torch.ops.mv2_`, -which family code reaches with `load_family_ops(__name__)`. -`cpp/tensorrt_llm/modeling_v2/README.md` is the guide, and -`models/fixture/`, a temporary infrastructure fixture rather than a model, is -the smallest complete example. +A target's C++/CUDA lives with its Python code: `/csrc/`, and the +family's `csrc/` for code its targets share, declared by the family's +`csrc/CMakeLists.txt`. It builds into one torch-op library per family, +`libth_modeling_v2_.so`, installed next to `libth_common.so`. Nothing of +it is linked into `libtensorrt_llm.so` or `libth_common.so`, and nothing loads +it until routing imports one of the family's targets, whose `__init__.py` +starts with `native_ops = load_target_ops(__name__)`. As with the stock code's +optional native modules, a library this installation does not carry is a +`NoTarget`, so `auto` keeps the built-in model; one that is installed but +broken is a `NativeLoadError`, never a fallback. Adding a model with kernels +edits no file under `cpp/`. `NATIVE.md` is the guide, with the directories and +files a new model adds; `models/gpt_oss/` is the worked example. An entry, shared or owned, is one `OpWrapper` subclass: `__call__` is the op call a target makes, `reference` is what it is supposed to compute in plain diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/_native.py b/tensorrt_llm/_torch/_experimental/modeling_v2/_native.py index f72ae866d880..1d874e677eb2 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/_native.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/_native.py @@ -2,11 +2,12 @@ # SPDX-License-Identifier: Apache-2.0 """A target's native (C++/CUDA) code: where it is built, and how the target loads it. -Native code of a model family lives in ``cpp/tensorrt_llm/modeling_v2//`` -and builds into one torch-op library, ``libth_modeling_v2_.so``, installed -next to ``libth_common.so``. Nothing of it is in the stock libraries, and nothing -loads it until routing picks a target of the family: a target package that has -native code starts with:: +Native code of a model family lives with its Python code, in ``models//csrc/`` +(the family's ``CMakeLists.txt`` and shared code) and ``models///csrc/`` +(a target's own), and builds into one torch-op library, +``libth_modeling_v2_.so``, installed next to ``libth_common.so``. Nothing of +it is in the stock libraries, and nothing loads it until routing picks a target of +the family: a target package that has native code starts with:: from tensorrt_llm._torch._experimental.modeling_v2._native import load_target_ops @@ -30,7 +31,7 @@ Ops of a family's shared code are in ``torch.ops.mv2_``; code of the family reaches them through :func:`load_family_ops`. -``cpp/tensorrt_llm/modeling_v2/README.md`` is the guide for authors. +The "Native code" section of the modeling_v2 README is the guide for authors. """ from __future__ import annotations @@ -65,13 +66,6 @@ class NativeLoadError(RuntimeError): # --- Naming (mirrors cpp/cmake/modules/modeling_v2.cmake) -------------------- -def camel(name: str) -> str: - """The C++ directory of a family or target: ``scale_add__sm_10x__default`` -> - ``scaleAddSm10xDefault``.""" - parts = [p for p in name.split("_") if p] - return parts[0] + "".join(p[:1].upper() + p[1:] for p in parts[1:]) - - def identifier(name: str) -> str: """``name`` with ``__`` collapsed, usable as a C++ or torch identifier.""" while "__" in name: diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/__init__.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/__init__.py deleted file mode 100644 index 79047741cdd5..000000000000 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Temporary infrastructure fixture for native code: a family with no model behind it. - -Its one target, ``scale_add__sm_10x__default``, carries one native op so that CI -drives the whole native path -- the family library's build and packaging, the -loader's checks, the fake op, the GPU op -- on B300. No routing module names it; -tests import the target directly. - -Remove the family, its C++ half ``cpp/tensorrt_llm/modeling_v2/fixture/`` and -its tests once a family with native code has GPU unit tests that run in CI. -Until then it is also the smallest complete example of a target with native -code, which ``cpp/tensorrt_llm/modeling_v2/README.md`` walks through. -""" diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_10x__default/__init__.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_10x__default/__init__.py deleted file mode 100644 index 1f6251103cc9..000000000000 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_10x__default/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Fixture / sm_10x / default: the native-code fixture's only target. - -Loading the package loads its native code: ``native_ops`` is -``torch.ops.mv2_fixture_scale_add_sm_10x_default``, built from -``cpp/tensorrt_llm/modeling_v2/fixture/scaleAddSm10xDefault/``. A target with -native code starts this way, so that ``route`` importing the target is what -loads -- and checks -- its library. (Not ``ops``: that name is the target's -``ops/`` package, which would replace it on import.) -""" - -from tensorrt_llm._torch._experimental.modeling_v2._native import load_target_ops - -native_ops = load_target_ops(__name__) diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_10x__default/fake_ops.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_10x__default/fake_ops.py deleted file mode 100644 index 454ba14dfcde..000000000000 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_10x__default/fake_ops.py +++ /dev/null @@ -1,20 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Fake (meta) implementations of this target's native ops. - -The loader imports this module right after it opens the family library, so a -fake for an op the library does not define fails the load. Every op the -target's C++ registers has a fake here; the native claims test checks that by -reading both sides. -""" - -import torch - -from tensorrt_llm._torch._experimental.modeling_v2._native import namespace_of - -_NS = namespace_of(__package__) - - -@torch.library.register_fake(f"{_NS}::scale_add") -def _(x, alpha, beta): - return torch.empty_like(x) diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_10x__default/ops/__init__.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_10x__default/ops/__init__.py deleted file mode 100644 index 1059445198db..000000000000 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_10x__default/ops/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Ops used only by this target. Shared ops stay in the catalog.""" diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_10x__default/ops/elementwise/__init__.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_10x__default/ops/elementwise/__init__.py deleted file mode 100644 index 52a7a9daf028..000000000000 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_10x__default/ops/elementwise/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_10x__default/ops/elementwise/scale_add.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_10x__default/ops/elementwise/scale_add.py deleted file mode 100644 index 0546d58f1c57..000000000000 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/fixture/scale_add__sm_10x__default/ops/elementwise/scale_add.py +++ /dev/null @@ -1,56 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""`x * alpha + beta`: the native-code fixture's op, an entry like any other. - -The kernel is ``cpp/tensorrt_llm/modeling_v2/fixture/scaleAddSm10xDefault/``; the -op is reached through the target's ``native_ops`` namespace, which loading the -target package filled. Nothing here is specific to native code beyond that one name. -""" - -from __future__ import annotations - -import torch - -from tensorrt_llm._torch._experimental.modeling_v2.catalog._op import Arch, Cell, OpWrapper - -from ... import native_ops - - -class ScaleAdd(OpWrapper): - """`(x.float() * alpha + beta).to(x.dtype)`, elementwise, new tensor out. - - The product and the sum are each rounded to fp32 and the result once to - `x`'s dtype; the kernel does the same with no fused multiply-add, so the two - agree bitwise. - """ - - ARCHS = frozenset({Arch.SM_103}) - - CELLS: tuple[Cell, ...] = ( - Cell(why="fp32, one element", spec=dict(numel=1, dtype=torch.float32)), - Cell( - why="fp32, not a multiple of the block size", - spec=dict(numel=1000, dtype=torch.float32), - ), - Cell( - why="bf16, more elements than one grid covers", - spec=dict(numel=65535 * 256 + 3, dtype=torch.bfloat16), - ), - Cell(why="empty", spec=dict(numel=0, dtype=torch.bfloat16)), - ) - - note = """ - A fixture: it proves the native path works, not anything about a model. - """ - - def __call__(self, x: torch.Tensor, alpha: float, beta: float) -> torch.Tensor: - return native_ops.scale_add(x, alpha, beta) - - def reference(self, x: torch.Tensor, alpha: float, beta: float) -> torch.Tensor: - return (x.float() * alpha + beta).to(x.dtype) - - def is_valid(self, x: torch.Tensor, alpha: float, beta: float) -> None: - assert x.is_contiguous(), "the kernel indexes x as one contiguous run" - - -scale_add = ScaleAdd() diff --git a/cpp/tensorrt_llm/modeling_v2/fixture/CMakeLists.txt b/tensorrt_llm/_torch/_experimental/modeling_v2/models/gpt_oss/csrc/CMakeLists.txt similarity index 57% rename from cpp/tensorrt_llm/modeling_v2/fixture/CMakeLists.txt rename to tensorrt_llm/_torch/_experimental/modeling_v2/models/gpt_oss/csrc/CMakeLists.txt index 07cb27deb2c7..a700ed5efa4e 100644 --- a/cpp/tensorrt_llm/modeling_v2/fixture/CMakeLists.txt +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/gpt_oss/csrc/CMakeLists.txt @@ -15,15 +15,11 @@ # the License. # -# Temporary infrastructure fixture, not a model: one target with one op, so that -# CI drives the whole native path -- build, packaging, load, built_targets(), -# fake op, GPU op. Remove this family (with its Python package and tests) once a -# family with native code has GPU unit tests that run in CI. It is built for -# 100f only, the B300 its GPU test runs on, to keep the library it adds to every -# wheel small. -add_modeling_v2_family(fixture) +# gpt_oss native code. Each target keeps its own in /csrc/; see the +# modeling_v2 README. +add_modeling_v2_family(gpt_oss) add_modeling_v2_target( - fixture scale_add__sm_10x__default + gpt_oss gpt_oss_120b__sm_10x__default ARCHS 100f - SOURCES kernels/scaleAdd.cu ops/scaleAddOp.cpp) + SOURCES *.cu *.cpp) diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/gpt_oss/gpt_oss_120b__sm_10x__default/__init__.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/gpt_oss/gpt_oss_120b__sm_10x__default/__init__.py index 0b5adcfe48f8..81d9013e3e5e 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/gpt_oss/gpt_oss_120b__sm_10x__default/__init__.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/gpt_oss/gpt_oss_120b__sm_10x__default/__init__.py @@ -1,3 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 """gpt-oss-120b / sm_10x / default: every parallel layout the routing table admits.""" + +from tensorrt_llm._torch._experimental.modeling_v2._native import load_target_ops + +native_ops = load_target_ops(__name__) # torch.ops.mv2_gpt_oss_gpt_oss_120b_sm_10x_default diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/gpt_oss/gpt_oss_120b__sm_10x__default/csrc/swizzleBlockScales.cu b/tensorrt_llm/_torch/_experimental/modeling_v2/models/gpt_oss/gpt_oss_120b__sm_10x__default/csrc/swizzleBlockScales.cu new file mode 100644 index 000000000000..4eaed982506a --- /dev/null +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/gpt_oss/gpt_oss_120b__sm_10x__default/csrc/swizzleBlockScales.cu @@ -0,0 +1,67 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "gpt_oss/gpt_oss_120b__sm_10x__default/csrc/swizzleBlockScales.h" + +#include + +TRTLLM_MV2_NAMESPACE_BEGIN + +namespace kernels +{ + +namespace +{ + +// One 32-bit word per thread: the four scales (m, c..c+3) of a 4-column group stay together in the swizzled +// layout, so each word moves whole. In words, one 128x4 tile is 128 words. +__global__ void swizzleBlockScalesKernel( + uint32_t const* __restrict__ in, uint32_t* __restrict__ out, int64_t rows, int64_t words, int64_t total) +{ + for (int64_t i = blockIdx.x * static_cast(blockDim.x) + threadIdx.x; i < total; + i += static_cast(gridDim.x) * blockDim.x) + { + int64_t const word = i % words; + int64_t const m = (i / words) % rows; + int64_t const expert = i / (words * rows); + int64_t const dst + = expert * rows * words + (m / 128) * 128 * words + word * 128 + (m % 32) * 4 + (m % 128) / 32; + out[dst] = in[i]; + } +} + +} // namespace + +cudaError_t invokeSwizzleBlockScales( + uint8_t const* in, uint8_t* out, int64_t experts, int64_t rows, int64_t cols, cudaStream_t stream) +{ + int64_t const words = cols / 4; + int64_t const total = experts * rows * words; + if (total == 0) + { + return cudaSuccess; + } + constexpr int kThreads = 256; + int64_t const blocks = std::min((total + kThreads - 1) / kThreads, 65535); + swizzleBlockScalesKernel<<(blocks), kThreads, 0, stream>>>( + reinterpret_cast(in), reinterpret_cast(out), rows, words, total); + return cudaGetLastError(); +} + +} // namespace kernels + +TRTLLM_MV2_NAMESPACE_END diff --git a/cpp/tensorrt_llm/modeling_v2/fixture/scaleAddSm10xDefault/kernels/scaleAdd.h b/tensorrt_llm/_torch/_experimental/modeling_v2/models/gpt_oss/gpt_oss_120b__sm_10x__default/csrc/swizzleBlockScales.h similarity index 63% rename from cpp/tensorrt_llm/modeling_v2/fixture/scaleAddSm10xDefault/kernels/scaleAdd.h rename to tensorrt_llm/_torch/_experimental/modeling_v2/models/gpt_oss/gpt_oss_120b__sm_10x__default/csrc/swizzleBlockScales.h index 9a093e7b544e..1d4cd171e16a 100644 --- a/cpp/tensorrt_llm/modeling_v2/fixture/scaleAddSm10xDefault/kernels/scaleAdd.h +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/gpt_oss/gpt_oss_120b__sm_10x__default/csrc/swizzleBlockScales.h @@ -20,24 +20,19 @@ #include "tensorrt_llm/modeling_v2/common/namespace.h" #include -#include +#include TRTLLM_MV2_NAMESPACE_BEGIN namespace kernels { -enum class ScaleAddDtype -{ - kFloat32, - kBfloat16, -}; - -//! out[i] = x[i] * alpha + beta over `numel` contiguous elements of `dtype`. The product and the sum are each -//! rounded to fp32 (no fused multiply-add), then the result is rounded once to `dtype`, which is what -//! `(x.float() * alpha + beta).to(dtype)` computes in torch. -cudaError_t invokeScaleAdd( - void const* x, void* out, int64_t numel, ScaleAddDtype dtype, float alpha, float beta, cudaStream_t stream); +//! Rearranges MXFP4 block scales, `[experts, rows, cols]` E8M0 bytes, into the 128x4 swizzled layout the MoE +//! kernel reads: within an expert, the byte at `(m, c)` moves to offset +//! `(m / 128) * 512 * (cols / 4) + (c / 4) * 512 + (m % 32) * 16 + ((m % 128) / 32) * 4 + c % 4`. +//! Requires `rows % 128 == 0` and `cols % 4 == 0`. +cudaError_t invokeSwizzleBlockScales( + uint8_t const* in, uint8_t* out, int64_t experts, int64_t rows, int64_t cols, cudaStream_t stream); } // namespace kernels diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/gpt_oss/gpt_oss_120b__sm_10x__default/csrc/swizzleBlockScalesOp.cpp b/tensorrt_llm/_torch/_experimental/modeling_v2/models/gpt_oss/gpt_oss_120b__sm_10x__default/csrc/swizzleBlockScalesOp.cpp new file mode 100644 index 000000000000..112198a5a529 --- /dev/null +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/gpt_oss/gpt_oss_120b__sm_10x__default/csrc/swizzleBlockScalesOp.cpp @@ -0,0 +1,68 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "gpt_oss/gpt_oss_120b__sm_10x__default/csrc/swizzleBlockScales.h" +#include "tensorrt_llm/common/assert.h" +#include "tensorrt_llm/modeling_v2/common/namespace.h" +#include "tensorrt_llm/modeling_v2/common/torchLibrary.h" + +#include +#include +#include + +#include + +TRTLLM_MV2_NAMESPACE_BEGIN + +namespace +{ + +at::Tensor swizzleBlockScales(at::Tensor const& scales) +{ + TORCH_CHECK( + scales.is_cuda() && scales.is_contiguous(), "swizzle_block_scales: scales must be a contiguous CUDA tensor"); + TORCH_CHECK(scales.scalar_type() == at::kByte, "swizzle_block_scales: scales must be uint8 (E8M0), got ", + scales.scalar_type()); + TORCH_CHECK( + scales.dim() == 3, "swizzle_block_scales: scales must be [experts, rows, cols], got ", scales.dim(), " dims"); + int64_t const rows = scales.size(1); + int64_t const cols = scales.size(2); + TORCH_CHECK(rows % 128 == 0 && cols % 4 == 0, + "swizzle_block_scales: needs rows % 128 == 0 and cols % 4 == 0, got rows ", rows, ", cols ", cols); + TORCH_CHECK(reinterpret_cast(scales.data_ptr()) % 4 == 0, + "swizzle_block_scales: scales must start on a 4-byte boundary"); + c10::cuda::CUDAGuard const deviceGuard{scales.device()}; + at::Tensor out = at::empty_like(scales); + cudaError_t const status = kernels::invokeSwizzleBlockScales(scales.data_ptr(), out.data_ptr(), + scales.size(0), rows, cols, at::cuda::getCurrentCUDAStream(scales.get_device())); + TLLM_CHECK_WITH_INFO(status == cudaSuccess, "swizzle_block_scales launch failed: %s", cudaGetErrorString(status)); + return out; +} + +} // namespace + +TRTLLM_MV2_NAMESPACE_END + +TRTLLM_MV2_LIBRARY_FRAGMENT(m) +{ + m.def("swizzle_block_scales(Tensor scales) -> Tensor"); +} + +TRTLLM_MV2_LIBRARY_IMPL(CUDA, m) +{ + m.impl("swizzle_block_scales", &TRTLLM_MV2_NS::swizzleBlockScales); +} diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/gpt_oss/gpt_oss_120b__sm_10x__default/fake_ops.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/gpt_oss/gpt_oss_120b__sm_10x__default/fake_ops.py new file mode 100644 index 000000000000..c1e0f2cdc10c --- /dev/null +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/gpt_oss/gpt_oss_120b__sm_10x__default/fake_ops.py @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Fake (meta) implementations of this target's native ops, built from its csrc/. + +The loader imports this module right after the target is found in the family +library, so a fake for an op the library does not define fails the load. +""" + +import torch + +from tensorrt_llm._torch._experimental.modeling_v2._native import namespace_of + +_NS = namespace_of(__package__) + + +@torch.library.register_fake(f"{_NS}::swizzle_block_scales") +def _(scales): + return torch.empty_like(scales) diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/gpt_oss/gpt_oss_120b__sm_10x__default/weights.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/gpt_oss/gpt_oss_120b__sm_10x__default/weights.py index 190cd8faad3c..44781c025096 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/gpt_oss/gpt_oss_120b__sm_10x__default/weights.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/gpt_oss/gpt_oss_120b__sm_10x__default/weights.py @@ -49,6 +49,8 @@ import torch import torch.nn as nn +from . import native_ops + #: `layers=ALL_LAYERS` is the common case and the default. A sentinel rather #: than a lambda so the table reads as data, and rather than `True` so the #: field has one type of answer: which layers. @@ -184,11 +186,9 @@ def _fc1_perm(rows: int, device) -> torch.Tensor: def _swizzle_scales(s: torch.Tensor) -> torch.Tensor: """Per-expert 128x4 block-scale swizzle, viewed back as `[E, M, C]`: the byte at `(m, c)` moves to flat offset `(m//128)*512*(C//4) + (c//4)*512 - + (m%32)*16 + ((m%128)//32)*4 + (c%4)`.""" - e, m, c = s.shape - assert m % 128 == 0 and c % 4 == 0, (m, c) - v = s.reshape(e, m // 128, 4, 32, c // 4, 4) # (e, m/128, (m%128)/32, m%32, c/4, c%4) - return v.permute(0, 1, 4, 3, 2, 5).reshape(e, m, c).contiguous() + + (m%32)*16 + ((m%128)//32)*4 + (c%4)`. This target's own kernel + (csrc/swizzleBlockScales.cu) moves it in one pass on the device.""" + return native_ops.swizzle_block_scales(s) def _experts(t: torch.Tensor, d) -> torch.Tensor: diff --git a/tests/integration/test_lists/test-db/l0_b300.yml b/tests/integration/test_lists/test-db/l0_b300.yml index ae2096685c87..ab454869f8fe 100644 --- a/tests/integration/test_lists/test-db/l0_b300.yml +++ b/tests/integration/test_lists/test-db/l0_b300.yml @@ -142,8 +142,7 @@ l0_b300: # cpu_only files run on the CPU stage instead), so 30 per entry is at least 2.2x headroom. - unittest/_torch/modeling_v2 --ignore=unittest/_torch/modeling_v2/comm --ignore=unittest/_torch/modeling_v2/models TIMEOUT (30) - unittest/_torch/modeling_v2/models/deepseek_v3 TIMEOUT (30) - # Temporary native-code infrastructure fixture (no model); built for 100f only. - - unittest/_torch/modeling_v2/models/fixture TIMEOUT (30) + - unittest/_torch/modeling_v2/models/gpt_oss TIMEOUT (30) - unittest/_torch/modeling_v2/models/glm_moe_dsa TIMEOUT (30) - unittest/_torch/modeling_v2/models/qwen4_exp TIMEOUT (30) - accuracy/modeling_v2/test_gpt_oss.py::TestModelingV2GptOss120bSm10xDefault::test_gsm8k[tp1] diff --git a/tests/unittest/_torch/modeling_v2/models/fixture/scale_add__sm_10x__default/elementwise/test_modeling_v2_scale_add.py b/tests/unittest/_torch/modeling_v2/models/fixture/scale_add__sm_10x__default/elementwise/test_modeling_v2_scale_add.py deleted file mode 100644 index c6168e989aa5..000000000000 --- a/tests/unittest/_torch/modeling_v2/models/fixture/scale_add__sm_10x__default/elementwise/test_modeling_v2_scale_add.py +++ /dev/null @@ -1,67 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""GPU test for the native-code fixture's scale_add entry. - -Importing the entry imports its target package, which loads the fixture family -library through ``_native.load_target_ops``: a pass here means the library was -built for this GPU, packaged, opened, listed the target in its built_targets() -and registered its fake op -- before the first cell runs. -""" - -import zlib - -import pytest -import torch - -from tensorrt_llm._torch._experimental.modeling_v2.models.fixture.scale_add__sm_10x__default import ( # noqa: E501 - native_ops, -) -from tensorrt_llm._torch._experimental.modeling_v2.models.fixture.scale_add__sm_10x__default.ops.elementwise.scale_add import ( # noqa: E501 - scale_add as op, -) - -__extra_import_path__ = ["../../../.."] -from _validating import certifying, validating # noqa: E402,F401 — path above; fixture by name - -assert torch.cuda.is_available(), "scale_add requires a CUDA device" - -_ALPHA, _BETA = 1.5, -0.25 - - -def _input(spec, why): - g = torch.Generator(device="cuda").manual_seed(zlib.crc32(why.encode())) - return torch.randn(spec["numel"], generator=g, device="cuda").to(spec["dtype"]) - - -@pytest.mark.parametrize("cell", op.CELLS, ids=[c.why for c in op.CELLS]) -def test_certified_cells(cell, certifying) -> None: # noqa: F811 — pytest resolves the fixture by this name - certifying(op) - x = _input(cell.spec, cell.why) - with validating(op): - result = op(x, _ALPHA, _BETA) - expected = op.reference(x, _ALPHA, _BETA) - assert result.dtype == x.dtype and result.shape == x.shape - # Both sides round the product and the sum to fp32 separately, then once to - # the output dtype: the band is zero. - assert torch.equal(result, expected) - - -def test_is_valid_rejects_a_strided_input() -> None: - x = torch.randn(64, device="cuda")[::2] - with pytest.raises(AssertionError, match="contiguous"): - with validating(op): - op(x, _ALPHA, _BETA) - - -def test_the_op_rejects_an_unsupported_dtype() -> None: - with pytest.raises(RuntimeError, match="float32 or bfloat16"): - native_ops.scale_add(torch.ones(4, device="cuda", dtype=torch.float16), _ALPHA, _BETA) - - -def test_the_fake_op_describes_the_output() -> None: - from torch._subclasses.fake_tensor import FakeTensorMode - - with FakeTensorMode(): - x = torch.empty(10, 3, device="cuda", dtype=torch.bfloat16) - out = native_ops.scale_add(x, _ALPHA, _BETA) - assert out.shape == (10, 3) and out.dtype == torch.bfloat16 diff --git a/tests/unittest/_torch/modeling_v2/models/gpt_oss/gpt_oss_120b__sm_10x__default/test_modeling_v2_gpt_oss_swizzle_block_scales.py b/tests/unittest/_torch/modeling_v2/models/gpt_oss/gpt_oss_120b__sm_10x__default/test_modeling_v2_gpt_oss_swizzle_block_scales.py new file mode 100644 index 000000000000..4e6ff9ab63ea --- /dev/null +++ b/tests/unittest/_torch/modeling_v2/models/gpt_oss/gpt_oss_120b__sm_10x__default/test_modeling_v2_gpt_oss_swizzle_block_scales.py @@ -0,0 +1,79 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""gpt-oss's own kernel: the MXFP4 block-scale swizzle of its weight loader. + +Importing the target package loads ``libth_modeling_v2_gpt_oss.so`` through +``_native.load_target_ops``, as routing does: a pass here means the family library +was built for this GPU, packaged, opened, carries the target, and registered the +fake. The kernel only moves bytes, so it must match the plain-torch swizzle it +replaces bit for bit. +""" + +import pytest +import torch +from torch._subclasses.fake_tensor import FakeTensorMode + +from tensorrt_llm._torch._experimental.modeling_v2.models.gpt_oss.gpt_oss_120b__sm_10x__default import ( # noqa: E501 + native_ops, + weights, +) + +assert torch.cuda.is_available(), "the gpt_oss family library needs a CUDA device" +pytestmark = pytest.mark.skipif( + torch.cuda.get_device_capability()[0] != 10, reason="built for sm_100f only" +) + + +def _reference(s: torch.Tensor) -> torch.Tensor: + """The plain-torch swizzle the loader used before the kernel.""" + e, m, c = s.shape + v = s.reshape(e, m // 128, 4, 32, c // 4, 4) + return v.permute(0, 1, 4, 3, 2, 5).reshape(e, m, c).contiguous() + + +def _scales(experts: int, rows: int, cols: int, seed: int) -> torch.Tensor: + g = torch.Generator(device="cuda").manual_seed(seed) + return torch.randint( + 0, 256, (experts, rows, cols), generator=g, device="cuda", dtype=torch.uint8 + ) + + +@pytest.mark.parametrize( + "experts, rows, cols", + [ + (1, 128, 4), # one 128x4 tile + (2, 256, 8), # tiles in both directions + (3, 384, 96), # gpt-oss-120b FC2: H / 32 = 90 scales, padded to 96 + (4, 5888, 92), # FC1 rows, many tiles + (0, 128, 4), # no experts + ], +) +def test_the_kernel_matches_the_torch_swizzle(experts, rows, cols): + s = _scales(experts, rows, cols, seed=rows * 131 + cols) + expected = _reference(s) + assert torch.equal(native_ops.swizzle_block_scales(s), expected) + # The loader's call site, which the FC1 and FC2 scale preparation both use. + assert torch.equal(weights._swizzle_scales(s), expected) + + +@pytest.mark.parametrize( + "scales, match", + [ + (lambda: _scales(1, 100, 4, 0), "rows % 128 == 0"), + (lambda: _scales(1, 128, 6, 0), "cols % 4 == 0"), + (lambda: _scales(1, 128, 4, 0).float(), "uint8"), + (lambda: _scales(2, 128, 8, 0)[:, :, :4], "contiguous"), + (lambda: _scales(1, 128, 4, 0).cpu(), "CUDA"), + ], + ids=["rows", "cols", "dtype", "strided", "host"], +) +def test_the_op_rejects_what_the_kernel_cannot_swizzle(scales, match): + with pytest.raises(RuntimeError, match=match): + native_ops.swizzle_block_scales(scales()) + + +def test_the_fake_op_describes_the_output(): + with FakeTensorMode(): + s = torch.empty(2, 256, 8, device="cuda", dtype=torch.uint8) + out = native_ops.swizzle_block_scales(s) + assert out.shape == (2, 256, 8) and out.dtype == torch.uint8 diff --git a/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_claims.py b/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_claims.py index 9f2a3f7b4637..690096a4bbaf 100644 --- a/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_claims.py +++ b/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_claims.py @@ -2,8 +2,9 @@ # SPDX-License-Identifier: Apache-2.0 """The boundary between stock code and modeling_v2 native code, read off disk. -modeling_v2 native code lives in ``cpp/tensorrt_llm/modeling_v2/`` and builds -one library per family that loads only when routing picks one of the family's +modeling_v2 native code lives with the Python code of its family, in +``models//csrc/`` and ``models///csrc/``, and builds one +library per family that loads only when routing picks one of the family's targets (``_native``). Three things rest on that staying true: ``off`` loading nothing, a target never depending on a sibling, and CBTS testing a family-only change with that family's tests alone @@ -25,10 +26,11 @@ import pytest -from tensorrt_llm._torch._experimental.modeling_v2._native import camel, identifier, torch_namespace +from tensorrt_llm._torch._experimental.modeling_v2._native import identifier, torch_namespace _REPO = Path(__file__).resolve().parents[4] -_NATIVE = _REPO / "cpp/tensorrt_llm/modeling_v2" +# What every family shares: discovery, common/ macros, contractSurface.txt. +_INFRA = _REPO / "cpp/tensorrt_llm/modeling_v2" _PY = _REPO / "tensorrt_llm/_torch/_experimental/modeling_v2" _MODELS = _PY / "models" _CMAKE_MODULE = _REPO / "cpp/cmake/modules/modeling_v2.cmake" @@ -36,12 +38,10 @@ pytestmark = [ pytest.mark.cpu_only, - pytest.mark.skipif(not _NATIVE.is_dir(), reason="needs the source checkout"), + pytest.mark.skipif(not _INFRA.is_dir(), reason="needs the source checkout"), ] _NATIVE_SUFFIXES = (".cu", ".cuh", ".h", ".hpp", ".inl", ".cpp") -# Native directories that are not families. -_NON_FAMILY_DIRS = {"common", "catalog"} # A target name's sm segment that names a family of architectures. _SM_GROUPS = {"10x": ((10, 0), (10, 3))} @@ -80,9 +80,9 @@ def _keyword_values(tokens: list[str], keyword: str) -> list[str]: def _families() -> dict[str, Path]: - """family -> native family directory, from every family CMakeLists.txt.""" + """family -> its csrc/ directory, from every models//csrc/CMakeLists.txt.""" families = {} - for cmake in sorted(_NATIVE.glob("*/CMakeLists.txt")): + for cmake in sorted(_MODELS.glob("*/csrc/CMakeLists.txt")): calls = _calls(cmake.read_text(), "add_modeling_v2_family") assert len(calls) == 1 and len(calls[0]) == 1, ( f"{cmake.relative_to(_REPO)} must call add_modeling_v2_family() exactly once" @@ -94,18 +94,28 @@ def _families() -> dict[str, Path]: def _targets() -> list[tuple[str, str, list[str]]]: """(family, target, tokens) of every add_modeling_v2_target call.""" out = [] - for family_dir in _families().values(): - for tokens in _calls((family_dir / "CMakeLists.txt").read_text(), "add_modeling_v2_target"): + for csrc in _families().values(): + for tokens in _calls((csrc / "CMakeLists.txt").read_text(), "add_modeling_v2_target"): out.append((tokens[0], tokens[1], tokens[2:])) return out +def _units() -> list[tuple[str, str | None, Path]]: + """(family, target or None for shared code, csrc directory) of every unit with code.""" + units = [(family, None, csrc) for family, csrc in _families().items()] + for family, target, _ in _targets(): + csrc = _MODELS / family / target / "csrc" + if csrc.is_dir(): + units.append((family, target, csrc)) + return units + + def _native_files(root: Path) -> list[Path]: return sorted(p for p in root.rglob("*") if p.is_file() and p.suffix in _NATIVE_SUFFIXES) def _contract_surface() -> list[str]: - lines = (_NATIVE / "contractSurface.txt").read_text().splitlines() + lines = (_INFRA / "contractSurface.txt").read_text().splitlines() return [line.strip() for line in lines if line.strip() and not line.startswith("#")] @@ -113,33 +123,61 @@ def _contract_surface() -> list[str]: def test_every_native_family_is_a_python_family(): - for family, family_dir in _families().items(): - assert family_dir.name == camel(family), ( - f"family {family!r} must live in {_NATIVE.relative_to(_REPO)}/{camel(family)}" + for family, csrc in _families().items(): + assert csrc.parent.name == family, ( + f"{csrc.relative_to(_REPO)}/CMakeLists.txt declares family {family!r}; " + f"it belongs in models/{family}/csrc/" ) - assert (_MODELS / family / "__init__.py").is_file(), ( + assert (csrc.parent / "__init__.py").is_file(), ( f"native family {family!r} has no Python family package models/{family}/" ) +def test_native_code_lives_only_in_csrc(): + """In the Python tree, C++/CUDA and CMake live only in a family's or a + target's csrc/, where the build finds them and these checks read them.""" + offenders = [] + for path in sorted(_MODELS.rglob("*")): + if not path.is_file() or "__pycache__" in path.parts: + continue + native = path.suffix in _NATIVE_SUFFIXES or path.name == "CMakeLists.txt" + if not native: + continue + parts = path.relative_to(_MODELS).parts + in_family_csrc = len(parts) >= 3 and parts[1] == "csrc" + in_target_csrc = len(parts) >= 4 and parts[2] == "csrc" + if path.name == "CMakeLists.txt": + ok = len(parts) == 3 and parts[1] == "csrc" + else: + ok = in_family_csrc or in_target_csrc + if not ok: + offenders.append(str(path.relative_to(_REPO))) + assert not offenders, ( + "native code outside models//csrc/ or models///csrc/ " + f"(CMakeLists.txt only in models//csrc/): {offenders}" + ) + + def test_every_native_target_is_a_python_target_that_loads_it(): - declared = set() - for family, target, _ in _targets(): - declared.add((family, target)) + with_code = set() + for family, target, tokens in _targets(): package = _MODELS / family / target / "__init__.py" assert package.is_file(), f"native target {family}/{target} has no Python target package" assert "load_target_ops(__name__)" in package.read_text(), ( f"{package.relative_to(_REPO)} must start by loading its native code: " "native_ops = load_target_ops(__name__)" ) - for family, family_dir in _families().items(): - for sub in sorted(p for p in family_dir.iterdir() if p.is_dir()): - if sub.name == "shared": - continue - assert any(camel(t) == sub.name for f, t in declared if f == family), ( - f"{sub.relative_to(_REPO)} is not the directory of any add_modeling_v2_target " - f"of family {family!r}" + if _keyword_values(tokens, "SOURCES"): + with_code.add((family, target)) + assert (_MODELS / family / target / "csrc").is_dir(), ( + f"{family}/{target} declares SOURCES but has no {target}/csrc/" ) + for csrc in sorted(_MODELS.glob("*/*/csrc")): + family, target = csrc.parent.parent.name, csrc.parent.name + assert (family, target) in with_code, ( + f"{csrc.relative_to(_REPO)} holds code no add_modeling_v2_target({family} {target} " + "... SOURCES ...) builds" + ) def test_python_targets_that_load_native_code_have_it(): @@ -179,24 +217,26 @@ def test_declared_archs_cover_the_sm_in_the_target_name(): def test_native_code_includes_only_its_own_code_and_the_contract_surface(): surface = _contract_surface() + python_families = {p.name for p in _MODELS.iterdir() if (p / "__init__.py").is_file()} problems = [] - for family, family_dir in _families().items(): - for unit_dir in sorted(p for p in family_dir.iterdir() if p.is_dir()): - own = f"tensorrt_llm/modeling_v2/{family_dir.name}/{unit_dir.name}/" - allowed = (own, "tensorrt_llm/modeling_v2/common/") - if unit_dir.name != "shared": - allowed += (f"tensorrt_llm/modeling_v2/{family_dir.name}/shared/",) - for path in _native_files(unit_dir): - for include in _INCLUDE.findall(path.read_text()): - where = f'{path.relative_to(_REPO)}: "{include}"' - if include.startswith("tensorrt_llm/modeling_v2/"): - if not include.startswith(allowed): - problems.append(f"{where} reaches another target or family") - elif include.startswith("tensorrt_llm/"): - if not any(fnmatch.fnmatch(include, p) for p in surface): - problems.append(f"{where} is not in contractSurface.txt") - elif ".." in include: - problems.append(f"{where} leaves its directory") + for family, target, csrc in _units(): + allowed = [f"{family}/csrc/", "tensorrt_llm/modeling_v2/common/"] + if target is not None: + allowed.append(f"{family}/{target}/csrc/") + for path in _native_files(csrc): + for include in _INCLUDE.findall(path.read_text()): + where = f'{path.relative_to(_REPO)}: "{include}"' + head = include.split("/", 1)[0] + if head in python_families or include.startswith("tensorrt_llm/modeling_v2/"): + if not include.startswith(tuple(allowed)): + problems.append(f"{where} reaches another target or family") + elif include.startswith("tensorrt_llm/"): + if not any(fnmatch.fnmatch(include, p) for p in surface): + problems.append(f"{where} is not in contractSurface.txt") + elif ".." in include or "/" not in include and (csrc / include).exists(): + problems.append( + f'{where}: include the family\'s code from models/, "{family}/..."' + ) assert not problems, "\n".join(problems) @@ -215,9 +255,9 @@ def test_native_code_registers_only_into_its_own_namespace(): namespace; the TRTLLM_MV2_ macros register into the unit's own only.""" offenders = [ str(path.relative_to(_REPO)) - for path in _native_files(_NATIVE) - if not path.is_relative_to(_NATIVE / "common") - and _RAW_REGISTRATION.search(path.read_text()) + for _, _, csrc in _units() + for path in _native_files(csrc) + if _RAW_REGISTRATION.search(path.read_text()) ] assert not offenders, ( f"use TRTLLM_MV2_LIBRARY_FRAGMENT / TRTLLM_MV2_LIBRARY_IMPL, not TORCH_LIBRARY*: {offenders}" @@ -233,11 +273,8 @@ def test_every_native_op_has_a_fake(): cannot see, and a fake without an op fails the load. A target's ops have their fakes in the target's fake_ops.py, the family's shared ops in the family's.""" - units = [(_families()[f] / camel(t), _MODELS / f / t) for f, t, _ in _targets()] - units += [(d / "shared", _MODELS / f) for f, d in _families().items()] - for native_dir, python_dir in units: - if not native_dir.is_dir(): - continue + for family, target, native_dir in _units(): + python_dir = _MODELS / family if target is None else _MODELS / family / target ops = sorted( {op for p in _native_files(native_dir) for op in _OP_DEF.findall(p.read_text())} ) @@ -257,7 +294,8 @@ def test_every_native_op_has_a_fake(): _STOCK_PY_REFERENCE = re.compile(r"modeling_v2[./]models\b|torch\.ops\.mv2_") _STOCK_CPP_REFERENCE = re.compile(r'#\s*include\s*"tensorrt_llm/modeling_v2/') -_STOCK_CMAKE_REFERENCE = re.compile(r"modeling_v2/\w") +# A path into one family's directory; the discovery's models/*/csrc glob names none. +_STOCK_FAMILY_PATH = re.compile(r"_experimental/modeling_v2/models/[a-z]") def test_stock_python_does_not_reference_targets(): @@ -273,22 +311,40 @@ def test_stock_python_does_not_reference_targets(): ) -def test_stock_native_code_does_not_reference_targets(): +def test_stock_build_does_not_reference_a_family(): + """A family's native code reaches the build only through + cpp/tensorrt_llm/modeling_v2/'s discovery: no stock CMake, C++ or packaging + file names a family, so adding one edits none of them.""" offenders = [] - for root in ("cpp/tensorrt_llm", "cpp/include", "cpp/tests"): + for root in ("cpp/tensorrt_llm", "cpp/include", "cpp/tests", "cpp/cmake"): for path in sorted((_REPO / root).rglob("*")): - if not path.is_file() or path.is_relative_to(_NATIVE): + if not path.is_file() or path.is_relative_to(_INFRA): continue # Skip generated cubin sources; they include nothing of interest. if path.suffix in _NATIVE_SUFFIXES and path.stat().st_size < (1 << 20): if _STOCK_CPP_REFERENCE.search(path.read_text(errors="ignore")): offenders.append(str(path.relative_to(_REPO))) - elif path.name == "CMakeLists.txt": - if _STOCK_CMAKE_REFERENCE.search(path.read_text()): + elif path.name == "CMakeLists.txt" or path.suffix == ".cmake": + if _STOCK_FAMILY_PATH.search(path.read_text()): offenders.append(str(path.relative_to(_REPO))) - assert not offenders, ( - f"stock native code reaches into cpp/tensorrt_llm/modeling_v2: {offenders}" - ) + for name in ("cpp/CMakeLists.txt", "setup.py", "scripts/build_wheel.py"): + if _STOCK_FAMILY_PATH.search((_REPO / name).read_text()): + offenders.append(name) + assert not offenders, f"stock build files name a modeling_v2 family: {offenders}" + + +def test_native_files_carry_the_license_header(): + """The C++ license checker scans cpp/ only; this covers the families' csrc/.""" + missing = [] + for _, _, csrc in _units(): + for path in [*_native_files(csrc), *csrc.glob("CMakeLists.txt")]: + head = path.read_text(errors="ignore")[:400] + if ( + "SPDX-FileCopyrightText" not in head + or "SPDX-License-Identifier: Apache-2.0" not in head + ): + missing.append(str(path.relative_to(_REPO))) + assert not missing, f"no NVIDIA SPDX header: {missing}" def test_stock_custom_ops_register_no_fakes_for_targets(): @@ -427,21 +483,15 @@ def test_accuracy_entries_name_a_family(): @pytest.mark.skipif(shutil.which("cmake") is None, reason="needs cmake") def test_cmake_and_python_agree_on_names_and_architectures(tmp_path): - """The loader and CBTS compute directory, namespace and architecture - coverage in Python; the build computes them in modeling_v2.cmake.""" - names = [ - "fixture", - "minimax_m3", - "scale_add__sm_10x__default", - "r1_0528_nvfp4__sm_10x__default", - ] + """The loader and these checks compute identifiers, namespaces and + architecture coverage in Python; the build computes them in modeling_v2.cmake.""" + names = ["gpt_oss", "minimax_m3", "r1_0528_nvfp4__sm_10x__default"] script = tmp_path / "probe.cmake" lines = [f'include("{_CMAKE_MODULE}")'] for name in names: lines += [ - f'_modeling_v2_camel(c "{name}")', f'_modeling_v2_ident(i "{name}")', - f'message(STATUS "name {name} ${{c}} ${{i}}")', + f'message(STATUS "name {name} ${{i}}")', ] lines += [ "_modeling_v2_resolve_archs(r DECLARED 80 90a 100f 103a 120f REQUESTED 86 90 103 107)", @@ -452,9 +502,9 @@ def test_cmake_and_python_agree_on_names_and_architectures(tmp_path): ["cmake", "-P", str(script)], capture_output=True, text=True, check=True ).stdout for name in names: - assert f"name {name} {camel(name)} {identifier(name)}" in output, output + assert f"name {name} {identifier(name)}" in output, output assert "archs 90a-real;100f-real;103a-real" in output, output # The family namespace the build registers is the loader's. - assert torch_namespace("fixture", "scale_add__sm_10x__default") == ( - "mv2_fixture_scale_add_sm_10x_default" + assert torch_namespace("gpt_oss", "gpt_oss_120b__sm_10x__default") == ( + "mv2_gpt_oss_gpt_oss_120b_sm_10x_default" ) diff --git a/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_loader.py b/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_loader.py index 3bc88df07087..2106a0c35093 100644 --- a/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_loader.py +++ b/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_loader.py @@ -28,7 +28,6 @@ from tensorrt_llm._torch._experimental.modeling_v2 import _native from tensorrt_llm._torch._experimental.modeling_v2._native import ( NativeLoadError, - camel, family_of, identifier, library_name, @@ -43,7 +42,9 @@ pytestmark = pytest.mark.cpu_only -_FIXTURE = "tensorrt_llm._torch._experimental.modeling_v2.models.fixture.scale_add__sm_10x__default" +_TARGET = ( + "tensorrt_llm._torch._experimental.modeling_v2.models.gpt_oss.gpt_oss_120b__sm_10x__default" +) @pytest.fixture @@ -56,9 +57,9 @@ def libs(monkeypatch, tmp_path): def _install(libs: Path, monkeypatch, opens) -> None: - """Put a fixture library in `libs`; `opens` stands in for dlopen, and no + """Put a gpt_oss library in `libs`; `opens` stands in for dlopen, and no fake_ops module is imported, since the stand-in defines no op.""" - (libs / library_name("fixture")).write_bytes(b"") + (libs / library_name("gpt_oss")).write_bytes(b"") monkeypatch.setattr(_native, "_open_library", lambda path, family: opens()) monkeypatch.setattr(_native, "_import_fakes", lambda module, family: None) @@ -66,30 +67,16 @@ def _install(libs: Path, monkeypatch, opens) -> None: # --- Naming ---------------------------------------------------------------------- -@pytest.mark.parametrize( - "name, expected", - [ - ("fixture", "fixture"), - ("minimax_m3", "minimaxM3"), - ("glm_moe_dsa", "glmMoeDsa"), - ("minimax_m3_nvfp4__sm_107__tp2", "minimaxM3Nvfp4Sm107Tp2"), - ("r1_0528_nvfp4__sm_10x__default", "r10528Nvfp4Sm10xDefault"), - ], -) -def test_camel_is_the_native_directory(name, expected): - assert camel(name) == expected - - def test_names_of_libraries_and_namespaces(): assert library_name("minimax_m3") == "libth_modeling_v2_minimax_m3.so" - assert identifier("scale_add__sm_10x__default") == "scale_add_sm_10x_default" - assert torch_namespace("fixture") == "mv2_fixture" - assert namespace_of(_FIXTURE) == "mv2_fixture_scale_add_sm_10x_default" - assert split_target_package(_FIXTURE) == ("fixture", "scale_add__sm_10x__default") + assert identifier("gpt_oss_120b__sm_10x__default") == "gpt_oss_120b_sm_10x_default" + assert torch_namespace("gpt_oss") == "mv2_gpt_oss" + assert namespace_of(_TARGET) == "mv2_gpt_oss_gpt_oss_120b_sm_10x_default" + assert split_target_package(_TARGET) == ("gpt_oss", "gpt_oss_120b__sm_10x__default") with pytest.raises(ValueError): split_target_package("tensorrt_llm._torch.models") - assert family_of(f"{_FIXTURE}.ops.elementwise.scale_add") == "fixture" - assert family_of(_FIXTURE.rpartition(".")[0] + ".fake_ops") == "fixture" + assert family_of(f"{_TARGET}.weights") == "gpt_oss" + assert family_of(_TARGET.rpartition(".")[0] + ".fake_ops") == "gpt_oss" with pytest.raises(ValueError): family_of("tensorrt_llm._torch._experimental.modeling_v2.models") @@ -98,8 +85,8 @@ def test_names_of_libraries_and_namespaces(): def test_a_library_this_installation_lacks_is_no_target(libs): - with pytest.raises(NoTarget, match="has no libth_modeling_v2_fixture.so"): - load_target_ops(_FIXTURE) + with pytest.raises(NoTarget, match="has no libth_modeling_v2_gpt_oss.so"): + load_target_ops(_TARGET) def test_a_library_that_fails_to_open_is_a_broken_installation(libs, monkeypatch): @@ -108,13 +95,13 @@ def fail(): _install(libs, monkeypatch, fail) with pytest.raises(NativeLoadError, match="undefined symbol"): - load_target_ops(_FIXTURE) + load_target_ops(_TARGET) def test_a_target_the_library_does_not_carry_is_no_target(libs, monkeypatch): _install(libs, monkeypatch, lambda: frozenset({"other__sm_90__default"})) - with pytest.raises(NoTarget, match="does not carry scale_add__sm_10x__default"): - load_target_ops(_FIXTURE) + with pytest.raises(NoTarget, match="does not carry gpt_oss_120b__sm_10x__default"): + load_target_ops(_TARGET) def test_a_library_opens_once(libs, monkeypatch): @@ -122,29 +109,31 @@ def test_a_library_opens_once(libs, monkeypatch): def opens(): opened.append(1) - return frozenset({"scale_add__sm_10x__default"}) + return frozenset({"gpt_oss_120b__sm_10x__default"}) _install(libs, monkeypatch, opens) - assert load_family_library("fixture") == {"scale_add__sm_10x__default"} - assert load_family_library("fixture") == {"scale_add__sm_10x__default"} + assert load_family_library("gpt_oss") == {"gpt_oss_120b__sm_10x__default"} + assert load_family_library("gpt_oss") == {"gpt_oss_120b__sm_10x__default"} assert len(opened) == 1 @pytest.fixture def torch_ops(monkeypatch): - """A stand-in torch.ops holding the fixture's namespaces.""" - ops = types.SimpleNamespace(mv2_fixture=object(), mv2_fixture_scale_add_sm_10x_default=object()) + """A stand-in torch.ops holding the gpt_oss namespaces.""" + ops = types.SimpleNamespace( + mv2_gpt_oss=object(), mv2_gpt_oss_gpt_oss_120b_sm_10x_default=object() + ) monkeypatch.setitem(sys.modules, "torch", types.SimpleNamespace(ops=ops)) return ops def test_opening_a_library_registers_the_family_fakes_first(libs, monkeypatch, torch_ops): - _install(libs, monkeypatch, lambda: frozenset({"scale_add__sm_10x__default"})) + _install(libs, monkeypatch, lambda: frozenset({"gpt_oss_120b__sm_10x__default"})) imported = [] monkeypatch.setattr(_native, "_import_fakes", lambda module, family: imported.append(module)) - assert load_target_ops(_FIXTURE) is torch_ops.mv2_fixture_scale_add_sm_10x_default - family_package = _FIXTURE.rpartition(".")[0] - assert imported == [f"{family_package}.fake_ops", f"{_FIXTURE}.fake_ops"] + assert load_target_ops(_TARGET) is torch_ops.mv2_gpt_oss_gpt_oss_120b_sm_10x_default + family_package = _TARGET.rpartition(".")[0] + assert imported == [f"{family_package}.fake_ops", f"{_TARGET}.fake_ops"] def test_family_ops_are_the_family_namespace_and_load_once(libs, monkeypatch, torch_ops): @@ -152,24 +141,24 @@ def test_family_ops_are_the_family_namespace_and_load_once(libs, monkeypatch, to def opens(): opened.append(1) - return frozenset({"scale_add__sm_10x__default"}) + return frozenset({"gpt_oss_120b__sm_10x__default"}) _install(libs, monkeypatch, opens) - module = f"{_FIXTURE}.ops.elementwise.scale_add" - assert load_family_ops(module) is torch_ops.mv2_fixture - assert load_family_ops(module) is torch_ops.mv2_fixture + module = f"{_TARGET}.weights" + assert load_family_ops(module) is torch_ops.mv2_gpt_oss + assert load_family_ops(module) is torch_ops.mv2_gpt_oss assert len(opened) == 1 def test_family_ops_of_a_family_this_installation_lacks_are_no_target(libs): - with pytest.raises(NoTarget, match="has no libth_modeling_v2_fixture.so"): - load_family_ops(f"{_FIXTURE}.ops.elementwise.scale_add") + with pytest.raises(NoTarget, match="has no libth_modeling_v2_gpt_oss.so"): + load_family_ops(f"{_TARGET}.weights") def test_windows_has_no_native_targets(libs, monkeypatch): monkeypatch.setattr(sys, "platform", "win32") with pytest.raises(NoTarget, match="Windows"): - load_family_library("fixture") + load_family_library("gpt_oss") # --- The installed wheel --------------------------------------------------------- diff --git a/tests/unittest/scripts/cbts/test_modeling_v2_rule.py b/tests/unittest/scripts/cbts/test_modeling_v2_rule.py index 2be9e76788c4..a4aea22f6302 100644 --- a/tests/unittest/scripts/cbts/test_modeling_v2_rule.py +++ b/tests/unittest/scripts/cbts/test_modeling_v2_rule.py @@ -13,18 +13,16 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Tests for CBTS ModelingV2Rule: per-family selection over Python and native paths.""" +"""Tests for CBTS ModelingV2Rule: per-family selection, native code included.""" from __future__ import annotations -from pathlib import Path - import pytest __extra_import_path__ = ["~/jenkins/scripts/cbts"] from cbts.blocks import Block, Stage, YAMLIndex from cbts.rules.base import PRInputs -from cbts.rules.modeling_v2 import ModelingV2Rule, _is_mv2_claim, _snake +from cbts.rules.modeling_v2 import ModelingV2Rule, _is_mv2_claim pytestmark = pytest.mark.cpu_only @@ -34,23 +32,14 @@ "unittest/_torch/modeling_v2 --ignore=unittest/_torch/modeling_v2/comm " "--ignore=unittest/_torch/modeling_v2/models TIMEOUT (30)" ) -_FIXTURE_UNIT = "unittest/_torch/modeling_v2/models/fixture" +_DEEPSEEK_UNIT = "unittest/_torch/modeling_v2/models/deepseek_v3" _GPT_OSS_UNIT = "unittest/_torch/modeling_v2/models/gpt_oss" _GPT_OSS_ACCURACY = "accuracy/modeling_v2/test_gpt_oss.py::TestGptOss::test_gsm8k[tp1]" _CLAIMS = "unittest/_torch/modeling_v2/test_modeling_v2_claims.py" @pytest.fixture -def repo_root(tmp_path: Path) -> Path: - for family in ("fixture", "gpt_oss", "minimax_m3"): - package = tmp_path / _PY / "models" / family - package.mkdir(parents=True) - (package / "__init__.py").write_text('"""Family."""\n') - return tmp_path - - -@pytest.fixture -def rule(repo_root: Path) -> ModelingV2Rule: +def rule() -> ModelingV2Rule: yaml_index = YAMLIndex() yaml_index.blocks = [ Block( @@ -60,7 +49,7 @@ def rule(repo_root: Path) -> ModelingV2Rule: tests=[ "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4", _CATALOG_UNIT, - _FIXTURE_UNIT, + _DEEPSEEK_UNIT, _GPT_OSS_UNIT, _GPT_OSS_ACCURACY, ], @@ -71,7 +60,7 @@ def rule(repo_root: Path) -> ModelingV2Rule: "B300-PyTorch-1": Stage("B300-PyTorch-1", "l0_b300", "aarch64", 1, 2), "CPU-Generic-x86-1": Stage("CPU-Generic-x86-1", "l0_cpu", "x86_64", 1, 1), } - return ModelingV2Rule(yaml_index, stages, repo_root=repo_root) + return ModelingV2Rule(yaml_index, stages) def _apply(rule: ModelingV2Rule, *files: str): @@ -88,8 +77,11 @@ def _selected(result) -> set[str]: def test_claims_python_and_native_sources_but_not_docs_or_stock(): - assert _is_mv2_claim(f"{_PY}/models/fixture/scale_add__sm_10x__default/__init__.py") - assert _is_mv2_claim(f"{_CPP}/fixture/scaleAddSm10xDefault/kernels/scaleAdd.cu") + assert _is_mv2_claim(f"{_PY}/models/gpt_oss/gpt_oss_120b__sm_10x__default/__init__.py") + assert _is_mv2_claim( + f"{_PY}/models/gpt_oss/gpt_oss_120b__sm_10x__default/csrc/swizzleBlockScales.cu" + ) + assert _is_mv2_claim(f"{_PY}/models/gpt_oss/csrc/CMakeLists.txt") assert _is_mv2_claim(f"{_CPP}/contractSurface.txt") assert _is_mv2_claim("cpp/cmake/modules/modeling_v2.cmake") assert not _is_mv2_claim(f"{_CPP}/README.md") @@ -97,43 +89,40 @@ def test_claims_python_and_native_sources_but_not_docs_or_stock(): assert not _is_mv2_claim("tensorrt_llm/_torch/pyexecutor/model_loader.py") -def test_family_of_reads_both_trees(rule): +def test_family_of_reads_the_family_directory_native_code_included(rule): assert rule.family_of(f"{_PY}/models/minimax_m3/routing.py") == "minimax_m3" - assert rule.family_of(f"{_CPP}/minimaxM3/shared/attnMega.cu") == "minimax_m3" + assert rule.family_of(f"{_PY}/models/minimax_m3/csrc/attnMega.cu") == "minimax_m3" + assert ( + rule.family_of(f"{_PY}/models/minimax_m3/m3__sm_107__tp2/csrc/draftStep.cu") == "minimax_m3" + ) assert rule.family_of(f"{_PY}/catalog/_op.py") is None assert rule.family_of(f"{_PY}/models/__init__.py") is None assert rule.family_of(f"{_CPP}/common/torchLibrary.h") is None assert rule.family_of(f"{_CPP}/CMakeLists.txt") is None -def test_snake_inverts_camel_without_a_repo_root(): - assert _snake("glmMoeDsa") == "glm_moe_dsa" - assert _snake("minimaxM3") == "minimax_m3" - assert _snake("kimiK3Vl") == "kimi_k3_vl" - - @pytest.mark.parametrize( "changed", [ - f"{_PY}/models/fixture/scale_add__sm_10x__default/ops/elementwise/scale_add.py", - f"{_CPP}/fixture/scaleAddSm10xDefault/kernels/scaleAdd.cu", - f"{_CPP}/fixture/CMakeLists.txt", + f"{_PY}/models/gpt_oss/gpt_oss_120b__sm_10x__default/weights.py", + f"{_PY}/models/gpt_oss/gpt_oss_120b__sm_10x__default/csrc/swizzleBlockScales.cu", + f"{_PY}/models/gpt_oss/csrc/CMakeLists.txt", ], ) def test_a_family_change_selects_that_family_and_the_checks(rule, changed): result = _apply(rule, changed) assert result.scope == "modelingv2only" - assert _selected(result) == {_FIXTURE_UNIT, _CLAIMS} + assert _selected(result) == {_GPT_OSS_UNIT, _GPT_OSS_ACCURACY, _CLAIMS} assert result.sanity_relevant is False def test_two_families_union(rule): result = _apply( rule, - f"{_PY}/models/fixture/scale_add__sm_10x__default/__init__.py", - f"{_PY}/models/gpt_oss/routing.py", + f"{_PY}/models/deepseek_v3/routing.py", + f"{_PY}/models/gpt_oss/csrc/CMakeLists.txt", ) - assert _selected(result) == {_FIXTURE_UNIT, _GPT_OSS_UNIT, _GPT_OSS_ACCURACY, _CLAIMS} + assert _selected(result) == {_DEEPSEEK_UNIT, _GPT_OSS_UNIT, _GPT_OSS_ACCURACY, _CLAIMS} @pytest.mark.parametrize( @@ -143,6 +132,7 @@ def test_two_families_union(rule): f"{_PY}/_native.py", f"{_CPP}/common/torchLibrary.h", f"{_CPP}/contractSurface.txt", + f"{_CPP}/CMakeLists.txt", "cpp/cmake/modules/modeling_v2.cmake", ], ) @@ -151,7 +141,7 @@ def test_shared_modeling_v2_code_selects_everything_of_modeling_v2(rule, changed assert result.scope == "modelingv2only" assert _selected(result) == { _CATALOG_UNIT, - _FIXTURE_UNIT, + _DEEPSEEK_UNIT, _GPT_OSS_UNIT, _GPT_OSS_ACCURACY, _CLAIMS, @@ -159,7 +149,7 @@ def test_shared_modeling_v2_code_selects_everything_of_modeling_v2(rule, changed def test_a_family_with_no_test_entry_falls_back(rule): - result = _apply(rule, f"{_CPP}/minimaxM3/shared/attnMega.cu") + result = _apply(rule, f"{_PY}/models/minimax_m3/csrc/attnMega.cu") assert result.scope is None assert "minimax_m3" in result.reason From f1a287417aae2eced5d5993f9f75d3c029520525 Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Mon, 26 Oct 2026 02:45:23 -0700 Subject: [PATCH 08/20] [None][fix] modeling_v2: stop installing the library of a family that no longer exists A family removed or renamed since an earlier configure left its build directory behind with a non-empty cuda_architectures.txt, so build_wheel.py kept installing its old library (found by the installed-library smoke test after the fixture family was removed). The discovery now empties the cuda_architectures.txt of every family build directory without a family, so build_wheel.py skips it and removes the library it installed before. Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- cpp/tensorrt_llm/modeling_v2/CMakeLists.txt | 22 +++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/cpp/tensorrt_llm/modeling_v2/CMakeLists.txt b/cpp/tensorrt_llm/modeling_v2/CMakeLists.txt index 86d75a63f630..0f84199709ba 100644 --- a/cpp/tensorrt_llm/modeling_v2/CMakeLists.txt +++ b/cpp/tensorrt_llm/modeling_v2/CMakeLists.txt @@ -36,6 +36,28 @@ file( CONFIGURE_DEPENDS "${modeling_v2_models_dir}/*/csrc/CMakeLists.txt") list(SORT family_lists) +# A family removed or renamed since an earlier configure leaves its build +# directory, and a library in it, behind: empty its cuda_architectures.txt so +# that build_wheel.py neither installs that library nor keeps it installed. +set(families "") +foreach(family_list IN LISTS family_lists) + get_filename_component(csrc_dir "${family_list}" DIRECTORY) + get_filename_component(family_dir "${csrc_dir}" DIRECTORY) + get_filename_component(family "${family_dir}" NAME) + list(APPEND families "${family}") +endforeach() +file( + GLOB previous_arch_files + LIST_DIRECTORIES false + "${CMAKE_CURRENT_BINARY_DIR}/*/cuda_architectures.txt") +foreach(arch_file IN LISTS previous_arch_files) + get_filename_component(binary_dir "${arch_file}" DIRECTORY) + get_filename_component(name "${binary_dir}" NAME) + if(NOT name IN_LIST families) + file(WRITE "${arch_file}" "") + endif() +endforeach() + if(WIN32 OR NOT BUILD_MODELING_V2) # As deep_ep does on a build without a matching architecture: an empty # cuda_architectures.txt per family, so build_wheel.py installs nothing, From eb47e3c6badf41ef1fad900ca1a9df7e33d1dccd Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Mon, 26 Oct 2026 02:48:24 -0700 Subject: [PATCH 09/20] [None][fix] modeling_v2: link family libraries against Python, as th_common is , which op files and stock headers on the contract surface include (thop/thUtils.h among them), references the Python C API. th_common links ${Python3_LIBRARIES} for it; the family libraries, linked with --no-undefined, failed to link without it (found porting Kimi K3's op bindings). They now link it the same way. Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- cpp/cmake/modules/modeling_v2.cmake | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/cpp/cmake/modules/modeling_v2.cmake b/cpp/cmake/modules/modeling_v2.cmake index 2a105a72b1e8..ef02a6473112 100644 --- a/cpp/cmake/modules/modeling_v2.cmake +++ b/cpp/cmake/modules/modeling_v2.cmake @@ -644,11 +644,19 @@ function(_modeling_v2_add_family_library family) endforeach() add_library(${library} MODULE "${registration}" ${object_sources}) _modeling_v2_link_libraries(dep_libraries ${deps}) - # Linked the way th_common is: torch, th_utils and libtensorrt_llm.so, whose - # singletons (logger, mcast registry, driver wrapper) stay single. + # Linked the way th_common is: torch, th_utils, Python (which + # , included by stock headers such as thop/thUtils.h, + # references) and libtensorrt_llm.so, whose singletons (logger, mcast + # registry, driver wrapper) stay single. target_link_libraries( - ${library} PRIVATE ${TORCH_LIBRARIES} th_utils ${SHARED_TARGET} - ${CUDA_RT_LIB} ${CUDA_DRV_LIB} ${dep_libraries}) + ${library} + PRIVATE ${TORCH_LIBRARIES} + th_utils + ${Python3_LIBRARIES} + ${SHARED_TARGET} + ${CUDA_RT_LIB} + ${CUDA_DRV_LIB} + ${dep_libraries}) set(version_script "${PROJECT_SOURCE_DIR}/tensorrt_llm/modeling_v2/common/familyLibrary.version" ) From a78626ab9f951f60813f5974f3c8074984a6c8b5 Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Mon, 26 Oct 2026 02:51:03 -0700 Subject: [PATCH 10/20] [None][fix] modeling_v2: accept the license headers the C++ license checker accepts The csrc/ license check required SPDX lines; the C++ license checker, which it stands in for outside cpp/, also accepts NVIDIA's copyright with the Apache-2.0 license text. Kimi K3's sources use that form. Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- .../test_modeling_v2_native_claims.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_claims.py b/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_claims.py index 690096a4bbaf..7f48e333944d 100644 --- a/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_claims.py +++ b/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_claims.py @@ -334,17 +334,20 @@ def test_stock_build_does_not_reference_a_family(): def test_native_files_carry_the_license_header(): - """The C++ license checker scans cpp/ only; this covers the families' csrc/.""" + """The C++ license checker (jenkins/license_cpp.json, Apache) scans cpp/ only; + this holds the families' csrc/ to the same header: NVIDIA's copyright and the + Apache-2.0 license, as SPDX lines or the license text.""" missing = [] for _, _, csrc in _units(): for path in [*_native_files(csrc), *csrc.glob("CMakeLists.txt")]: - head = path.read_text(errors="ignore")[:400] - if ( - "SPDX-FileCopyrightText" not in head - or "SPDX-License-Identifier: Apache-2.0" not in head - ): + head = path.read_text(errors="ignore")[:1200] + apache = ( + "SPDX-License-Identifier: Apache-2.0" in head + or "Apache License, Version 2.0" in head + ) + if "NVIDIA CORPORATION" not in head or not apache: missing.append(str(path.relative_to(_REPO))) - assert not missing, f"no NVIDIA SPDX header: {missing}" + assert not missing, f"no NVIDIA Apache-2.0 license header: {missing}" def test_stock_custom_ops_register_no_fakes_for_targets(): From 1a914e8d15b50dcd34fa5cbea1f6fb70b0d83691 Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Mon, 26 Oct 2026 19:12:42 -0700 Subject: [PATCH 11/20] [None][feat] modeling_v2: let native code include any stock header Removes contractSurface.txt, the allowlist of stock headers modeling_v2 native code could include. It did not protect CI scoping (a stock header change runs the full suite regardless), the build (stock and family libraries compile from one tree) or the link (--no-undefined catches a missing symbol), but it made every model that needed another stock header edit a file under cpp/. What keeps targets isolated stays checked: a target's includes reach only its own csrc/, its family's csrc/ and common/, never another target or family. Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- .github/CODEOWNERS | 6 ++--- cpp/cmake/modules/modeling_v2.cmake | 2 +- cpp/tensorrt_llm/modeling_v2/README.md | 1 - .../modeling_v2/contractSurface.txt | 17 ------------- .../scripts/cbts/cbts/rules/modeling_v2.py | 2 +- .../_experimental/modeling_v2/NATIVE.md | 12 +++++----- .../test_modeling_v2_native_claims.py | 24 ++++--------------- .../scripts/cbts/test_modeling_v2_rule.py | 3 +-- 8 files changed, 17 insertions(+), 50 deletions(-) delete mode 100644 cpp/tensorrt_llm/modeling_v2/contractSurface.txt diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 1392d91ea5e3..3ebf14e3b741 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -387,9 +387,9 @@ # Individual handles rather than a team, like SCAFFOLDING: this is one bounded # experiment with named owners, not a standing domain. /tensorrt_llm/_torch/_experimental/modeling_v2 @tianyuxbear @Wanli-Jiang @WeiHaocheng @litaotju -# What every modeling_v2 family library shares (discovery, common/ macros, the -# contract surface): the modeling_v2 owners, and the kernels owners because it is -# CUDA and links against the stock libraries. +# What every modeling_v2 family library shares (the discovery and the common/ +# macros): the modeling_v2 owners, and the kernels owners because it is CUDA and +# links against the stock libraries. /cpp/tensorrt_llm/modeling_v2 @tianyuxbear @Wanli-Jiang @WeiHaocheng @litaotju @NVIDIA/trt-llm-kernels-devs /cpp/cmake/modules/modeling_v2.cmake @tianyuxbear @Wanli-Jiang @WeiHaocheng @litaotju @NVIDIA/trt-llm-kernels-devs /tests/unittest/_torch/modeling_v2 @tianyuxbear @Wanli-Jiang @WeiHaocheng @litaotju diff --git a/cpp/cmake/modules/modeling_v2.cmake b/cpp/cmake/modules/modeling_v2.cmake index ef02a6473112..af5b10bfd716 100644 --- a/cpp/cmake/modules/modeling_v2.cmake +++ b/cpp/cmake/modules/modeling_v2.cmake @@ -31,7 +31,7 @@ libtensorrt_llm.so or libth_common.so; the Python loader (tensorrt_llm/_torch/_experimental/modeling_v2/_native.py) opens it only when routing picks one of the family's targets. modeling_v2/NATIVE.md is the guide for authors; cpp/tensorrt_llm/modeling_v2/ holds only what every family shares -(this discovery, common/ macros, contractSurface.txt). +(this discovery, the common/ macros). A family's csrc/CMakeLists.txt uses three functions; names are the Python package names: diff --git a/cpp/tensorrt_llm/modeling_v2/README.md b/cpp/tensorrt_llm/modeling_v2/README.md index 613f77cd92da..fd5b6395679b 100644 --- a/cpp/tensorrt_llm/modeling_v2/README.md +++ b/cpp/tensorrt_llm/modeling_v2/README.md @@ -17,4 +17,3 @@ changes nothing here: | `CMakeLists.txt` | Finds every `models//csrc/CMakeLists.txt` and builds the family libraries (`cpp/cmake/modules/modeling_v2.cmake`). | | `common/namespace.h`, `common/torchLibrary.h` | The per-unit C++ and torch namespace macros native code registers through. | | `common/familyLibrary.version` | The version script: a family library exports no symbol. | -| `contractSurface.txt` | The stock headers native code may include. Adding one is reviewed by the header's stock owners. | diff --git a/cpp/tensorrt_llm/modeling_v2/contractSurface.txt b/cpp/tensorrt_llm/modeling_v2/contractSurface.txt deleted file mode 100644 index 54628fb5df4a..000000000000 --- a/cpp/tensorrt_llm/modeling_v2/contractSurface.txt +++ /dev/null @@ -1,17 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# The stock headers modeling_v2 native code may include, as written in #include -# (resolved against cpp/include and cpp/): the stock surface target C++ -# depends on. tests/unittest/_torch/modeling_v2/test_modeling_v2_native_claims.py -# rejects any other stock include under cpp/tensorrt_llm/modeling_v2. -# -# Adding a header widens what stock changes can affect modeling_v2 targets; -# it is reviewed by the stock owners of that header. -tensorrt_llm/common/*.h -tensorrt_llm/common/*.cuh -tensorrt_llm/kernels/kvCacheUtils.h -tensorrt_llm/kernels/quantization.cuh -tensorrt_llm/kernels/quantization.h -tensorrt_llm/runtime/mcastDeviceMemory.h -tensorrt_llm/thop/thUtils.h diff --git a/jenkins/scripts/cbts/cbts/rules/modeling_v2.py b/jenkins/scripts/cbts/cbts/rules/modeling_v2.py index 8f5e6dbe7f44..3b0871a60f44 100644 --- a/jenkins/scripts/cbts/cbts/rules/modeling_v2.py +++ b/jenkins/scripts/cbts/cbts/rules/modeling_v2.py @@ -68,7 +68,7 @@ _PY_ROOT = "tensorrt_llm/_torch/_experimental/modeling_v2/" _PY_MODELS = _PY_ROOT + "models/" -# What every family library shares: discovery, common/ macros, contractSurface.txt. +# What every family library shares: the discovery and the common/ macros. _CPP_ROOT = "cpp/tensorrt_llm/modeling_v2/" # The native build module: shared by every family library. _CPP_BUILD_FILES = frozenset({"cpp/cmake/modules/modeling_v2.cmake"}) diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/NATIVE.md b/tensorrt_llm/_torch/_experimental/modeling_v2/NATIVE.md index 91d8c7d8e290..d5720d8f286e 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/NATIVE.md +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/NATIVE.md @@ -206,10 +206,10 @@ sibling targets can share file and function names. - Its own target's `csrc/` and its family's `csrc/`, written from `models/`: `"foo/foo_nvfp4__sm_103__tp4/csrc/x.h"`, `"foo/csrc/y.h"`. - `tensorrt_llm/modeling_v2/common/` (the macros). -- The stock headers listed in `cpp/tensorrt_llm/modeling_v2/contractSurface.txt`. - Adding one widens what stock changes can affect targets, so the header's stock - owners review it. -- System, CUDA, torch and cutlass headers. +- Any stock header (`cpp/include/`, `cpp/`), and system, CUDA, torch and cutlass + headers. + +It may not include another target's or another family's code. It links against `libtensorrt_llm.so` (logger, `TllmException`, mcast registry and other singletons stay there), `th_utils` and torch. **When a target needs @@ -241,8 +241,8 @@ installs the library only when that list is not empty. The narrowing is safe because family libraries are never in the stock libraries, and the CPU checks that keep it so run in every scope: `test_modeling_v2_native_claims.py` (native code only in `csrc/`, stock never -names a family, includes stay within the target, its family, `common/` and the -contract surface, every op has a fake, license headers), +names a family, no include reaches another target or family, every op has a +fake, license headers), `test_modeling_v2_native_loader.py` and `test_modeling_v2_rank_agreement.py`. Keep a family's pre-merge entry to one short test on hardware matching its SM; the accuracy matrix goes post-merge. diff --git a/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_claims.py b/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_claims.py index 7f48e333944d..12d42447874d 100644 --- a/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_claims.py +++ b/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_claims.py @@ -18,7 +18,6 @@ from __future__ import annotations import ast -import fnmatch import re import shutil import subprocess @@ -29,7 +28,7 @@ from tensorrt_llm._torch._experimental.modeling_v2._native import identifier, torch_namespace _REPO = Path(__file__).resolve().parents[4] -# What every family shares: discovery, common/ macros, contractSurface.txt. +# What every family shares: the discovery and the common/ macros. _INFRA = _REPO / "cpp/tensorrt_llm/modeling_v2" _PY = _REPO / "tensorrt_llm/_torch/_experimental/modeling_v2" _MODELS = _PY / "models" @@ -114,11 +113,6 @@ def _native_files(root: Path) -> list[Path]: return sorted(p for p in root.rglob("*") if p.is_file() and p.suffix in _NATIVE_SUFFIXES) -def _contract_surface() -> list[str]: - lines = (_INFRA / "contractSurface.txt").read_text().splitlines() - return [line.strip() for line in lines if line.strip() and not line.startswith("#")] - - # --- Layout ------------------------------------------------------------------ @@ -215,8 +209,10 @@ def test_declared_archs_cover_the_sm_in_the_target_name(): _INCLUDE = re.compile(r'^\s*#\s*include\s*"([^"]+)"', re.MULTILINE) -def test_native_code_includes_only_its_own_code_and_the_contract_surface(): - surface = _contract_surface() +def test_native_code_includes_no_other_target_or_family(): + """Native code may include stock headers, but of modeling_v2 only its own + target's csrc/, its family's csrc/ and common/: a target never reaches a + sibling or another family.""" python_families = {p.name for p in _MODELS.iterdir() if (p / "__init__.py").is_file()} problems = [] for family, target, csrc in _units(): @@ -230,9 +226,6 @@ def test_native_code_includes_only_its_own_code_and_the_contract_surface(): if head in python_families or include.startswith("tensorrt_llm/modeling_v2/"): if not include.startswith(tuple(allowed)): problems.append(f"{where} reaches another target or family") - elif include.startswith("tensorrt_llm/"): - if not any(fnmatch.fnmatch(include, p) for p in surface): - problems.append(f"{where} is not in contractSurface.txt") elif ".." in include or "/" not in include and (csrc / include).exists(): problems.append( f'{where}: include the family\'s code from models/, "{family}/..."' @@ -240,13 +233,6 @@ def test_native_code_includes_only_its_own_code_and_the_contract_surface(): assert not problems, "\n".join(problems) -def test_contract_surface_patterns_name_existing_headers(): - for pattern in _contract_surface(): - assert any((_REPO / root).glob(pattern) for root in ("cpp/include", "cpp")), ( - f"contractSurface.txt pattern {pattern!r} matches no header" - ) - - _RAW_REGISTRATION = re.compile(r"\bTORCH_LIBRARY(?:_FRAGMENT|_IMPL)?\s*\(") diff --git a/tests/unittest/scripts/cbts/test_modeling_v2_rule.py b/tests/unittest/scripts/cbts/test_modeling_v2_rule.py index a4aea22f6302..35432eff21e1 100644 --- a/tests/unittest/scripts/cbts/test_modeling_v2_rule.py +++ b/tests/unittest/scripts/cbts/test_modeling_v2_rule.py @@ -82,7 +82,7 @@ def test_claims_python_and_native_sources_but_not_docs_or_stock(): f"{_PY}/models/gpt_oss/gpt_oss_120b__sm_10x__default/csrc/swizzleBlockScales.cu" ) assert _is_mv2_claim(f"{_PY}/models/gpt_oss/csrc/CMakeLists.txt") - assert _is_mv2_claim(f"{_CPP}/contractSurface.txt") + assert _is_mv2_claim(f"{_CPP}/common/namespace.h") assert _is_mv2_claim("cpp/cmake/modules/modeling_v2.cmake") assert not _is_mv2_claim(f"{_CPP}/README.md") assert not _is_mv2_claim("cpp/tensorrt_llm/thop/attentionOp.cpp") @@ -131,7 +131,6 @@ def test_two_families_union(rule): f"{_PY}/catalog/_op.py", f"{_PY}/_native.py", f"{_CPP}/common/torchLibrary.h", - f"{_CPP}/contractSurface.txt", f"{_CPP}/CMakeLists.txt", "cpp/cmake/modules/modeling_v2.cmake", ], From 23d806644dd73429f031b8d217c7791325487e66 Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Mon, 26 Oct 2026 19:24:12 -0700 Subject: [PATCH 12/20] [None][chore] modeling_v2: narrow the native loader's API to four names _native.py exported ten names; native code and its fakes use four: load_target_ops, load_family_ops, namespace_of and NativeLoadError. The naming helpers and the family-library loader become private, and namespace_of now takes any module of a family: a module of a target package (target directories are ____) gives the target's namespace, any other module of the family the family's. Every fake_ops.py, a target's or a family's, registers in namespace_of(__name__). load_target_ops also rejects a module that is not a target package itself, since it imports that package's fake_ops. No behavior changes otherwise. Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- .../_experimental/modeling_v2/NATIVE.md | 4 +- .../_experimental/modeling_v2/_native.py | 141 ++++++++---------- .../gpt_oss_120b__sm_10x__default/fake_ops.py | 2 +- .../test_modeling_v2_native_claims.py | 6 +- .../test_modeling_v2_native_loader.py | 59 ++++---- 5 files changed, 101 insertions(+), 111 deletions(-) diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/NATIVE.md b/tensorrt_llm/_torch/_experimental/modeling_v2/NATIVE.md index d5720d8f286e..1c677942e105 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/NATIVE.md +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/NATIVE.md @@ -116,7 +116,7 @@ The gpt_oss files are the example for each step. ```python from tensorrt_llm._torch._experimental.modeling_v2._native import namespace_of - _NS = namespace_of(__package__) + _NS = namespace_of(__name__) @torch.library.register_fake(f"{_NS}::my_op") def _(x): @@ -149,7 +149,7 @@ add_modeling_v2_target(foo foo_nvfp4__sm_100__tp8 ARCHS 100f) ``` - **Fakes** of the shared ops go in the family's `models/foo/fake_ops.py`, - with `torch_namespace(family_of(__name__))`; the loader imports it when it + with `namespace_of(__name__)` as in a target's; the loader imports it when it opens the library. - **Callers** anywhere in the family use `load_family_ops(__name__).my_op(...)`. diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/_native.py b/tensorrt_llm/_torch/_experimental/modeling_v2/_native.py index 1d874e677eb2..e945acc474a1 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/_native.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/_native.py @@ -7,37 +7,33 @@ (a target's own), and builds into one torch-op library, ``libth_modeling_v2_.so``, installed next to ``libth_common.so``. Nothing of it is in the stock libraries, and nothing loads it until routing picks a target of -the family: a target package that has native code starts with:: - - from tensorrt_llm._torch._experimental.modeling_v2._native import load_target_ops - - native_ops = load_target_ops(__name__) # torch.ops.mv2__ - -``route`` imports the target only on the branch that returns it, so loading is part -of routing, and it follows how the stock code treats its optional native modules -(``flash_mla``, ``deep_ep``): - -* a library this installation does not have is a feature that is not there: - :class:`NoTarget`, so ``auto`` keeps the built-in model and ``require`` says why; -* a library that is there but fails to load is an error, :class:`NativeLoadError`, - never a fallback; -* which targets a library carries it answers itself, - ``torch.ops.mv2_.built_targets()``, the way ``th_common`` answers feature - queries; a target it does not carry is :class:`NoTarget`; -* the family's ``fake_ops.py`` (its shared ops) registers right after the library - opens, and the target's ``fake_ops.py`` right after the target is found in it; - either fails if an op it declares is missing from the library. - -Ops of a family's shared code are in ``torch.ops.mv2_``; code of the family -reaches them through :func:`load_family_ops`. - -The "Native code" section of the modeling_v2 README is the guide for authors. +the family. The API is four names: + +* :func:`load_target_ops` -- a target package with native code starts with + ``native_ops = load_target_ops(__name__)``; +* :func:`load_family_ops` -- the family's shared ops, ``load_family_ops(__name__)`` + from any module of the family; +* :func:`namespace_of` -- the torch namespace a ``fake_ops.py`` registers its fakes + in, ``namespace_of(__name__)``; +* :class:`NativeLoadError` -- a library that is installed but broken. + +``route`` imports a target only on the branch that returns it, so loading is part of +routing, and it follows how the stock code treats its optional native modules +(``flash_mla``, ``deep_ep``): a library this installation does not have, or a target +it did not build (it answers ``torch.ops.mv2_.built_targets()``), is a +:class:`NoTarget`, so ``auto`` keeps the built-in model; a library that fails to +load, or lacks an op a ``fake_ops.py`` declares, is a :class:`NativeLoadError`, +never a fallback. The family's ``fake_ops.py`` registers when the library opens, a +target's when the target is found in it. + +``NATIVE.md`` next to this module is the guide for authors. """ from __future__ import annotations import importlib import importlib.util +import re import sys from pathlib import Path from typing import Any, Dict, FrozenSet, Optional, Tuple @@ -57,57 +53,47 @@ class NativeLoadError(RuntimeError): """A family library is installed but cannot be used. - Raised when the library fails to open or an op the target's ``fake_ops.py`` - declares is missing from it. It is deliberately not a :class:`NoTarget`, so - ``auto`` does not hide a broken installation behind the built-in model. + Raised when the library fails to open or an op a ``fake_ops.py`` declares is + missing from it. It is deliberately not a :class:`NoTarget`, so ``auto`` does + not hide a broken installation behind the built-in model. + """ + + +def namespace_of(module: str) -> str: + """The torch namespace of the ops a module's ``fake_ops.py`` fakes. + + ``mv2__`` for a module of a target package, ``mv2_`` for + a module of the family itself (its shared ops). Target directories are named + ``____``, so the module path tells the two apart. """ + return _namespace(*_split(module)) # --- Naming (mirrors cpp/cmake/modules/modeling_v2.cmake) -------------------- -def identifier(name: str) -> str: +def _identifier(name: str) -> str: """``name`` with ``__`` collapsed, usable as a C++ or torch identifier.""" - while "__" in name: - name = name.replace("__", "_") - return name + return re.sub("_{2,}", "_", name) -def torch_namespace(family: str, target: Optional[str] = None) -> str: - """The torch namespace of a target's ops, or of the family's shared ops.""" - if target is None: - return f"mv2_{family}" - return f"mv2_{family}_{identifier(target)}" +def _namespace(family: str, target: Optional[str] = None) -> str: + return f"mv2_{family}" if target is None else f"mv2_{family}_{_identifier(target)}" -def library_name(family: str) -> str: - """The file name of a family's library.""" +def _library_name(family: str) -> str: return f"libth_modeling_v2_{family}.so" -def split_target_package(package: str) -> Tuple[str, str]: - """``(family, target)`` of a target package name.""" +def _split(module: str) -> Tuple[str, Optional[str]]: + """``(family, target)`` of a module under ``models.``; ``target`` is None + for a module of the family itself.""" prefix = f"{_MODELS_PACKAGE}." - if not package.startswith(prefix): - raise ValueError(f"{package!r} is not a modeling_v2 target package") - parts = package[len(prefix) :].split(".") - if len(parts) != 2: - raise ValueError(f"{package!r} is not a models.. package") - return parts[0], parts[1] - - -def family_of(module: str) -> str: - """The family of a module anywhere under ``models.``.""" - prefix = f"{_MODELS_PACKAGE}." - family = module[len(prefix) :].partition(".")[0] if module.startswith(prefix) else "" - if not family: + parts = module[len(prefix) :].split(".") if module.startswith(prefix) else [] + if not parts or not parts[0]: raise ValueError(f"{module!r} is not a module of a modeling_v2 family") - return family - - -def namespace_of(package: str) -> str: - """The torch namespace of the target package ``package``, e.g. for its fake_ops.py.""" - return torch_namespace(*split_target_package(package)) + target = parts[1] if len(parts) > 1 and "__" in parts[1] else None + return parts[0], target # --- Loading ------------------------------------------------------------------ @@ -122,7 +108,7 @@ def _import_fakes(module: str, family: str) -> None: importlib.import_module(module) except (AttributeError, RuntimeError, ValueError) as error: raise NativeLoadError( - f"{module} does not match the ops in {library_name(family)}: {error}" + f"{module} does not match the ops in {_library_name(family)}: {error}" ) from None @@ -131,21 +117,17 @@ def _open_library(path: Path, family: str) -> FrozenSet[str]: import torch torch.ops.load_library(str(path)) - answer = getattr(torch.ops, torch_namespace(family)).built_targets() + answer = getattr(torch.ops, _namespace(family)).built_targets() return frozenset(t for t in answer.split(",") if t) -def load_family_library(family: str) -> FrozenSet[str]: - """Open ``libth_modeling_v2_.so`` and return the targets it carries. - - Idempotent, and independent of the device, so it can run anywhere the stock - libraries load. :func:`load_target_ops` adds the target-level checks. - """ +def _load_family_library(family: str) -> FrozenSet[str]: + """Open ``libth_modeling_v2_.so`` once and return the targets it carries.""" if family in _built_targets: return _built_targets[family] if sys.platform == "win32": raise NoTarget("modeling_v2 native code is not built on Windows") - path = _LIBS_DIR / library_name(family) + path = _LIBS_DIR / _library_name(family) if not path.is_file(): raise NoTarget( f"this installation has no {path.name}: the build compiles a family's library " @@ -169,20 +151,21 @@ def load_target_ops(package: str) -> Any: :class:`NoTarget` when this installation does not carry the target and :class:`NativeLoadError` when its library is broken. """ - family, target = split_target_package(package) - built = load_family_library(family) + family, target = _split(package) + if target is None or package != f"{_MODELS_PACKAGE}.{family}.{target}": + raise ValueError(f"{package!r} is not a models.. package") + built = _load_family_library(family) if target not in built: carried = ", ".join(sorted(built)) or "none" raise NoTarget( - f"{library_name(family)} does not carry {target}: this build did not compile " + f"{_library_name(family)} does not carry {target}: this build did not compile " f"it (its ARCHS miss the build's, a REQUIRES is unmet, or " f"TRTLLM_MODELING_V2_TARGETS excluded it); it carries {carried}" ) - _import_fakes(f"{package}.fake_ops", family) import torch - return getattr(torch.ops, torch_namespace(family, target)) + return getattr(torch.ops, _namespace(family, target)) def load_family_ops(module: str) -> Any: @@ -191,14 +174,14 @@ def load_family_ops(module: str) -> Any: Call it with ``__name__`` from any module of the family, where the op is used; after the first call it is a dictionary lookup. Raises like - :func:`load_family_library`. Whether a target is carried is - :func:`load_target_ops`'s check, which the target's ``__init__.py`` runs first. + :func:`load_target_ops`, whose target check the target's ``__init__.py`` runs + first. """ ops = _family_ops.get(module) if ops is None: - family = family_of(module) - load_family_library(family) + family = _split(module)[0] + _load_family_library(family) import torch - ops = _family_ops[module] = getattr(torch.ops, torch_namespace(family)) + ops = _family_ops[module] = getattr(torch.ops, _namespace(family)) return ops diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/gpt_oss/gpt_oss_120b__sm_10x__default/fake_ops.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/gpt_oss/gpt_oss_120b__sm_10x__default/fake_ops.py index c1e0f2cdc10c..a71f9e242756 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/gpt_oss/gpt_oss_120b__sm_10x__default/fake_ops.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/gpt_oss/gpt_oss_120b__sm_10x__default/fake_ops.py @@ -10,7 +10,7 @@ from tensorrt_llm._torch._experimental.modeling_v2._native import namespace_of -_NS = namespace_of(__package__) +_NS = namespace_of(__name__) @torch.library.register_fake(f"{_NS}::swizzle_block_scales") diff --git a/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_claims.py b/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_claims.py index 12d42447874d..208fc08c05a3 100644 --- a/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_claims.py +++ b/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_claims.py @@ -25,7 +25,7 @@ import pytest -from tensorrt_llm._torch._experimental.modeling_v2._native import identifier, torch_namespace +from tensorrt_llm._torch._experimental.modeling_v2 import _native _REPO = Path(__file__).resolve().parents[4] # What every family shares: the discovery and the common/ macros. @@ -491,9 +491,9 @@ def test_cmake_and_python_agree_on_names_and_architectures(tmp_path): ["cmake", "-P", str(script)], capture_output=True, text=True, check=True ).stdout for name in names: - assert f"name {name} {identifier(name)}" in output, output + assert f"name {name} {_native._identifier(name)}" in output, output assert "archs 90a-real;100f-real;103a-real" in output, output # The family namespace the build registers is the loader's. - assert torch_namespace("gpt_oss", "gpt_oss_120b__sm_10x__default") == ( + assert _native._namespace("gpt_oss", "gpt_oss_120b__sm_10x__default") == ( "mv2_gpt_oss_gpt_oss_120b_sm_10x_default" ) diff --git a/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_loader.py b/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_loader.py index 2106a0c35093..d35c77d78f17 100644 --- a/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_loader.py +++ b/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_loader.py @@ -28,23 +28,16 @@ from tensorrt_llm._torch._experimental.modeling_v2 import _native from tensorrt_llm._torch._experimental.modeling_v2._native import ( NativeLoadError, - family_of, - identifier, - library_name, - load_family_library, load_family_ops, load_target_ops, namespace_of, - split_target_package, - torch_namespace, ) from tensorrt_llm._torch._experimental.modeling_v2._router_index import NoTarget pytestmark = pytest.mark.cpu_only -_TARGET = ( - "tensorrt_llm._torch._experimental.modeling_v2.models.gpt_oss.gpt_oss_120b__sm_10x__default" -) +_FAMILY = "tensorrt_llm._torch._experimental.modeling_v2.models.gpt_oss" +_TARGET = f"{_FAMILY}.gpt_oss_120b__sm_10x__default" @pytest.fixture @@ -59,7 +52,7 @@ def libs(monkeypatch, tmp_path): def _install(libs: Path, monkeypatch, opens) -> None: """Put a gpt_oss library in `libs`; `opens` stands in for dlopen, and no fake_ops module is imported, since the stand-in defines no op.""" - (libs / library_name("gpt_oss")).write_bytes(b"") + (libs / "libth_modeling_v2_gpt_oss.so").write_bytes(b"") monkeypatch.setattr(_native, "_open_library", lambda path, family: opens()) monkeypatch.setattr(_native, "_import_fakes", lambda module, family: None) @@ -67,18 +60,32 @@ def _install(libs: Path, monkeypatch, opens) -> None: # --- Naming ---------------------------------------------------------------------- -def test_names_of_libraries_and_namespaces(): - assert library_name("minimax_m3") == "libth_modeling_v2_minimax_m3.so" - assert identifier("gpt_oss_120b__sm_10x__default") == "gpt_oss_120b_sm_10x_default" - assert torch_namespace("gpt_oss") == "mv2_gpt_oss" - assert namespace_of(_TARGET) == "mv2_gpt_oss_gpt_oss_120b_sm_10x_default" - assert split_target_package(_TARGET) == ("gpt_oss", "gpt_oss_120b__sm_10x__default") - with pytest.raises(ValueError): - split_target_package("tensorrt_llm._torch.models") - assert family_of(f"{_TARGET}.weights") == "gpt_oss" - assert family_of(_TARGET.rpartition(".")[0] + ".fake_ops") == "gpt_oss" +@pytest.mark.parametrize( + "module, namespace", + [ + (_TARGET, "mv2_gpt_oss_gpt_oss_120b_sm_10x_default"), + (f"{_TARGET}.fake_ops", "mv2_gpt_oss_gpt_oss_120b_sm_10x_default"), + (f"{_TARGET}.ops.moe.entry", "mv2_gpt_oss_gpt_oss_120b_sm_10x_default"), + (f"{_FAMILY}.fake_ops", "mv2_gpt_oss"), + (f"{_FAMILY}.ops.comm.entry", "mv2_gpt_oss"), + ], +) +def test_namespace_of_a_target_or_family_module(module, namespace): + assert namespace_of(module) == namespace + + +@pytest.mark.parametrize( + "module", ["tensorrt_llm._torch.models", "tensorrt_llm._torch._experimental.modeling_v2.models"] +) +def test_namespace_of_rejects_modules_outside_a_family(module): with pytest.raises(ValueError): - family_of("tensorrt_llm._torch._experimental.modeling_v2.models") + namespace_of(module) + + +@pytest.mark.parametrize("package", [_FAMILY, f"{_TARGET}.weights"]) +def test_load_target_ops_takes_a_target_package_only(package): + with pytest.raises(ValueError, match="models.."): + load_target_ops(package) # --- Decisions ----------------------------------------------------------------- @@ -112,8 +119,8 @@ def opens(): return frozenset({"gpt_oss_120b__sm_10x__default"}) _install(libs, monkeypatch, opens) - assert load_family_library("gpt_oss") == {"gpt_oss_120b__sm_10x__default"} - assert load_family_library("gpt_oss") == {"gpt_oss_120b__sm_10x__default"} + assert _native._load_family_library("gpt_oss") == {"gpt_oss_120b__sm_10x__default"} + assert _native._load_family_library("gpt_oss") == {"gpt_oss_120b__sm_10x__default"} assert len(opened) == 1 @@ -158,7 +165,7 @@ def test_family_ops_of_a_family_this_installation_lacks_are_no_target(libs): def test_windows_has_no_native_targets(libs, monkeypatch): monkeypatch.setattr(sys, "platform", "win32") with pytest.raises(NoTarget, match="Windows"): - load_family_library("gpt_oss") + _native._load_family_library("gpt_oss") # --- The installed wheel --------------------------------------------------------- @@ -170,7 +177,7 @@ def test_windows_has_no_native_targets(libs, monkeypatch): import tensorrt_llm # noqa: F401 -- loads libth_common.so from tensorrt_llm._torch._experimental.modeling_v2 import _native - built = sorted(_native.load_family_library(sys.argv[1])) + built = sorted(_native._load_family_library(sys.argv[1])) schemas = sorted(s.name for s in torch._C._jit_get_all_schemas() if s.name.startswith("mv2_")) print(json.dumps({"built": built, "ops": schemas})) """ @@ -198,7 +205,7 @@ def test_every_installed_family_library_loads(): assert (models / family / target / "__init__.py").is_file(), ( f"{library.name} carries {target}, which has no Python target package" ) - prefix = f"{torch_namespace(family, target)}::" + prefix = f"{_native._namespace(family, target)}::" target_ops = [op[len(prefix) :] for op in report["ops"] if op.startswith(prefix)] fake_ops = models / family / target / "fake_ops.py" text = fake_ops.read_text() if fake_ops.is_file() else "" From 58c47ce8d2eadb789196641595f2c3a25ae8ad4e Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Mon, 26 Oct 2026 19:36:07 -0700 Subject: [PATCH 13/20] [None][chore] modeling_v2: leave modeling_v2_resolve as it was modeling_v2_resolve_on_all_ranks no longer needs routing split out of modeling_v2_resolve: it calls modeling_v2_resolve on each rank and exchanges which target, if any, each rank builds and any error it raised. The resolver is back to main's, and the change to _router_index.py is the new function alone. A fallback's reason stays at INFO on each rank, as before; rank 0 logs the target every rank builds. Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- .../modeling_v2/_router_index.py | 77 ++++++------------- .../test_modeling_v2_rank_agreement.py | 24 +++--- 2 files changed, 38 insertions(+), 63 deletions(-) diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/_router_index.py b/tensorrt_llm/_torch/_experimental/modeling_v2/_router_index.py index 339208634476..3814683f7fd7 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/_router_index.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/_router_index.py @@ -175,19 +175,21 @@ def engine_kv_cache_quant_algo(config: "ModelConfig", llm_args: "TorchLlmArgs"): return {"fp8": QuantAlgo.FP8, "nvfp4": QuantAlgo.NVFP4}.get(dtype, dtype) -def _route( - config: "ModelConfig", llm_args: "TorchLlmArgs", mode: ModelingV2Mode -) -> Tuple[Optional[type], Optional[str]]: - """``(target class, None)``, or ``(None, why)`` for the built-in implementation. - - ``why`` is None when the checkpoint names no architecture, which is not a - fallback worth reporting. Under ``require`` a missing target raises instead. - A target's native code loads when ``route`` imports the target (see - ``_native``), so its ``NoTarget`` lands here like any other. +def modeling_v2_resolve(config: "ModelConfig", llm_args: "TorchLlmArgs") -> Optional[type]: + """Decide which modeling_v2 target, if any, this deployment builds. + + Returns the target class, or None for the built-in implementation: + when ``llm_args.modeling_v2`` is ``off``, when no ``route`` is registered + for the architecture, or when that ``route`` raises ``NoTarget``. + In ``require`` every one of those but ``off`` raises ``ValueError`` instead. """ + mode = ModelingV2Mode.of(llm_args) + if mode is ModelingV2Mode.OFF: + return None + pretrained_config = config.pretrained_config if not getattr(pretrained_config, "architectures", None): - return None, None + return None arch = pretrained_config.architectures[0] try: @@ -195,31 +197,14 @@ def _route( if route is None: known = ", ".join(sorted(registered_routes())) or "(none)" raise NoTarget(f"no route for this architecture; routed architectures: {known}") - return route(config, llm_args, _device_compute_capability()), None + return route(config, llm_args, _device_compute_capability()) except NoTarget as why: if mode is ModelingV2Mode.REQUIRE: raise ValueError(f"modeling_v2='require', but no target fits {arch!r}: {why}") from None - return None, f"no target fits {arch!r} ({why})" - - -def modeling_v2_resolve(config: "ModelConfig", llm_args: "TorchLlmArgs") -> Optional[type]: - """Decide which modeling_v2 target, if any, this deployment builds. - - Returns the target class, or None for the built-in implementation: - when ``llm_args.modeling_v2`` is ``off``, when no ``route`` is registered - for the architecture, or when that ``route`` raises ``NoTarget``. - In ``require`` every one of those but ``off`` raises ``ValueError`` instead. - - This decides for the calling rank only; the model loader uses - :func:`modeling_v2_resolve_on_all_ranks`. - """ - mode = ModelingV2Mode.of(llm_args) - if mode is ModelingV2Mode.OFF: + logger.info( + f"modeling_v2: no target fits {arch!r} ({why}); using the built-in implementation" + ) return None - target, fallback = _route(config, llm_args, mode) - if fallback is not None: - logger.info(f"modeling_v2: {fallback}; using the built-in implementation") - return target def _qualified(target: Optional[type]) -> Optional[str]: @@ -240,20 +225,15 @@ def modeling_v2_resolve_on_all_ranks( ``off`` returns before any exchange: nothing in this package runs then. """ - mode = ModelingV2Mode.of(llm_args) - if mode is ModelingV2Mode.OFF: + if ModelingV2Mode.of(llm_args) is ModelingV2Mode.OFF: return None - target = fallback = error = None + target = error = None try: - target, fallback = _route(config, llm_args, mode) + target = modeling_v2_resolve(config, llm_args) except Exception as caught: # noqa: BLE001 -- re-raised below, after every rank has reported error = caught - outcome = ( - _qualified(target), - fallback, - None if error is None else f"{type(error).__name__}: {error}", - ) + outcome = (_qualified(target), None if error is None else f"{type(error).__name__}: {error}") mapping = config.mapping outcomes = [outcome] @@ -264,25 +244,18 @@ def modeling_v2_resolve_on_all_ranks( if error is not None: raise error - failed = {rank: o[2] for rank, o in enumerate(outcomes) if o[2] is not None} + failed = {rank: why for rank, (_, why) in enumerate(outcomes) if why is not None} if failed: detail = "; ".join(f"rank {rank}: {why}" for rank, why in sorted(failed.items())) raise RuntimeError(f"modeling_v2: other ranks failed to resolve a target ({detail})") - if len({o[0] for o in outcomes}) > 1: + if len({built for built, _ in outcomes}) > 1: detail = "; ".join( - f"rank {rank}: {o[0] or 'built-in (' + str(o[1]) + ')'}" - for rank, o in enumerate(outcomes) + f"rank {rank}: {built or 'built-in'}" for rank, (built, _) in enumerate(outcomes) ) raise RuntimeError( f"modeling_v2: ranks disagree on the implementation to build ({detail}); " "every rank has to run the same code" ) - - if mapping.rank == 0: - if target is not None: - logger.warning( - f"modeling_v2: building {_qualified(target)} on all {len(outcomes)} rank(s)" - ) - elif fallback is not None: - logger.warning(f"modeling_v2: {fallback}; using the built-in implementation") + if target is not None and mapping.rank == 0: + logger.warning(f"modeling_v2: building {_qualified(target)} on all {len(outcomes)} rank(s)") return target diff --git a/tests/unittest/_torch/modeling_v2/test_modeling_v2_rank_agreement.py b/tests/unittest/_torch/modeling_v2/test_modeling_v2_rank_agreement.py index 27db451ec8fe..b296dedc1f01 100644 --- a/tests/unittest/_torch/modeling_v2/test_modeling_v2_rank_agreement.py +++ b/tests/unittest/_torch/modeling_v2/test_modeling_v2_rank_agreement.py @@ -56,13 +56,15 @@ def dist(monkeypatch): return fake -def _route_to(monkeypatch, target=None, fallback=None, error=None): - def route(config, llm_args, mode): +def _route_to(monkeypatch, target=None, error=None): + """This rank's own decision: what ``modeling_v2_resolve`` returns or raises.""" + + def resolve(config, llm_args): if error is not None: raise error - return target, fallback + return target - monkeypatch.setattr(_router_index, "_route", route) + monkeypatch.setattr(_router_index, "modeling_v2_resolve", resolve) def test_off_exchanges_nothing(monkeypatch, dist): @@ -78,27 +80,27 @@ def test_one_rank_exchanges_nothing(monkeypatch, dist): def test_agreeing_ranks_build_the_target(monkeypatch, dist): - dist.others = [(_TARGET, None, None)] + dist.others = [(_TARGET, None)] _route_to(monkeypatch, target=Target) assert modeling_v2_resolve_on_all_ranks(_config(), _args("require")) is Target assert dist.calls == 1 def test_agreeing_fallbacks_build_the_builtin_model(monkeypatch, dist): - dist.others = [(None, "no target fits 'X' (needs sm_107)", None)] - _route_to(monkeypatch, fallback="no target fits 'X' (needs sm_107)") + dist.others = [(None, None)] + _route_to(monkeypatch) assert modeling_v2_resolve_on_all_ranks(_config(), _args("auto")) is None def test_disagreeing_ranks_all_fail(monkeypatch, dist): - dist.others = [(None, "no target fits 'X' (native code not built)", None)] + dist.others = [(None, None)] _route_to(monkeypatch, target=Target) - with pytest.raises(RuntimeError, match="ranks disagree"): + with pytest.raises(RuntimeError, match="ranks disagree.*rank 1: built-in"): modeling_v2_resolve_on_all_ranks(_config(), _args("auto")) def test_another_ranks_error_fails_this_rank(monkeypatch, dist): - dist.others = [(None, None, "NativeLoadError: libth_modeling_v2_x.so cannot be loaded")] + dist.others = [(None, "NativeLoadError: libth_modeling_v2_x.so cannot be loaded")] _route_to(monkeypatch, target=Target) with pytest.raises(RuntimeError, match="rank 1: NativeLoadError"): modeling_v2_resolve_on_all_ranks(_config(), _args("auto")) @@ -106,7 +108,7 @@ def test_another_ranks_error_fails_this_rank(monkeypatch, dist): def test_this_ranks_error_is_raised_after_the_exchange(monkeypatch, dist): """Raising before the exchange would leave the other ranks waiting in it.""" - dist.others = [(_TARGET, None, None)] + dist.others = [(_TARGET, None)] _route_to(monkeypatch, error=ValueError("modeling_v2='require', but no target fits")) with pytest.raises(ValueError, match="no target fits"): modeling_v2_resolve_on_all_ranks(_config(), _args("require")) From 587ccd0d292ef09406f2714e325c0671e7ac66cb Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Mon, 26 Oct 2026 19:39:24 -0700 Subject: [PATCH 14/20] [None][chore] modeling_v2: fold the native-code guide into the README NATIVE.md repeated what the code and its checks already state. Its essentials -- where native code lives, the five steps to add it to a target, shared code, architectures and other keywords, the two load failures, building one family -- are now a short "Native code" section of the modeling_v2 README, placed after the entry and test layout it builds on, and NATIVE.md is removed. Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- cpp/cmake/modules/modeling_v2.cmake | 6 +- cpp/tensorrt_llm/modeling_v2/README.md | 4 +- .../_experimental/modeling_v2/NATIVE.md | 248 ------------------ .../_experimental/modeling_v2/README.md | 73 ++++-- .../_experimental/modeling_v2/_native.py | 2 +- 5 files changed, 63 insertions(+), 270 deletions(-) delete mode 100644 tensorrt_llm/_torch/_experimental/modeling_v2/NATIVE.md diff --git a/cpp/cmake/modules/modeling_v2.cmake b/cpp/cmake/modules/modeling_v2.cmake index af5b10bfd716..8cb64739272a 100644 --- a/cpp/cmake/modules/modeling_v2.cmake +++ b/cpp/cmake/modules/modeling_v2.cmake @@ -29,9 +29,9 @@ Each family builds into one torch-op library, libth_modeling_v2_.so, installed next to libth_common.so. Nothing in it is linked into libtensorrt_llm.so or libth_common.so; the Python loader (tensorrt_llm/_torch/_experimental/modeling_v2/_native.py) opens it only when -routing picks one of the family's targets. modeling_v2/NATIVE.md is the guide -for authors; cpp/tensorrt_llm/modeling_v2/ holds only what every family shares -(this discovery, the common/ macros). +routing picks one of the family's targets. The "Native code" section of the +modeling_v2 README is the guide for authors; cpp/tensorrt_llm/modeling_v2/ +holds only what every family shares (this discovery, the common/ macros). A family's csrc/CMakeLists.txt uses three functions; names are the Python package names: diff --git a/cpp/tensorrt_llm/modeling_v2/README.md b/cpp/tensorrt_llm/modeling_v2/README.md index fd5b6395679b..301cb1167af2 100644 --- a/cpp/tensorrt_llm/modeling_v2/README.md +++ b/cpp/tensorrt_llm/modeling_v2/README.md @@ -6,8 +6,8 @@ Each modeling_v2 model family keeps its C++/CUDA with its Python code, in `tensorrt_llm/_torch/_experimental/modeling_v2/models//csrc/` and `//csrc/`, and builds into its own torch-op library, -`libth_modeling_v2_.so`. The guide for authors is -`tensorrt_llm/_torch/_experimental/modeling_v2/NATIVE.md`. +`libth_modeling_v2_.so`. The guide for authors is the "Native code" +section of `tensorrt_llm/_torch/_experimental/modeling_v2/README.md`. This directory holds only what every family library shares; adding a model changes nothing here: diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/NATIVE.md b/tensorrt_llm/_torch/_experimental/modeling_v2/NATIVE.md deleted file mode 100644 index 1c677942e105..000000000000 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/NATIVE.md +++ /dev/null @@ -1,248 +0,0 @@ - - - -# Native code of modeling_v2 targets - -C++/CUDA that only modeling_v2 targets use lives with the Python code of its -model family, in this directory tree, not in `cpp/tensorrt_llm/kernels/` or -`thop/`. Each family builds into one torch-op library, -`libth_modeling_v2_.so`, installed next to `libth_common.so`. - -- **Not linked into stock.** Nothing in a family library is linked into - `libtensorrt_llm.so` or `libth_common.so`. -- **Loaded on demand.** The library opens only when routing picks one of the - family's targets. With `modeling_v2=off` (the default), nothing loads. -- **No stock file per model.** Adding a model, its kernels included, edits - nothing under `cpp/`, nor `setup.py` or `scripts/build_wheel.py`: the build - finds every `models//csrc/CMakeLists.txt` by itself. - -`models/gpt_oss/` is the worked example: its target -`gpt_oss_120b__sm_10x__default` carries one kernel, the MXFP4 block-scale -swizzle its weight loader runs. - -## Onboarding a model with native code - -The files a new family `foo` with one target adds. Everything except the last -three rows is inside its own directories: - -``` -tensorrt_llm/_torch/_experimental/modeling_v2/models/foo/ - __init__.py docstring only - routing.py @routes("FooForCausalLM") def route(...) - fake_ops.py fakes of the family's shared ops, if it has any - csrc/ - CMakeLists.txt add_modeling_v2_family / _shared / _target - *.cu *.cuh *.h *.cpp code several targets of the family share, if any - foo_nvfp4__sm_103__tp4/ ____ - __init__.py native_ops = load_target_ops(__name__) - modeling.py weights.py - fake_ops.py a fake for every op of csrc/ - ops//.py entries only this target calls - csrc/ - *.cu *.cuh *.h kernels, compiled for ARCHS without torch - *Op.cpp torch bindings -tests/unittest/_torch/modeling_v2/models/foo/foo_nvfp4__sm_103__tp4/test_modeling_v2_*.py -tests/integration/defs/accuracy/modeling_v2/test_foo.py - -models/__init__.py + from .foo import routing as foo_routing -tests/integration/test_lists/test-db/l0_.yml - + unittest/_torch/modeling_v2/models/foo -.github/CODEOWNERS optional: the family's owners -``` - -Naming is the Python package names throughout: family `foo`, target -`foo_nvfp4__sm_103__tp4`, its torch namespace `mv2_foo_foo_nvfp4_sm_103_tp4`. -`tests/unittest/_torch/modeling_v2/test_modeling_v2_native_claims.py` checks -every rule below on CPU; run it before sending the PR. - -## Step by step - -The gpt_oss files are the example for each step. - -1. **Declare the family and the target** in `models/foo/csrc/CMakeLists.txt` - (`models/gpt_oss/csrc/CMakeLists.txt`): - - ```cmake - add_modeling_v2_family(foo) - - add_modeling_v2_target( - foo foo_nvfp4__sm_103__tp4 - ARCHS 103a - SOURCES *.cu *.cpp - REQUIRES MULTI_DEVICE) - ``` - -2. **Write the kernel** in `foo_nvfp4__sm_103__tp4/csrc/`, in the target's C++ - namespace (`gpt_oss_120b__sm_10x__default/csrc/swizzleBlockScales.{h,cu}`): - - ```cpp - #include "tensorrt_llm/modeling_v2/common/namespace.h" - - TRTLLM_MV2_NAMESPACE_BEGIN // tensorrt_llm::modeling_v2::foo::foo_nvfp4_sm_103_tp4 - namespace kernels - { - cudaError_t invokeMyOp(/* ... */, cudaStream_t stream); - } - TRTLLM_MV2_NAMESPACE_END - ``` - -3. **Bind it to torch** in the same `csrc/`, through the target's torch - namespace (`swizzleBlockScalesOp.cpp`). Include the family's own headers - from `models/`: - - ```cpp - #include "foo/foo_nvfp4__sm_103__tp4/csrc/myOp.h" - #include "tensorrt_llm/modeling_v2/common/torchLibrary.h" - - TRTLLM_MV2_LIBRARY_FRAGMENT(m) { m.def("my_op(Tensor x) -> Tensor"); } - TRTLLM_MV2_LIBRARY_IMPL(CUDA, m) { m.impl("my_op", &TRTLLM_MV2_NS::myOp); } - ``` - -4. **Load it** from the target package's `__init__.py`: - - ```python - from tensorrt_llm._torch._experimental.modeling_v2._native import load_target_ops - - native_ops = load_target_ops(__name__) # torch.ops.mv2_foo_foo_nvfp4_sm_103_tp4 - ``` - - The target's modules use it as `from . import native_ops` - (`gpt_oss_120b__sm_10x__default/weights.py`). The name is not `ops`: that - is the target's `ops/` package. - -5. **Fake every op** in the target's `fake_ops.py`; the loader imports it right - after it finds the target in the library: - - ```python - from tensorrt_llm._torch._experimental.modeling_v2._native import namespace_of - - _NS = namespace_of(__name__) - - @torch.library.register_fake(f"{_NS}::my_op") - def _(x): - return torch.empty_like(x) - ``` - -6. **Test it** under `tests/unittest/_torch/modeling_v2/models/foo/`, and list - the family's directory in the test list of the GPU its SM names; accuracy - gates go in `accuracy/modeling_v2/test_foo.py` - (`test_modeling_v2_gpt_oss_swizzle_block_scales.py`, and - `unittest/_torch/modeling_v2/models/gpt_oss` in `l0_b300.yml`). CBTS - selects a family's tests by these two names (see [CI](#ci)). - -## Code the family's targets share - -When several targets call the same op, its code goes once in the family's -`csrc/`, next to its `CMakeLists.txt`, and its ops are in the family's torch -namespace, `torch.ops.mv2_`: - -```cmake -add_modeling_v2_shared( - foo - ARCHS 100f - SOURCES *.cu *.cpp - REQUIRES MULTI_DEVICE) - -# Targets with no code of their own declare no SOURCES and have no csrc/; -# declaring them is what makes the family library carry them. -add_modeling_v2_target(foo foo_nvfp4__sm_100__tp8 ARCHS 100f) -``` - -- **Fakes** of the shared ops go in the family's `models/foo/fake_ops.py`, - with `namespace_of(__name__)` as in a target's; the loader imports it when it - opens the library. -- **Callers** anywhere in the family use - `load_family_ops(__name__).my_op(...)`. -- **A target without code of its own** is skipped whenever the shared code is, - so the library never carries a target whose ops it lacks. - -Sharing moves code up, never sideways: target `csrc/`, then the family's -`csrc/`, then stock (`cpp/tensorrt_llm/kernels` + `thop`) once a built-in -model calls it, through normal stock review. A target never includes or calls -a sibling target's code. - -## CMake API - -Defined in `cpp/cmake/modules/modeling_v2.cmake`, in the style of -`add_th_common_family`. - -| Function | What it declares | -|---|---| -| `add_modeling_v2_family()` | The family; first call in `models//csrc/CMakeLists.txt`. | -| `add_modeling_v2_target( ARCHS ... [SOURCES ...] [DEPS ...] [REQUIRES ...])` | One target's code, in `/csrc/`; without `SOURCES`, a target that runs only its family's shared code. | -| `add_modeling_v2_shared( ARCHS ... SOURCES ... [DEPS ...] [REQUIRES ...])` | Code several targets call, in the family's `csrc/`; its ops are in `torch.ops.mv2_`. | - -| Keyword | Meaning | -|---|---| -| `ARCHS` | `NN`, `NNa` or `NNf`, intersected with the build's requested architectures, as flash_mla and deep_ep choose theirs. A target whose intersection is empty is **not built** (the configure step prints why) and is a `NoTarget` at run time. The architectures must cover the SM in the target's name. | -| `SOURCES` | Paths or globs relative to the unit's `csrc/`. `.cu` files compile for `ARCHS` without torch; `.cpp` files are the torch bindings. A per-file option goes on the file in the family's `CMakeLists.txt`: `set_source_files_properties(x.cu PROPERTIES COMPILE_OPTIONS --use_fast_math)`. | -| `DEPS` | `cublas`, `cublaslt`, `cuda_driver`, `internal_cutlass`, `nccl`, `trtllm_gen_fmha_interface`. Anything else is an addition to the module. | -| `REQUIRES` | `MULTI_DEVICE`, `NVSHMEM`. A target the build cannot satisfy is skipped. | - -**Build options.** `-DBUILD_MODELING_V2=OFF` builds no family library. -`-DTRTLLM_MODELING_V2_TARGETS=/;...` (globs allowed, e.g. -`gpt_oss/*`) builds only those; pass it with -`scripts/build_wheel.py -D "TRTLLM_MODELING_V2_TARGETS=..."`. A family library -is its own build target, so a kernel edit relinks only it: -`cmake --build cpp/build --target th_modeling_v2_`. - -## Namespaces - -| Kind | Name | -|---|---| -| Target torch ops | `torch.ops.mv2__`, with `__` collapsed to `_` | -| Family shared torch ops | `torch.ops.mv2_`; also holds `built_targets()` | -| Target C++ | `tensorrt_llm::modeling_v2::::` (`TRTLLM_MV2_NAMESPACE_BEGIN`) | -| Shared C++ | `tensorrt_llm::modeling_v2::::shared` | - -Each torch namespace is claimed exactly once, by the family library's generated -registration, so a second library claiming it fails at load. Op files only add -fragments through the `TRTLLM_MV2_*` macros; raw `TORCH_LIBRARY*` is rejected. -Every target has its own C++ namespace and the library exports no symbol, so -sibling targets can share file and function names. - -## What native code may include - -- Its own target's `csrc/` and its family's `csrc/`, written from `models/`: - `"foo/foo_nvfp4__sm_103__tp4/csrc/x.h"`, `"foo/csrc/y.h"`. -- `tensorrt_llm/modeling_v2/common/` (the macros). -- Any stock header (`cpp/include/`, `cpp/`), and system, CUDA, torch and cutlass - headers. - -It may not include another target's or another family's code. - -It links against `libtensorrt_llm.so` (logger, `TllmException`, mcast registry -and other singletons stay there), `th_utils` and torch. **When a target needs -stock to change**, e.g. a new option on a stock kernel, that is a separate -stock PR that leaves stock behavior unchanged by default. - -## How it loads - -`load_target_ops(__name__)` treats a family library the way stock code treats -its optional native modules (`flash_mla`, `deep_ep`): - -| Outcome | Raised as | Meaning | -|---|---|---| -| The library is not installed, or does not carry the target | `NoTarget` | This build did not compile it; `auto` keeps the built-in model, `require` reports why | -| The library is installed but fails to open, or an op a `fake_ops.py` declares is missing | `NativeLoadError` | A broken installation; never a fallback | - -Packaging follows `flash_mla` and `deep_ep` too: each family writes -`cuda_architectures.txt` into its build directory, and `scripts/build_wheel.py` -installs the library only when that list is not empty. - -## CI - -| PR changes only… | CI runs… | -|---|---| -| One family (`models//`, its `csrc/` included) | That family's test-db entries plus the modeling_v2 CPU checks | -| Shared modeling_v2 code (catalog, loader, resolver, `models/__init__.py`, `cpp/tensorrt_llm/modeling_v2/`, the CMake module) | Every modeling_v2 entry | -| Stock code | The full suite | - -The narrowing is safe because family libraries are never in the stock -libraries, and the CPU checks that keep it so run in every scope: -`test_modeling_v2_native_claims.py` (native code only in `csrc/`, stock never -names a family, no include reaches another target or family, every op has a -fake, license headers), -`test_modeling_v2_native_loader.py` and `test_modeling_v2_rank_agreement.py`. -Keep a family's pre-merge entry to one short test on hardware matching its SM; -the accuracy matrix goes post-merge. diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/README.md b/tensorrt_llm/_torch/_experimental/modeling_v2/README.md index 8aff69b93dd8..062c089f3b7a 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/README.md +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/README.md @@ -112,7 +112,6 @@ checkpoint of the same shape is ungated. ``` _router_index.py architectures[0] -> routing module. Small, stable, test-guarded. _native.py loads a target's native code (see "Native code" below) -NATIVE.md the guide to a target's native (C++/CUDA) code models// routing.py one forward-reading decision tree per architecture family fake_ops.py fakes of the family's shared native ops, if it has any @@ -125,21 +124,6 @@ models// catalog// entries more than one target calls ``` -### Native code - -A target's C++/CUDA lives with its Python code: `/csrc/`, and the -family's `csrc/` for code its targets share, declared by the family's -`csrc/CMakeLists.txt`. It builds into one torch-op library per family, -`libth_modeling_v2_.so`, installed next to `libth_common.so`. Nothing of -it is linked into `libtensorrt_llm.so` or `libth_common.so`, and nothing loads -it until routing imports one of the family's targets, whose `__init__.py` -starts with `native_ops = load_target_ops(__name__)`. As with the stock code's -optional native modules, a library this installation does not carry is a -`NoTarget`, so `auto` keeps the built-in model; one that is installed but -broken is a `NativeLoadError`, never a fallback. Adding a model with kernels -edits no file under `cpp/`. `NATIVE.md` is the guide, with the directories and -files a new model adds; `models/gpt_oss/` is the worked example. - An entry, shared or owned, is one `OpWrapper` subclass: `__call__` is the op call a target makes, `reference` is what it is supposed to compute in plain torch, `is_valid` names the inputs it accepts and answers wrongly, and `CELLS` @@ -185,6 +169,63 @@ read as a hierarchy that was never one -- every level had exactly one child, and each needed an `__init__.py` whose only job was to exist. The class name carries the same triple; `test_modeling_v2_claims.py` asserts they agree. +### Native code + +A target's C++/CUDA lives with its Python code: `/csrc/`, and the family's +`csrc/` for code several targets share. Each family builds into one torch-op library, +`libth_modeling_v2_.so`, installed next to `libth_common.so`; nothing of it is +linked into the stock libraries, and it loads only when routing picks one of the +family's targets. Adding a model with kernels edits nothing under `cpp/`. +`models/gpt_oss/` is the example: one kernel, used by its weight loader. + +Adding native code to target `foo_nvfp4__sm_103__tp4` of family `foo`: + +1. **Declare it** in `models/foo/csrc/CMakeLists.txt`: + + ```cmake + add_modeling_v2_family(foo) + add_modeling_v2_target(foo foo_nvfp4__sm_103__tp4 ARCHS 103a SOURCES *.cu *.cpp) + ``` + +2. **Write the kernels and their torch bindings** in `foo_nvfp4__sm_103__tp4/csrc/`, in + the target's namespaces: + + ```cpp + TRTLLM_MV2_NAMESPACE_BEGIN // tensorrt_llm::modeling_v2::foo::foo_nvfp4_sm_103_tp4 + ... + TRTLLM_MV2_NAMESPACE_END + TRTLLM_MV2_LIBRARY_FRAGMENT(m) { m.def("my_op(Tensor x) -> Tensor"); } + TRTLLM_MV2_LIBRARY_IMPL(CUDA, m) { m.impl("my_op", &TRTLLM_MV2_NS::myOp); } + ``` + + Include the family's own headers from `models/` (`"foo/foo_nvfp4__sm_103__tp4/csrc/x.h"`); + any stock header is fine, another target's or family's code is not. + +3. **Load it** from the target's `__init__.py`, `native_ops = load_target_ops(__name__)` + (from `_native`); the target's modules use `from . import native_ops`. + +4. **Fake every op** in the target's `fake_ops.py`, which the loader imports right after + the library opens: `@torch.library.register_fake(f"{namespace_of(__name__)}::my_op")`. + +5. **Test it** under `tests/unittest/_torch/modeling_v2/models/foo/`, and list + `unittest/_torch/modeling_v2/models/foo` in the test list of the GPU its SM names. + +Code several targets call goes once in the family's `csrc/`, declared with +`add_modeling_v2_shared(foo ARCHS ... SOURCES ...)`; its ops are in `torch.ops.mv2_foo`, +reached with `load_family_ops(__name__)` and faked in `models/foo/fake_ops.py`. A target +with no code of its own declares no `SOURCES`. + +- **`ARCHS`** (`NN`, `NNa`, `NNf`) is intersected with the build's architectures; a target + with none is not built, and is a `NoTarget` at run time. `REQUIRES` (`MULTI_DEVICE`, + `NVSHMEM`) and `DEPS` (a closed list) are the other keywords; a per-file compile option + goes on the file with `set_source_files_properties`. +- **Loading**: a library this installation does not carry, or a target it did not build, + is a `NoTarget` (`auto` keeps the built-in model); one that is installed but fails to + load, or lacks an op a `fake_ops.py` declares, is a `NativeLoadError`. +- **Building only your family**: `build_wheel.py -D "TRTLLM_MODELING_V2_TARGETS=foo/*"`, and + after an edit `cmake --build cpp/build --target th_modeling_v2_foo`. +- `test_modeling_v2_native_claims.py` checks these rules on CPU; run it before sending the PR. + ### Default and specialized targets The parallel segment of a target's name is either one topology (for diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/_native.py b/tensorrt_llm/_torch/_experimental/modeling_v2/_native.py index e945acc474a1..c76fb1b5caba 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/_native.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/_native.py @@ -26,7 +26,7 @@ never a fallback. The family's ``fake_ops.py`` registers when the library opens, a target's when the target is found in it. -``NATIVE.md`` next to this module is the guide for authors. +The "Native code" section of the modeling_v2 README is the guide for authors. """ from __future__ import annotations From 09193e4d64380bd799a2e41099b815ebec7e9732 Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Mon, 26 Oct 2026 19:58:15 -0700 Subject: [PATCH 15/20] [None][chore] modeling_v2: keep one B300 directory entry for the unit tests l0_b300.yml runs the modeling_v2 unit tests through its one directory entry again, which collects every family's tests under models/, so a new family changes no test list and cannot be left out by forgetting one. ModelingV2Rule counts a directory entry that collects a family's unit tests (and does not --ignore them) toward that family's scope. The claims check now requires that some entry runs a native family's tests rather than that each family has an entry of its own. The directory entry's waive (nvbug 6853741) is removed. Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- jenkins/scripts/cbts/README.md | 2 +- jenkins/scripts/cbts/cbts/rules/README.md | 22 +++--- .../scripts/cbts/cbts/rules/modeling_v2.py | 43 +++++++----- .../_experimental/modeling_v2/README.md | 5 +- .../test_lists/test-db/l0_b300.yml | 13 +--- tests/integration/test_lists/waives.txt | 4 -- .../test_modeling_v2_native_claims.py | 36 +++++----- .../scripts/cbts/test_modeling_v2_rule.py | 68 ++++++++++++++----- 8 files changed, 116 insertions(+), 77 deletions(-) diff --git a/jenkins/scripts/cbts/README.md b/jenkins/scripts/cbts/README.md index 94e30eb33dc1..e1a45484f168 100644 --- a/jenkins/scripts/cbts/README.md +++ b/jenkins/scripts/cbts/README.md @@ -67,7 +67,7 @@ See `cbts/rules/README.md` for per-rule logic. | `testlistonly` | `TestListRule` fired solo: PR only adds entries under `tests/integration/test_lists/test-db/*.yml`. | | `visualgenonly` | `VisualGenRule` fired solo: PR only touches VisualGen internal source paths (`examples/visual_gen/**`, `scripts/visualgen_eval/**`, `tensorrt_llm/_torch/visual_gen/**`; excl. docs; image fixtures like `cat_piano.png` are claimed). Narrows to blocks containing VG test entries. Outward-facing files under `tensorrt_llm/visual_gen/**` and `tensorrt_llm/media/**` (eagerly imported by `trtllm-serve`) force `null` fallback. | | `specdeconly` | `SpecDecRule` fired solo: PR only touches speculative-decoding source paths (`tensorrt_llm/_torch/speculative/**`, `tensorrt_llm/models/{eagle,medusa,redrafter}/**`, `examples/{eagle,medusa,redrafter,draft_target_model,ngram}/**`, `examples/llm-api/llm_speculative_decoding.py`; excl. docs). Narrows to blocks containing spec-dec test entries (eagle / medusa / redrafter / ngram / draft-target-model / MTP). | -| `modelingv2only` | `ModelingV2Rule` fired solo: PR only touches modeling_v2 code -- the subtree (`tensorrt_llm/_torch/_experimental/modeling_v2/**`, each family's native code in its `csrc/` included: one torch-op library per model family) or what the family libraries share (`cpp/tensorrt_llm/modeling_v2/**`, `cpp/cmake/modules/modeling_v2.cmake`); excl. docs, which are a fifth of the Python subtree — every catalog entry carries a contract document. A change confined to one family's directory narrows to that family's entries (`accuracy/modeling_v2/test_.py`, `unittest/_torch/modeling_v2/models/`) plus the modeling_v2 CPU checks; any other modeling_v2 change narrows to every modeling_v2 entry. No outward-facing fallback is needed: nothing beyond the resolver module is imported, and no family library is loaded, unless `LLM(modeling_v2=...)` is on; the CPU checks that keep this true run in every scope. The callers outside the subtree (`_torch/pyexecutor/model_loader.py`, `_torch/models/modeling_auto.py`) are left unclaimed, so touching them falls back to baseline. | +| `modelingv2only` | `ModelingV2Rule` fired solo: PR only touches modeling_v2 code -- the subtree (`tensorrt_llm/_torch/_experimental/modeling_v2/**`, each family's native code in its `csrc/` included: one torch-op library per model family) or what the family libraries share (`cpp/tensorrt_llm/modeling_v2/**`, `cpp/cmake/modules/modeling_v2.cmake`); excl. docs, which are a fifth of the Python subtree — every catalog entry carries a contract document. A change confined to one family's directory narrows to the entries that run that family's tests (`accuracy/modeling_v2/test_.py`, and the unit entries that collect `unittest/_torch/modeling_v2/models/`) plus the modeling_v2 CPU checks; any other modeling_v2 change narrows to every modeling_v2 entry. No outward-facing fallback is needed: nothing beyond the resolver module is imported, and no family library is loaded, unless `LLM(modeling_v2=...)` is on; the CPU checks that keep this true run in every scope. The callers outside the subtree (`_torch/pyexecutor/model_loader.py`, `_torch/models/modeling_auto.py`) are left unclaimed, so touching them falls back to baseline. | | `agentflowonly` | `AgentFlowRule` fired solo: PR only touches `agent-flow/**` source or test files (excl. docs). Runs `CPU-AgentFlow-UnitTest`. | | `openengineonly` | `OpenEngineRule` fired solo: PR only touches `tensorrt_llm/grpc/openengine/**` source files (excl. docs). Narrows to the registered OpenEngine unit tests: the stub-based ones on the always-run `CPU-Generic-*` stages, plus `test_capability_conformance.py` on `A10-PyTorch-*`, which needs a GPU. | | `docsonly` | `DocsRule` fired solo: all documentation changes run `CPU-Build_Docs`; changes under `docs/**` also run the complete `l0_cpu` suite on matching `CPU-Generic-*` stages. Docs-only orchestration skips the SBSA track, so its resolved ARM stage is not launched. | diff --git a/jenkins/scripts/cbts/cbts/rules/README.md b/jenkins/scripts/cbts/cbts/rules/README.md index 0b4cff97d632..7e31df823161 100644 --- a/jenkins/scripts/cbts/cbts/rules/README.md +++ b/jenkins/scripts/cbts/cbts/rules/README.md @@ -310,12 +310,14 @@ plus what every family library shares: `cpp/tensorrt_llm/modeling_v2/` and the native build module `cpp/cmake/modules/modeling_v2.cmake`. Scope — per family, else all of modeling_v2: a change confined to one -family's directory (`models//`, its native code included) keeps that family's -entries -- `accuracy/modeling_v2/test_.py` and -`unittest/_torch/modeling_v2/models/` -- plus the modeling_v2 CPU -checks (`unittest/_torch/modeling_v2/test_modeling_v2_*.py`). Several -families union. Any other modeling_v2 change keeps every modeling_v2 entry. -A claimed family with no entry in any test-db yaml falls back to baseline. +family's directory (`models//`, its native code included) keeps the +entries that run that family's tests -- `accuracy/modeling_v2/test_.py`, +and any unit entry that collects `unittest/_torch/modeling_v2/models/` +(the modeling_v2 directory entry, which does not `--ignore` it, or an entry +under it) -- plus the modeling_v2 CPU checks +(`unittest/_torch/modeling_v2/test_modeling_v2_*.py`). Several families union. +Any other modeling_v2 change keeps every modeling_v2 entry. A claimed family +no entry runs falls back to baseline. `.md` exclusion carries more weight here than elsewhere: every catalog entry ships a contract document, so roughly a fifth of the subtree is @@ -330,10 +332,10 @@ in `backend: pytorch` blocks beside everything else. A block belongs to modeling_v2 iff one of its `tests:` entries matches `_MV2_ENTRY_PATTERNS`: -- `unittest/_torch/modeling_v2/` — the op-level catalog matrix (one - directory entry with `--ignore` of `comm/` and `models/`, plus `comm/` - on `l0_gb300_multi_gpus` for the 4-rank collectives) and one entry per - family under `models/`, so a family scope can keep its own. +- `unittest/_torch/modeling_v2/` — the unit tests: one directory entry + (`--ignore` of `comm/`) that collects the catalog matrix and every + family's tests under `models/`, plus `comm/` on + `l0_gb300_multi_gpus` for the 4-rank collectives. - `test_modeling_v2_` — the accuracy gates, and any future unit file. Both markers are exact by construction rather than by luck: every test diff --git a/jenkins/scripts/cbts/cbts/rules/modeling_v2.py b/jenkins/scripts/cbts/cbts/rules/modeling_v2.py index 3b0871a60f44..152b2179bca6 100644 --- a/jenkins/scripts/cbts/cbts/rules/modeling_v2.py +++ b/jenkins/scripts/cbts/cbts/rules/modeling_v2.py @@ -24,12 +24,13 @@ Scope — per family, else all of modeling_v2: A change confined to one family's directory (`models//`, its native -code included) selects that family's test -entries -- `accuracy/modeling_v2/test_.py` and -`unittest/_torch/modeling_v2/models//` -- plus the modeling_v2 CPU -checks. Any other modeling_v2 change (catalog, resolver, loader, the native -build module, its discovery and common headers) selects every modeling_v2 entry. Several -families in one PR union their entries. +code included) selects the entries that run that family's tests -- +`accuracy/modeling_v2/test_.py`, and any unit entry that collects +`unittest/_torch/modeling_v2/models//` (the modeling_v2 directory entry, +or one under the family's directory) -- plus the modeling_v2 CPU checks. Any +other modeling_v2 change (catalog, resolver, loader, the native build module, its +discovery and common headers) selects every modeling_v2 entry. Several families +in one PR union their entries. Why no outward fallback is needed: nothing beyond the resolver is imported, and no family library is loaded, unless `LLM(modeling_v2=...)` is on -- the model @@ -95,6 +96,8 @@ # every scope, because they are what keeps the narrowing sound. _MV2_CHECK_ENTRY = re.compile(r"^unittest/_torch/modeling_v2/test_modeling_v2_[\w]+\.py$") +_IGNORE = re.compile(r"--ignore=(\S+)") + def _is_mv2_claim(path: str) -> bool: """Decide whether ModelingV2Rule claims `path`. @@ -156,15 +159,23 @@ def family_of(path: str) -> Optional[str]: return None @staticmethod - def _entry_in_family(target: str, family: str) -> bool: + def _entry_runs_family(entry: str, target: str, family: str) -> bool: + """Whether a test-db entry runs tests of `family`. + + Its accuracy gates, or unit tests under its directory, collected by the + entry itself or by a directory entry above it that does not `--ignore` it. + """ + path = target.split()[0] # the target keeps long options such as --ignore accuracy = f"accuracy/modeling_v2/test_{family}.py" + if path == accuracy or path.startswith(accuracy + "::"): + return True unit = f"unittest/_torch/modeling_v2/models/{family}" - return ( - target == accuracy - or target.startswith(accuracy + "::") - or target == unit - or target.startswith(unit + "/") - ) + if path == unit or path.startswith(unit + "/"): + return True + if not unit.startswith(path.rstrip("/") + "/"): + return False + ignored = (path.rstrip("/") for path in _IGNORE.findall(entry)) + return not any(unit == path or unit.startswith(path + "/") for path in ignored) def _select(self, families: Optional[set[str]]): """Select the modeling_v2 entries a change runs. @@ -183,7 +194,7 @@ def _select(self, families: Optional[set[str]]): if not target: continue if families is not None: - owners = {f for f in families if self._entry_in_family(target, f)} + owners = {f for f in families if self._entry_runs_family(entry, target, f)} if not owners and not _MV2_CHECK_ENTRY.match(target): continue matched_families |= owners @@ -203,8 +214,8 @@ def apply(self, pr: PRInputs) -> Optional[RuleResult]: if not block_filters or unmatched: # Do not fabricate stages -- fall back to baseline so the change - # still gets coverage. Reachable if a family registers no test - # entry, or the subtree's entries leave the test-db. + # still gets coverage. Reachable if no entry runs a family's + # tests, or the subtree's entries leave the test-db. missing = f" for {sorted(unmatched)}" if unmatched else "" return RuleResult( handled_files=claimed, diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/README.md b/tensorrt_llm/_torch/_experimental/modeling_v2/README.md index 062c089f3b7a..fe0fa24e2b0b 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/README.md +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/README.md @@ -207,8 +207,9 @@ Adding native code to target `foo_nvfp4__sm_103__tp4` of family `foo`: 4. **Fake every op** in the target's `fake_ops.py`, which the loader imports right after the library opens: `@torch.library.register_fake(f"{namespace_of(__name__)}::my_op")`. -5. **Test it** under `tests/unittest/_torch/modeling_v2/models/foo/`, and list - `unittest/_torch/modeling_v2/models/foo` in the test list of the GPU its SM names. +5. **Test it** under `tests/unittest/_torch/modeling_v2/models/foo/`. The modeling_v2 entry + of `l0_b300.yml` collects it, so a target that runs on GB300 changes no test list; a + test for another GPU, or for several GPUs, is listed in that GPU's test list. Code several targets call goes once in the family's `csrc/`, declared with `add_modeling_v2_shared(foo ARCHS ... SOURCES ...)`; its ops are in `torch.ops.mv2_foo`, diff --git a/tests/integration/test_lists/test-db/l0_b300.yml b/tests/integration/test_lists/test-db/l0_b300.yml index ab454869f8fe..c832c16e75af 100644 --- a/tests/integration/test_lists/test-db/l0_b300.yml +++ b/tests/integration/test_lists/test-db/l0_b300.yml @@ -135,16 +135,9 @@ l0_b300: - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_chunked_prefill[quant_dtype=fp8-kv_cache_reuse=True-fp8kv=True-overlap_scheduler=True] # Cover nvbugs 6084445 - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_flashinfer[enable_chunked_prefill=False] - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_flashinfer[enable_chunked_prefill=True] - # modeling_v2 unit tests: the catalog (shared by every target) as one entry and - # each family as its own, so that CBTS can select one family's tests for a - # change confined to that family (jenkins/scripts/cbts/cbts/rules/modeling_v2.py). - # TIMEOUT measured for the whole directory: 3122 cases in 13m54s on one GB300 (the - # cpu_only files run on the CPU stage instead), so 30 per entry is at least 2.2x headroom. - - unittest/_torch/modeling_v2 --ignore=unittest/_torch/modeling_v2/comm --ignore=unittest/_torch/modeling_v2/models TIMEOUT (30) - - unittest/_torch/modeling_v2/models/deepseek_v3 TIMEOUT (30) - - unittest/_torch/modeling_v2/models/gpt_oss TIMEOUT (30) - - unittest/_torch/modeling_v2/models/glm_moe_dsa TIMEOUT (30) - - unittest/_torch/modeling_v2/models/qwen4_exp TIMEOUT (30) + # TIMEOUT measured: 3122 cases in 13m54s on one GB300 (the cpu_only files run on the CPU + # stage instead), so 30 is 2.2x headroom. + - unittest/_torch/modeling_v2 --ignore=unittest/_torch/modeling_v2/comm TIMEOUT (30) - accuracy/modeling_v2/test_gpt_oss.py::TestModelingV2GptOss120bSm10xDefault::test_gsm8k[tp1] # qwen4_exp target, pre-merge: gsm8k; one MTP engine for gsm8k with the acceptance-length # floor (the only gate that sees a miscomputed draft layer), mmmu (the vision tower) and diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 8b0251c968da..d97277c60fd0 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -174,13 +174,9 @@ unittest/_torch/kernels/parallel/test_fp4_linear.py::test_fp4_gemm_bias_per_back unittest/_torch/kernels/parallel/test_fp4_linear.py::test_fp4_gemm_bias_per_backend[mnk3-cublaslt] SKIP (https://nvbugs/6581067) unittest/_torch/kernels/parallel/test_fp4_linear.py::test_fp4_gemm_bias_per_backend[mnk3-cutlass] SKIP (https://nvbugs/6581067) unittest/_torch/modeling/test_modeling_nemotron_h_multimodal.py::test_nemotron_nano_v2_vl_image_batch_equivalence SKIP (https://nvbugs/6758946) -unittest/_torch/modeling_v2 --ignore=unittest/_torch/modeling_v2/comm --ignore=unittest/_torch/modeling_v2/models SKIP (https://nvbugs/6853741) unittest/_torch/modeling_v2/comm/test_modeling_v2_allgather_op_matrix.py::test_allgather_op_matrix SKIP (https://nvbugs/6843474) unittest/_torch/modeling_v2/comm/test_modeling_v2_reducescatter_op_matrix.py::test_reducescatter_op_matrix SKIP (https://nvbugs/6843474) -unittest/_torch/modeling_v2/models/deepseek_v3 SKIP (https://nvbugs/6853741) -unittest/_torch/modeling_v2/models/glm_moe_dsa SKIP (https://nvbugs/6853741) unittest/_torch/modeling_v2/models/glm_moe_dsa/test_routing.py SKIP (https://nvbugs/6986044) -unittest/_torch/modeling_v2/models/qwen4_exp SKIP (https://nvbugs/6853741) unittest/_torch/modules/mamba/test_flashinfer_gdn_decode.py SKIP (https://nvbugs/6842831) unittest/_torch/modules/tests_lora_modules/test_nemotron_h_lora_sanity.py::TestNemotronHLoRA::test_lora_pp2_sanity SKIP (https://nvbugs/6428124) unittest/_torch/moe/test_moe_backend.py::test_moe_backend[act=Relu2-e60_k4_h2048_i1408-seq=8-dtype=torch.bfloat16-backend=TRTLLM-quant=NVFP4-routing=Renormalize] SKIP (https://nvbugs/5989912) diff --git a/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_claims.py b/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_claims.py index 208fc08c05a3..a6fa035b23f6 100644 --- a/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_claims.py +++ b/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_claims.py @@ -424,18 +424,6 @@ def _test_db_entries() -> list[tuple[str, str, str]]: return entries -def test_test_lists_keep_family_tests_in_their_own_entries(): - """CBTS selects one family's tests by entry; an entry covering models/ from - above would hide a family's unit tests inside the catalog's.""" - models = "unittest/_torch/modeling_v2/models" - for yml, target, options in _test_db_entries(): - if target in ("unittest/_torch/modeling_v2", "unittest/_torch/modeling_v2/"): - assert f"--ignore={models}" in options, ( - f"{yml}: {target} must carry --ignore={models}; list each family on its own" - ) - assert target != models, f"{yml}: list each family under {models}/ on its own" - - def test_family_unit_entries_point_at_tests(): """A family entry naming a directory with no tests is a collection error in CI.""" for yml, target, _ in _test_db_entries(): @@ -446,14 +434,28 @@ def test_family_unit_entries_point_at_tests(): ) -def test_families_with_native_code_have_test_entries(): - targets = {target for _, target, _ in _test_db_entries()} +def _runs(target: str, options: str, unit: str) -> bool: + """Whether a test-db entry runs tests under the directory ``unit``: it names + the directory or something in it, or a directory above it that it does not + ``--ignore``.""" + if target == unit or target.startswith(unit + "/"): + return True + if not unit.startswith(target.rstrip("/") + "/"): + return False + ignored = (path.rstrip("/") for path in re.findall(r"--ignore=(\S+)", options)) + return not any(unit == path or unit.startswith(path + "/") for path in ignored) + + +def test_families_with_native_code_have_tests_a_test_list_runs(): + entries = _test_db_entries() for family in _families(): unit = f"unittest/_torch/modeling_v2/models/{family}" accuracy = f"accuracy/modeling_v2/test_{family}.py" - assert any(t == unit or t.startswith((unit + "/", accuracy)) for t in targets), ( - f"family {family!r} has native code but no test-db entry under {unit} or {accuracy}" - ) + has_unit_tests = any((_REPO / "tests" / unit).rglob("test_*.py")) + assert any( + target.startswith(accuracy) or (has_unit_tests and _runs(target, options, unit)) + for _, target, options in entries + ), f"family {family!r} has native code but no test-db entry runs {unit} or {accuracy}" def test_accuracy_entries_name_a_family(): diff --git a/tests/unittest/scripts/cbts/test_modeling_v2_rule.py b/tests/unittest/scripts/cbts/test_modeling_v2_rule.py index 35432eff21e1..5348193d5cb8 100644 --- a/tests/unittest/scripts/cbts/test_modeling_v2_rule.py +++ b/tests/unittest/scripts/cbts/test_modeling_v2_rule.py @@ -20,7 +20,7 @@ import pytest __extra_import_path__ = ["~/jenkins/scripts/cbts"] -from cbts.blocks import Block, Stage, YAMLIndex +from cbts.blocks import Block, Stage, YAMLIndex, _entry_target from cbts.rules.base import PRInputs from cbts.rules.modeling_v2 import ModelingV2Rule, _is_mv2_claim @@ -28,12 +28,10 @@ _PY = "tensorrt_llm/_torch/_experimental/modeling_v2" _CPP = "cpp/tensorrt_llm/modeling_v2" -_CATALOG_UNIT = ( - "unittest/_torch/modeling_v2 --ignore=unittest/_torch/modeling_v2/comm " - "--ignore=unittest/_torch/modeling_v2/models TIMEOUT (30)" -) -_DEEPSEEK_UNIT = "unittest/_torch/modeling_v2/models/deepseek_v3" -_GPT_OSS_UNIT = "unittest/_torch/modeling_v2/models/gpt_oss" +_UNIT = "unittest/_torch/modeling_v2 --ignore=unittest/_torch/modeling_v2/comm TIMEOUT (30)" +_COMM = "unittest/_torch/modeling_v2/comm" +_KIMI_UNIT = "unittest/_torch/modeling_v2/models/kimi_k3_vl/test_modeling_v2_kimi_k3_native.py" +_DEEPSEEK_ACCURACY = "accuracy/modeling_v2/test_deepseek_v3.py::TestDeepseek::test_gsm8k[tp4]" _GPT_OSS_ACCURACY = "accuracy/modeling_v2/test_gpt_oss.py::TestGptOss::test_gsm8k[tp1]" _CLAIMS = "unittest/_torch/modeling_v2/test_modeling_v2_claims.py" @@ -48,16 +46,25 @@ def rule() -> ModelingV2Rule: condition={}, tests=[ "accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4", - _CATALOG_UNIT, - _DEEPSEEK_UNIT, - _GPT_OSS_UNIT, + _UNIT, _GPT_OSS_ACCURACY, ], ), + Block( + yaml_stem="l0_gb300_multi_gpus", + block_index=0, + condition={}, + tests=[_COMM, _DEEPSEEK_ACCURACY], + ), + Block(yaml_stem="l0_b200", block_index=0, condition={}, tests=[_KIMI_UNIT]), Block(yaml_stem="l0_cpu", block_index=0, condition={}, tests=["unittest/tools", _CLAIMS]), ] stages = { "B300-PyTorch-1": Stage("B300-PyTorch-1", "l0_b300", "aarch64", 1, 2), + "GB300-4_GPUs-PyTorch-1": Stage( + "GB300-4_GPUs-PyTorch-1", "l0_gb300_multi_gpus", "aarch64", 1, 1 + ), + "B200-PyTorch-1": Stage("B200-PyTorch-1", "l0_b200", "x86_64", 1, 1), "CPU-Generic-x86-1": Stage("CPU-Generic-x86-1", "l0_cpu", "x86_64", 1, 1), } return ModelingV2Rule(yaml_index, stages) @@ -109,20 +116,45 @@ def test_family_of_reads_the_family_directory_native_code_included(rule): f"{_PY}/models/gpt_oss/csrc/CMakeLists.txt", ], ) -def test_a_family_change_selects_that_family_and_the_checks(rule, changed): +def test_a_family_change_selects_what_runs_that_family_and_the_checks(rule, changed): + """The directory entry collects every family's unit tests; comm/ is --ignored.""" result = _apply(rule, changed) assert result.scope == "modelingv2only" - assert _selected(result) == {_GPT_OSS_UNIT, _GPT_OSS_ACCURACY, _CLAIMS} + assert _selected(result) == {_UNIT, _GPT_OSS_ACCURACY, _CLAIMS} assert result.sanity_relevant is False +def test_a_family_keeps_its_own_entries_on_other_gpus(rule): + result = _apply(rule, f"{_PY}/models/kimi_k3_vl/csrc/k3AttnResDecode.cu") + assert _selected(result) == {_UNIT, _KIMI_UNIT, _CLAIMS} + + def test_two_families_union(rule): result = _apply( rule, f"{_PY}/models/deepseek_v3/routing.py", f"{_PY}/models/gpt_oss/csrc/CMakeLists.txt", ) - assert _selected(result) == {_DEEPSEEK_UNIT, _GPT_OSS_UNIT, _GPT_OSS_ACCURACY, _CLAIMS} + assert _selected(result) == {_UNIT, _DEEPSEEK_ACCURACY, _GPT_OSS_ACCURACY, _CLAIMS} + + +@pytest.mark.parametrize( + ("entry", "runs"), + [ + ("unittest/_torch/modeling_v2", True), + ("unittest/_torch/modeling_v2/ --ignore=unittest/_torch/modeling_v2/comm", True), + ("unittest/_torch/modeling_v2/models", True), + ("unittest/_torch/modeling_v2/models/gpt_oss/test_x.py::test_y", True), + ("unittest/_torch/modeling_v2 --ignore=unittest/_torch/modeling_v2/models", False), + ("unittest/_torch/modeling_v2 --ignore=unittest/_torch/modeling_v2/models/gpt_oss/", False), + ("unittest/_torch/modeling_v2/models/gpt_oss_mini", False), + ("unittest/_torch/modeling_v2/comm", False), + ("accuracy/modeling_v2/test_gpt_oss.py::TestGptOss", True), + ("accuracy/modeling_v2/test_deepseek_v3.py", False), + ], +) +def test_entry_runs_family(entry, runs): + assert ModelingV2Rule._entry_runs_family(entry, _entry_target(entry), "gpt_oss") is runs @pytest.mark.parametrize( @@ -139,15 +171,17 @@ def test_shared_modeling_v2_code_selects_everything_of_modeling_v2(rule, changed result = _apply(rule, changed) assert result.scope == "modelingv2only" assert _selected(result) == { - _CATALOG_UNIT, - _DEEPSEEK_UNIT, - _GPT_OSS_UNIT, + _UNIT, + _COMM, + _KIMI_UNIT, + _DEEPSEEK_ACCURACY, _GPT_OSS_ACCURACY, _CLAIMS, } -def test_a_family_with_no_test_entry_falls_back(rule): +def test_a_family_no_entry_runs_falls_back(rule): + rule.yaml_index.blocks[0].tests.remove(_UNIT) result = _apply(rule, f"{_PY}/models/minimax_m3/csrc/attnMega.cu") assert result.scope is None assert "minimax_m3" in result.reason From c66125ae581aa480be7f6fa202d6393ac72126de Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Mon, 26 Oct 2026 20:04:13 -0700 Subject: [PATCH 16/20] [None][test] modeling_v2: fold the native-code tests into fewer cases The native-code tests check the same things in fewer test cases: related checks share a test, and each assertion keeps its own message. - boundary checks: 19 tests to 6. The test-list entry existence check is left to the test-list validation pre-commit runs, and the default-off check to test_modeling_v2_routing.py, which already covers it. - loader: 19 cases to 4; rank agreement: 7 to 2; CBTS rule: 24 cases to 3; build_wheel install: 3 to 1. - gpt_oss swizzle kernel: 11 cases to 1 (the FC1 and FC2 shapes and the empty case, one rejection, the fake). Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- ...odeling_v2_gpt_oss_swizzle_block_scales.py | 54 +-- .../test_modeling_v2_native_claims.py | 345 ++++++------------ .../test_modeling_v2_native_loader.py | 137 ++----- .../test_modeling_v2_rank_agreement.py | 123 +++---- .../scripts/cbts/test_modeling_v2_rule.py | 133 +++---- .../scripts/test_build_wheel_modeling_v2.py | 65 ++-- 6 files changed, 279 insertions(+), 578 deletions(-) diff --git a/tests/unittest/_torch/modeling_v2/models/gpt_oss/gpt_oss_120b__sm_10x__default/test_modeling_v2_gpt_oss_swizzle_block_scales.py b/tests/unittest/_torch/modeling_v2/models/gpt_oss/gpt_oss_120b__sm_10x__default/test_modeling_v2_gpt_oss_swizzle_block_scales.py index 4e6ff9ab63ea..53ba2a743735 100644 --- a/tests/unittest/_torch/modeling_v2/models/gpt_oss/gpt_oss_120b__sm_10x__default/test_modeling_v2_gpt_oss_swizzle_block_scales.py +++ b/tests/unittest/_torch/modeling_v2/models/gpt_oss/gpt_oss_120b__sm_10x__default/test_modeling_v2_gpt_oss_swizzle_block_scales.py @@ -31,49 +31,19 @@ def _reference(s: torch.Tensor) -> torch.Tensor: return v.permute(0, 1, 4, 3, 2, 5).reshape(e, m, c).contiguous() -def _scales(experts: int, rows: int, cols: int, seed: int) -> torch.Tensor: - g = torch.Generator(device="cuda").manual_seed(seed) - return torch.randint( - 0, 256, (experts, rows, cols), generator=g, device="cuda", dtype=torch.uint8 - ) +def test_swizzle_block_scales(): + g = torch.Generator(device="cuda").manual_seed(0) + # gpt-oss-120b's FC1 and FC2 scales (FC2's 90 scales per row padded to 96), and no experts. + for shape in ((4, 5888, 92), (3, 384, 96), (0, 128, 4)): + s = torch.randint(0, 256, shape, generator=g, device="cuda", dtype=torch.uint8) + assert torch.equal(weights._swizzle_scales(s), _reference(s)), shape + strided = torch.zeros(2, 128, 8, device="cuda", dtype=torch.uint8)[:, :, :4] + with pytest.raises(RuntimeError, match="contiguous"): + native_ops.swizzle_block_scales(strided) -@pytest.mark.parametrize( - "experts, rows, cols", - [ - (1, 128, 4), # one 128x4 tile - (2, 256, 8), # tiles in both directions - (3, 384, 96), # gpt-oss-120b FC2: H / 32 = 90 scales, padded to 96 - (4, 5888, 92), # FC1 rows, many tiles - (0, 128, 4), # no experts - ], -) -def test_the_kernel_matches_the_torch_swizzle(experts, rows, cols): - s = _scales(experts, rows, cols, seed=rows * 131 + cols) - expected = _reference(s) - assert torch.equal(native_ops.swizzle_block_scales(s), expected) - # The loader's call site, which the FC1 and FC2 scale preparation both use. - assert torch.equal(weights._swizzle_scales(s), expected) - - -@pytest.mark.parametrize( - "scales, match", - [ - (lambda: _scales(1, 100, 4, 0), "rows % 128 == 0"), - (lambda: _scales(1, 128, 6, 0), "cols % 4 == 0"), - (lambda: _scales(1, 128, 4, 0).float(), "uint8"), - (lambda: _scales(2, 128, 8, 0)[:, :, :4], "contiguous"), - (lambda: _scales(1, 128, 4, 0).cpu(), "CUDA"), - ], - ids=["rows", "cols", "dtype", "strided", "host"], -) -def test_the_op_rejects_what_the_kernel_cannot_swizzle(scales, match): - with pytest.raises(RuntimeError, match=match): - native_ops.swizzle_block_scales(scales()) - - -def test_the_fake_op_describes_the_output(): with FakeTensorMode(): - s = torch.empty(2, 256, 8, device="cuda", dtype=torch.uint8) - out = native_ops.swizzle_block_scales(s) + out = native_ops.swizzle_block_scales( + torch.empty(2, 256, 8, device="cuda", dtype=torch.uint8) + ) assert out.shape == (2, 256, 8) and out.dtype == torch.uint8 diff --git a/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_claims.py b/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_claims.py index a6fa035b23f6..86c8f5ee16c2 100644 --- a/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_claims.py +++ b/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_claims.py @@ -8,8 +8,8 @@ targets (``_native``). Three things rest on that staying true: ``off`` loading nothing, a target never depending on a sibling, and CBTS testing a family-only change with that family's tests alone -(``jenkins/scripts/cbts/cbts/rules/modeling_v2.py``). Each test here guards one -of the rules that keeps it true; none imports a target or needs a GPU. +(``jenkins/scripts/cbts/cbts/rules/modeling_v2.py``). The tests here guard the +rules that keep it true; none imports a target or needs a GPU. The paths are the source checkout's, found from this file: these are claims about the tree a PR changes, C++ included, not about an installed wheel. @@ -116,250 +116,160 @@ def _native_files(root: Path) -> list[Path]: # --- Layout ------------------------------------------------------------------ -def test_every_native_family_is_a_python_family(): +def test_native_code_is_where_the_build_and_the_loader_find_it(): + """Native code and CMake live only in a family's or target's csrc/, every + native family and target is a Python one, a target with native code loads it, + and its ARCHS cover the architecture its name promises.""" for family, csrc in _families().items(): - assert csrc.parent.name == family, ( + assert csrc.parent.name == family and (csrc.parent / "__init__.py").is_file(), ( f"{csrc.relative_to(_REPO)}/CMakeLists.txt declares family {family!r}; " - f"it belongs in models/{family}/csrc/" + f"it belongs in models/{family}/csrc/ of the Python family {family}" ) - assert (csrc.parent / "__init__.py").is_file(), ( - f"native family {family!r} has no Python family package models/{family}/" - ) - -def test_native_code_lives_only_in_csrc(): - """In the Python tree, C++/CUDA and CMake live only in a family's or a - target's csrc/, where the build finds them and these checks read them.""" - offenders = [] + misplaced = [] for path in sorted(_MODELS.rglob("*")): - if not path.is_file() or "__pycache__" in path.parts: - continue - native = path.suffix in _NATIVE_SUFFIXES or path.name == "CMakeLists.txt" - if not native: + if not path.is_file() or not ( + path.suffix in _NATIVE_SUFFIXES or path.name == "CMakeLists.txt" + ): continue parts = path.relative_to(_MODELS).parts - in_family_csrc = len(parts) >= 3 and parts[1] == "csrc" - in_target_csrc = len(parts) >= 4 and parts[2] == "csrc" if path.name == "CMakeLists.txt": ok = len(parts) == 3 and parts[1] == "csrc" else: - ok = in_family_csrc or in_target_csrc + ok = (len(parts) >= 3 and parts[1] == "csrc") or ( + len(parts) >= 4 and parts[2] == "csrc" + ) if not ok: - offenders.append(str(path.relative_to(_REPO))) - assert not offenders, ( + misplaced.append(str(path.relative_to(_REPO))) + assert not misplaced, ( "native code outside models//csrc/ or models///csrc/ " - f"(CMakeLists.txt only in models//csrc/): {offenders}" + f"(CMakeLists.txt only in models//csrc/): {misplaced}" ) - -def test_every_native_target_is_a_python_target_that_loads_it(): - with_code = set() + declared, with_code = set(), set() for family, target, tokens in _targets(): + declared.add((family, target)) package = _MODELS / family / target / "__init__.py" - assert package.is_file(), f"native target {family}/{target} has no Python target package" - assert "load_target_ops(__name__)" in package.read_text(), ( - f"{package.relative_to(_REPO)} must start by loading its native code: " + assert package.is_file() and "load_target_ops(__name__)" in package.read_text(), ( + f"native target {family}/{target}: its Python package must start with " "native_ops = load_target_ops(__name__)" ) if _keyword_values(tokens, "SOURCES"): with_code.add((family, target)) - assert (_MODELS / family / target / "csrc").is_dir(), ( - f"{family}/{target} declares SOURCES but has no {target}/csrc/" - ) - for csrc in sorted(_MODELS.glob("*/*/csrc")): - family, target = csrc.parent.parent.name, csrc.parent.name - assert (family, target) in with_code, ( - f"{csrc.relative_to(_REPO)} holds code no add_modeling_v2_target({family} {target} " - "... SOURCES ...) builds" - ) - - -def test_python_targets_that_load_native_code_have_it(): - declared = {(family, target) for family, target, _ in _targets()} - for package in sorted(_MODELS.glob("*/*/__init__.py")): - if "load_target_ops(" not in package.read_text(): - continue - family, target = package.parent.parent.name, package.parent.name - assert (family, target) in declared, ( - f"{package.relative_to(_REPO)} loads native code, but no " - f"add_modeling_v2_target({family} {target} ...) declares it" - ) - - -def test_declared_archs_cover_the_sm_in_the_target_name(): - for family, target, tokens in _targets(): archs = _keyword_values(tokens, "ARCHS") sm = target.split("__")[1][len("sm_") :] - if sm in _SM_GROUPS: - needed = _SM_GROUPS[sm] - else: - assert sm.isdigit(), ( - f"{family}/{target}: sm segment 'sm_{sm}' is neither a number nor in _SM_GROUPS" - ) - needed = (divmod(int(sm), 10),) - for capability in needed: + assert sm in _SM_GROUPS or sm.isdigit(), f"{family}/{target}: unknown sm segment sm_{sm}" + for capability in _SM_GROUPS[sm] if sm in _SM_GROUPS else (divmod(int(sm), 10),): assert any(arch_covers(a, capability) for a in archs), ( f"{family}/{target} declares ARCHS {archs}, none of which runs on " f"sm_{capability[0]}{capability[1]}, the architecture in its name" ) + for csrc in sorted(_MODELS.glob("*/*/csrc")): + unit = (csrc.parent.parent.name, csrc.parent.name) + assert unit in with_code, f"{csrc.relative_to(_REPO)} holds code no target's SOURCES builds" + for package in sorted(_MODELS.glob("*/*/__init__.py")): + unit = (package.parent.parent.name, package.parent.name) + assert "load_target_ops(" not in package.read_text() or unit in declared, ( + f"{package.relative_to(_REPO)} loads native code no add_modeling_v2_target declares" + ) # --- What native code may reach ----------------------------------------------- _INCLUDE = re.compile(r'^\s*#\s*include\s*"([^"]+)"', re.MULTILINE) +_RAW_REGISTRATION = re.compile(r"\bTORCH_LIBRARY(?:_FRAGMENT|_IMPL)?\s*\(") +_OP_DEF = re.compile(r'\bm\.def\(\s*"(\w+)\(') -def test_native_code_includes_no_other_target_or_family(): +def test_native_code_stays_in_its_own_unit(): """Native code may include stock headers, but of modeling_v2 only its own - target's csrc/, its family's csrc/ and common/: a target never reaches a - sibling or another family.""" + target's csrc/, its family's csrc/ and common/; it registers through the + TRTLLM_MV2_ macros, into its own namespace only; every op it defines has a + fake in the unit's fake_ops.py (which the loader imports); and every file + carries NVIDIA's Apache-2.0 header, which the C++ license checker only checks + under cpp/.""" python_families = {p.name for p in _MODELS.iterdir() if (p / "__init__.py").is_file()} problems = [] for family, target, csrc in _units(): allowed = [f"{family}/csrc/", "tensorrt_llm/modeling_v2/common/"] if target is not None: allowed.append(f"{family}/{target}/csrc/") - for path in _native_files(csrc): - for include in _INCLUDE.findall(path.read_text()): - where = f'{path.relative_to(_REPO)}: "{include}"' + ops = set() + for path in [*_native_files(csrc), *csrc.glob("CMakeLists.txt")]: + text = path.read_text(errors="ignore") + where = str(path.relative_to(_REPO)) + head = text[:1200] + apache = ( + "SPDX-License-Identifier: Apache-2.0" in head + or "Apache License, Version 2.0" in head + ) + if "NVIDIA CORPORATION" not in head or not apache: + problems.append(f"{where}: no NVIDIA Apache-2.0 license header") + if path.name == "CMakeLists.txt": + continue + for include in _INCLUDE.findall(text): head = include.split("/", 1)[0] if head in python_families or include.startswith("tensorrt_llm/modeling_v2/"): if not include.startswith(tuple(allowed)): - problems.append(f"{where} reaches another target or family") + problems.append(f'{where}: "{include}" reaches another target or family') elif ".." in include or "/" not in include and (csrc / include).exists(): problems.append( - f'{where}: include the family\'s code from models/, "{family}/..."' + f'{where}: "{include}": include it from models/, "{family}/..."' ) + if _RAW_REGISTRATION.search(text): + problems.append(f"{where}: use the TRTLLM_MV2_LIBRARY_* macros, not TORCH_LIBRARY*") + ops.update(_OP_DEF.findall(text)) + if ops: + python_dir = _MODELS / family if target is None else _MODELS / family / target + fake_ops = python_dir / "fake_ops.py" + text = fake_ops.read_text() if fake_ops.is_file() else "" + missing = sorted(op for op in ops if f'::{op}"' not in text) + if missing: + problems.append(f"{fake_ops.relative_to(_REPO)}: no fake for {missing}") assert not problems, "\n".join(problems) -_RAW_REGISTRATION = re.compile(r"\bTORCH_LIBRARY(?:_FRAGMENT|_IMPL)?\s*\(") - - -def test_native_code_registers_only_into_its_own_namespace(): - """Raw TORCH_LIBRARY* would let a file register into trtllm:: or a sibling's - namespace; the TRTLLM_MV2_ macros register into the unit's own only.""" - offenders = [ - str(path.relative_to(_REPO)) - for _, _, csrc in _units() - for path in _native_files(csrc) - if _RAW_REGISTRATION.search(path.read_text()) - ] - assert not offenders, ( - f"use TRTLLM_MV2_LIBRARY_FRAGMENT / TRTLLM_MV2_LIBRARY_IMPL, not TORCH_LIBRARY*: {offenders}" - ) - - -_OP_DEF = re.compile(r'\bm\.def\(\s*"(\w+)\(') - - -def test_every_native_op_has_a_fake(): - """fake_ops.py doubles as the Python side's list of a unit's ops: the loader - imports it, so an op without a fake is one torch.compile and fake tracing - cannot see, and a fake without an op fails the load. A target's ops have - their fakes in the target's fake_ops.py, the family's shared ops in the - family's.""" - for family, target, native_dir in _units(): - python_dir = _MODELS / family if target is None else _MODELS / family / target - ops = sorted( - {op for p in _native_files(native_dir) for op in _OP_DEF.findall(p.read_text())} - ) - if not ops: - continue - fake_ops = python_dir / "fake_ops.py" - assert fake_ops.is_file(), ( - f"{native_dir.relative_to(_REPO)} defines {ops}, " - f"but {fake_ops.relative_to(_REPO)} does not exist" - ) - text = fake_ops.read_text() - missing = [op for op in ops if f'::{op}"' not in text] - assert not missing, f"{fake_ops.relative_to(_REPO)} has no fake for {missing}" - - # --- What stock code may reach ------------------------------------------------ -_STOCK_PY_REFERENCE = re.compile(r"modeling_v2[./]models\b|torch\.ops\.mv2_") +_STOCK_PY_REFERENCE = re.compile( + r"modeling_v2[./]models\b|torch\.ops\.mv2_|register_fake\(\s*f?[\"']mv2_" +) _STOCK_CPP_REFERENCE = re.compile(r'#\s*include\s*"tensorrt_llm/modeling_v2/') # A path into one family's directory; the discovery's models/*/csrc glob names none. _STOCK_FAMILY_PATH = re.compile(r"_experimental/modeling_v2/models/[a-z]") -def test_stock_python_does_not_reference_targets(): - offenders = [] - for path in sorted((_REPO / "tensorrt_llm").rglob("*.py")): - if path.is_relative_to(_PY): - continue - if _STOCK_PY_REFERENCE.search(path.read_text(errors="ignore")): - offenders.append(str(path.relative_to(_REPO))) - assert not offenders, ( - f"stock code imports or calls modeling_v2 targets: {offenders}. Stock reaches " - "modeling_v2 only through the resolver; a target hands stock a callable instead." - ) - - -def test_stock_build_does_not_reference_a_family(): - """A family's native code reaches the build only through - cpp/tensorrt_llm/modeling_v2/'s discovery: no stock CMake, C++ or packaging - file names a family, so adding one edits none of them.""" - offenders = [] +def test_stock_does_not_reach_modeling_v2(): + """Stock Python reaches modeling_v2 only through the resolver (no target + import, op call or fake), and no stock CMake, C++ or packaging file names a + family: a family reaches the build only through + cpp/tensorrt_llm/modeling_v2/'s discovery, so adding one edits none of them.""" + offenders = [ + str(path.relative_to(_REPO)) + for path in sorted((_REPO / "tensorrt_llm").rglob("*.py")) + if not path.is_relative_to(_PY) + and _STOCK_PY_REFERENCE.search(path.read_text(errors="ignore")) + ] for root in ("cpp/tensorrt_llm", "cpp/include", "cpp/tests", "cpp/cmake"): for path in sorted((_REPO / root).rglob("*")): if not path.is_file() or path.is_relative_to(_INFRA): continue # Skip generated cubin sources; they include nothing of interest. if path.suffix in _NATIVE_SUFFIXES and path.stat().st_size < (1 << 20): - if _STOCK_CPP_REFERENCE.search(path.read_text(errors="ignore")): - offenders.append(str(path.relative_to(_REPO))) + pattern = _STOCK_CPP_REFERENCE elif path.name == "CMakeLists.txt" or path.suffix == ".cmake": - if _STOCK_FAMILY_PATH.search(path.read_text()): - offenders.append(str(path.relative_to(_REPO))) + pattern = _STOCK_FAMILY_PATH + else: + continue + if pattern.search(path.read_text(errors="ignore")): + offenders.append(str(path.relative_to(_REPO))) for name in ("cpp/CMakeLists.txt", "setup.py", "scripts/build_wheel.py"): if _STOCK_FAMILY_PATH.search((_REPO / name).read_text()): offenders.append(name) - assert not offenders, f"stock build files name a modeling_v2 family: {offenders}" + assert not offenders, f"stock code reaches modeling_v2 targets or families: {offenders}" -def test_native_files_carry_the_license_header(): - """The C++ license checker (jenkins/license_cpp.json, Apache) scans cpp/ only; - this holds the families' csrc/ to the same header: NVIDIA's copyright and the - Apache-2.0 license, as SPDX lines or the license text.""" - missing = [] - for _, _, csrc in _units(): - for path in [*_native_files(csrc), *csrc.glob("CMakeLists.txt")]: - head = path.read_text(errors="ignore")[:1200] - apache = ( - "SPDX-License-Identifier: Apache-2.0" in head - or "Apache License, Version 2.0" in head - ) - if "NVIDIA CORPORATION" not in head or not apache: - missing.append(str(path.relative_to(_REPO))) - assert not missing, f"no NVIDIA Apache-2.0 license header: {missing}" - - -def test_stock_custom_ops_register_no_fakes_for_targets(): - offenders = [ - str(path.relative_to(_REPO)) - for path in sorted((_REPO / "tensorrt_llm/_torch/custom_ops").rglob("*.py")) - if re.search(r"register_fake\(\s*f?[\"']mv2_", path.read_text()) - ] - assert not offenders, f"target fakes belong in the target's fake_ops.py: {offenders}" - - -# --- Targets ------------------------------------------------------------------ - - -def test_no_op_probing(): - """A target calls exactly the ops it declares. Choosing a path by whether an - op exists means `require` no longer promises the gated code ran.""" - # hasattr, or getattr with a default: an absent op becomes a path choice - # instead of an error. The loader's two-argument getattr raises instead. - probe = re.compile(r"\bhasattr\(\s*torch\.ops\b|\bgetattr\(\s*torch\.ops[\w.]*\s*,[^,()]+,") - offenders = [ - str(path.relative_to(_REPO)) - for path in sorted(_PY.rglob("*.py")) - if probe.search(path.read_text()) - ] - assert not offenders, f"op probing in {offenders}" +# --- Targets load only when routed ---------------------------------------------- def _is_docstring_or_future(node: ast.stmt) -> bool: @@ -368,15 +278,15 @@ def _is_docstring_or_future(node: ast.stmt) -> bool: return docstring or future -def test_family_packages_have_no_side_effects(): - """Native code loads per target, when ``route`` imports the target. - - ``models/__init__.py`` imports only the families' routing modules, a family - package holds a docstring only, and a routing module imports a target only - inside ``route``. Anything else would load a target's native code on every - resolve, or for every target of the family. - """ - for node in ast.parse((_MODELS / "__init__.py").read_text()).body: +def test_native_code_loads_only_when_routing_picks_its_target(): + """``models/__init__.py`` imports only the families' routing modules, a + family package holds a docstring only, and a routing module imports a target + only inside ``route``; anything else would load native code on every resolve. + A target calls exactly the ops it declares: choosing a path by whether an op + exists (hasattr, or getattr with a default) would mean ``require`` no longer + promises the gated code ran.""" + models_init = _MODELS / "__init__.py" + for node in ast.parse(models_init.read_text()).body: routing_import = ( isinstance(node, ast.ImportFrom) and node.level == 1 @@ -385,8 +295,8 @@ def test_family_packages_have_no_side_effects(): and [alias.name for alias in node.names] == ["routing"] ) assert _is_docstring_or_future(node) or routing_import, ( - f"{(_MODELS / '__init__.py').relative_to(_REPO)}:{node.lineno} -- " - "models/__init__.py imports the families' routing modules and nothing else" + f"{models_init.relative_to(_REPO)}:{node.lineno} -- imports something besides " + "the families' routing modules" ) for init in sorted(_MODELS.glob("*/__init__.py")): for node in ast.parse(init.read_text()).body: @@ -399,14 +309,11 @@ def test_family_packages_have_no_side_effects(): f"{routing.relative_to(_REPO)}:{node.lineno} -- imports .{node.module} at " "module level; a routing module imports a target only inside route" ) - - -def test_modeling_v2_defaults_to_off(): - """`off` loads nothing, which is what lets CBTS run only modeling_v2 tests - for a modeling_v2 change. Changing the default changes that rule too.""" - from tensorrt_llm.llmapi.llm_args import TorchLlmArgs - - assert TorchLlmArgs.model_fields["modeling_v2"].default == "off" + probe = re.compile(r"\bhasattr\(\s*torch\.ops\b|\bgetattr\(\s*torch\.ops[\w.]*\s*,[^,()]+,") + probing = [ + str(p.relative_to(_REPO)) for p in sorted(_PY.rglob("*.py")) if probe.search(p.read_text()) + ] + assert not probing, f"op probing in {probing}" # --- Test lists ---------------------------------------------------------------- @@ -414,26 +321,6 @@ def test_modeling_v2_defaults_to_off(): _MV2_ENTRY = re.compile(r"^\s*-\s+((?:unittest/_torch/modeling_v2|accuracy/modeling_v2)\S*)(.*)$") -def _test_db_entries() -> list[tuple[str, str, str]]: - entries = [] - for yml in sorted(_TEST_DB.glob("*.yml")): - for line in yml.read_text().splitlines(): - match = _MV2_ENTRY.match(line) - if match: - entries.append((yml.name, match.group(1), match.group(2))) - return entries - - -def test_family_unit_entries_point_at_tests(): - """A family entry naming a directory with no tests is a collection error in CI.""" - for yml, target, _ in _test_db_entries(): - if target.startswith("unittest/_torch/modeling_v2/models/"): - path = _REPO / "tests" / target - assert path.exists() and (path.is_file() or any(path.rglob("test_*.py"))), ( - f"{yml}: {target} names no test file or directory with tests" - ) - - def _runs(target: str, options: str, unit: str) -> bool: """Whether a test-db entry runs tests under the directory ``unit``: it names the directory or something in it, or a directory above it that it does not @@ -446,8 +333,15 @@ def _runs(target: str, options: str, unit: str) -> bool: return not any(unit == path or unit.startswith(path + "/") for path in ignored) -def test_families_with_native_code_have_tests_a_test_list_runs(): - entries = _test_db_entries() +def test_test_lists_run_every_native_family(): + """Some test-db entry runs each native family's tests, and accuracy files are + named after a family, since CBTS selects them by family name.""" + entries = [ + (yml.name, *match.groups()) + for yml in sorted(_TEST_DB.glob("*.yml")) + for match in map(_MV2_ENTRY.match, yml.read_text().splitlines()) + if match + ] for family in _families(): unit = f"unittest/_torch/modeling_v2/models/{family}" accuracy = f"accuracy/modeling_v2/test_{family}.py" @@ -456,17 +350,12 @@ def test_families_with_native_code_have_tests_a_test_list_runs(): target.startswith(accuracy) or (has_unit_tests and _runs(target, options, unit)) for _, target, options in entries ), f"family {family!r} has native code but no test-db entry runs {unit} or {accuracy}" - - -def test_accuracy_entries_name_a_family(): families = {p.name for p in _MODELS.iterdir() if (p / "__init__.py").is_file()} - for yml, target, _ in _test_db_entries(): + for yml, target, _ in entries: match = re.match(r"accuracy/modeling_v2/test_(\w+)\.py", target) - if match: - assert match.group(1) in families, ( - f"{yml}: {target} is named after no modeling_v2 family; CBTS selects " - "accuracy gates by family name" - ) + assert not match or match.group(1) in families, ( + f"{yml}: {target} is named after no modeling_v2 family" + ) # --- CMake and Python agree ------------------------------------------------------ diff --git a/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_loader.py b/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_loader.py index d35c77d78f17..3934771eeadb 100644 --- a/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_loader.py +++ b/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_loader.py @@ -2,16 +2,16 @@ # SPDX-License-Identifier: Apache-2.0 """What ``_native`` decides, and that the installed family libraries load. -The first half drives the loader's decisions without opening anything: which +The first tests drive the loader's decisions without opening anything: which outcomes are ``NoTarget`` (this installation does not carry the target, so ``auto`` keeps the built-in model) and which are ``NativeLoadError`` (an installed library is broken, which is never a fallback). -The second half is the load smoke test of the installed wheel: every family -library in ``tensorrt_llm/libs`` opens in a fresh process, names the targets it -carries, and defines no op the target's fake_ops.py does not fake. A library -aborts its process from a static initializer if two libraries claim one torch -namespace, so each one is opened in a child process. +The last is the load smoke test of the installed wheel: every family library in +``tensorrt_llm/libs`` opens in a fresh process, names the targets it carries, and +defines no op the target's fake_ops.py does not fake. A library aborts its +process from a static initializer if two libraries claim one torch namespace, so +each one is opened in a child process. """ from __future__ import annotations @@ -38,138 +38,75 @@ _FAMILY = "tensorrt_llm._torch._experimental.modeling_v2.models.gpt_oss" _TARGET = f"{_FAMILY}.gpt_oss_120b__sm_10x__default" +_TARGET_NS = "mv2_gpt_oss_gpt_oss_120b_sm_10x_default" @pytest.fixture def libs(monkeypatch, tmp_path): - """An empty libs directory and a cold library cache.""" + """An empty libs directory, a cold library cache, and ``libs.install(opens)``, + which puts a gpt_oss library there; ``opens`` stands in for dlopen, and no + fake_ops module is imported, since the stand-in defines no op.""" monkeypatch.setattr(_native, "_LIBS_DIR", tmp_path) monkeypatch.setattr(_native, "_built_targets", {}) monkeypatch.setattr(_native, "_family_ops", {}) - return tmp_path - - -def _install(libs: Path, monkeypatch, opens) -> None: - """Put a gpt_oss library in `libs`; `opens` stands in for dlopen, and no - fake_ops module is imported, since the stand-in defines no op.""" - (libs / "libth_modeling_v2_gpt_oss.so").write_bytes(b"") - monkeypatch.setattr(_native, "_open_library", lambda path, family: opens()) - monkeypatch.setattr(_native, "_import_fakes", lambda module, family: None) - -# --- Naming ---------------------------------------------------------------------- + def install(opens): + (tmp_path / "libth_modeling_v2_gpt_oss.so").write_bytes(b"") + monkeypatch.setattr(_native, "_open_library", lambda path, family: opens()) + monkeypatch.setattr(_native, "_import_fakes", lambda module, family: None) - -@pytest.mark.parametrize( - "module, namespace", - [ - (_TARGET, "mv2_gpt_oss_gpt_oss_120b_sm_10x_default"), - (f"{_TARGET}.fake_ops", "mv2_gpt_oss_gpt_oss_120b_sm_10x_default"), - (f"{_TARGET}.ops.moe.entry", "mv2_gpt_oss_gpt_oss_120b_sm_10x_default"), - (f"{_FAMILY}.fake_ops", "mv2_gpt_oss"), - (f"{_FAMILY}.ops.comm.entry", "mv2_gpt_oss"), - ], -) -def test_namespace_of_a_target_or_family_module(module, namespace): - assert namespace_of(module) == namespace + return types.SimpleNamespace(install=install) -@pytest.mark.parametrize( - "module", ["tensorrt_llm._torch.models", "tensorrt_llm._torch._experimental.modeling_v2.models"] -) -def test_namespace_of_rejects_modules_outside_a_family(module): +def test_names(): + assert namespace_of(_TARGET) == _TARGET_NS + assert namespace_of(f"{_TARGET}.ops.moe.entry") == _TARGET_NS + assert namespace_of(f"{_FAMILY}.fake_ops") == "mv2_gpt_oss" with pytest.raises(ValueError): - namespace_of(module) - - -@pytest.mark.parametrize("package", [_FAMILY, f"{_TARGET}.weights"]) -def test_load_target_ops_takes_a_target_package_only(package): + namespace_of("tensorrt_llm._torch.models") with pytest.raises(ValueError, match="models.."): - load_target_ops(package) + load_target_ops(f"{_TARGET}.weights") -# --- Decisions ----------------------------------------------------------------- - - -def test_a_library_this_installation_lacks_is_no_target(libs): +def test_what_is_no_target_and_what_is_broken(libs, monkeypatch): with pytest.raises(NoTarget, match="has no libth_modeling_v2_gpt_oss.so"): load_target_ops(_TARGET) - -def test_a_library_that_fails_to_open_is_a_broken_installation(libs, monkeypatch): def fail(): raise OSError("undefined symbol: _ZN12tensorrt_llm6common") - _install(libs, monkeypatch, fail) + libs.install(fail) with pytest.raises(NativeLoadError, match="undefined symbol"): load_target_ops(_TARGET) - -def test_a_target_the_library_does_not_carry_is_no_target(libs, monkeypatch): - _install(libs, monkeypatch, lambda: frozenset({"other__sm_90__default"})) + libs.install(lambda: frozenset({"other__sm_90__default"})) with pytest.raises(NoTarget, match="does not carry gpt_oss_120b__sm_10x__default"): load_target_ops(_TARGET) - -def test_a_library_opens_once(libs, monkeypatch): - opened = [] - - def opens(): - opened.append(1) - return frozenset({"gpt_oss_120b__sm_10x__default"}) - - _install(libs, monkeypatch, opens) - assert _native._load_family_library("gpt_oss") == {"gpt_oss_120b__sm_10x__default"} - assert _native._load_family_library("gpt_oss") == {"gpt_oss_120b__sm_10x__default"} - assert len(opened) == 1 + monkeypatch.setattr(sys, "platform", "win32") + monkeypatch.setattr(_native, "_built_targets", {}) + with pytest.raises(NoTarget, match="Windows"): + load_target_ops(_TARGET) -@pytest.fixture -def torch_ops(monkeypatch): - """A stand-in torch.ops holding the gpt_oss namespaces.""" - ops = types.SimpleNamespace( - mv2_gpt_oss=object(), mv2_gpt_oss_gpt_oss_120b_sm_10x_default=object() - ) +def test_a_library_opens_once_and_registers_the_family_fakes_first(libs, monkeypatch): + ops = types.SimpleNamespace(mv2_gpt_oss=object(), **{_TARGET_NS: object()}) monkeypatch.setitem(sys.modules, "torch", types.SimpleNamespace(ops=ops)) - return ops - - -def test_opening_a_library_registers_the_family_fakes_first(libs, monkeypatch, torch_ops): - _install(libs, monkeypatch, lambda: frozenset({"gpt_oss_120b__sm_10x__default"})) - imported = [] - monkeypatch.setattr(_native, "_import_fakes", lambda module, family: imported.append(module)) - assert load_target_ops(_TARGET) is torch_ops.mv2_gpt_oss_gpt_oss_120b_sm_10x_default - family_package = _TARGET.rpartition(".")[0] - assert imported == [f"{family_package}.fake_ops", f"{_TARGET}.fake_ops"] - - -def test_family_ops_are_the_family_namespace_and_load_once(libs, monkeypatch, torch_ops): - opened = [] + opened, imported = [], [] def opens(): opened.append(1) return frozenset({"gpt_oss_120b__sm_10x__default"}) - _install(libs, monkeypatch, opens) - module = f"{_TARGET}.weights" - assert load_family_ops(module) is torch_ops.mv2_gpt_oss - assert load_family_ops(module) is torch_ops.mv2_gpt_oss + libs.install(opens) + monkeypatch.setattr(_native, "_import_fakes", lambda module, family: imported.append(module)) + assert load_target_ops(_TARGET) is getattr(ops, _TARGET_NS) + assert load_family_ops(f"{_TARGET}.weights") is ops.mv2_gpt_oss + assert load_family_ops(f"{_TARGET}.weights") is ops.mv2_gpt_oss assert len(opened) == 1 + assert imported == [f"{_FAMILY}.fake_ops", f"{_TARGET}.fake_ops"] -def test_family_ops_of_a_family_this_installation_lacks_are_no_target(libs): - with pytest.raises(NoTarget, match="has no libth_modeling_v2_gpt_oss.so"): - load_family_ops(f"{_TARGET}.weights") - - -def test_windows_has_no_native_targets(libs, monkeypatch): - monkeypatch.setattr(sys, "platform", "win32") - with pytest.raises(NoTarget, match="Windows"): - _native._load_family_library("gpt_oss") - - -# --- The installed wheel --------------------------------------------------------- - _LOAD_ONE_FAMILY = textwrap.dedent( """ import json, sys diff --git a/tests/unittest/_torch/modeling_v2/test_modeling_v2_rank_agreement.py b/tests/unittest/_torch/modeling_v2/test_modeling_v2_rank_agreement.py index b296dedc1f01..25480462b733 100644 --- a/tests/unittest/_torch/modeling_v2/test_modeling_v2_rank_agreement.py +++ b/tests/unittest/_torch/modeling_v2/test_modeling_v2_rank_agreement.py @@ -31,85 +31,52 @@ class Target: _TARGET = f"{Target.__module__}.{Target.__qualname__}" -class _FakeDist: - def __init__(self, others): - self.others = others - self.calls = 0 - - def allgather(self, outcome): - self.calls += 1 - return [outcome, *self.others] - - -def _config(rank=0, world_size=2): - return SimpleNamespace(mapping=SimpleNamespace(rank=rank, world_size=world_size)) - - -def _args(mode): - return SimpleNamespace(modeling_v2=mode) - - @pytest.fixture -def dist(monkeypatch): - fake = _FakeDist(others=[]) - monkeypatch.setattr(communicator.Distributed, "get", staticmethod(lambda mapping: fake)) - return fake - - -def _route_to(monkeypatch, target=None, error=None): - """This rank's own decision: what ``modeling_v2_resolve`` returns or raises.""" - - def resolve(config, llm_args): - if error is not None: - raise error - return target - - monkeypatch.setattr(_router_index, "modeling_v2_resolve", resolve) - - -def test_off_exchanges_nothing(monkeypatch, dist): - _route_to(monkeypatch, error=AssertionError("routing ran under off")) - assert modeling_v2_resolve_on_all_ranks(_config(), _args("off")) is None - assert dist.calls == 0 - - -def test_one_rank_exchanges_nothing(monkeypatch, dist): - _route_to(monkeypatch, target=Target) - assert modeling_v2_resolve_on_all_ranks(_config(world_size=1), _args("auto")) is Target - assert dist.calls == 0 - - -def test_agreeing_ranks_build_the_target(monkeypatch, dist): - dist.others = [(_TARGET, None)] - _route_to(monkeypatch, target=Target) - assert modeling_v2_resolve_on_all_ranks(_config(), _args("require")) is Target - assert dist.calls == 1 - - -def test_agreeing_fallbacks_build_the_builtin_model(monkeypatch, dist): - dist.others = [(None, None)] - _route_to(monkeypatch) - assert modeling_v2_resolve_on_all_ranks(_config(), _args("auto")) is None - - -def test_disagreeing_ranks_all_fail(monkeypatch, dist): - dist.others = [(None, None)] - _route_to(monkeypatch, target=Target) +def ranks(monkeypatch): + """``ranks(mode, mine, others, world_size=2)``: resolve with this rank's own + decision ``mine`` (a target, None or an exception) and the other ranks' + outcomes ``others``; returns the result and the number of exchanges.""" + calls = [] + + def resolve_on_all_ranks(mode, mine, others, world_size=2): + def resolve(config, llm_args): + if isinstance(mine, Exception): + raise mine + return mine + + def allgather(outcome): + calls.append(outcome) + return [outcome, *others] + + monkeypatch.setattr(_router_index, "modeling_v2_resolve", resolve) + monkeypatch.setattr( + communicator.Distributed, + "get", + staticmethod(lambda mapping: SimpleNamespace(allgather=allgather)), + ) + calls.clear() + config = SimpleNamespace(mapping=SimpleNamespace(rank=0, world_size=world_size)) + return modeling_v2_resolve_on_all_ranks(config, SimpleNamespace(modeling_v2=mode)) + + resolve_on_all_ranks.calls = calls + return resolve_on_all_ranks + + +def test_off_and_one_rank_exchange_nothing(ranks): + assert ranks("off", AssertionError("routing ran under off"), []) is None + assert ranks("auto", Target, [], world_size=1) is Target + assert ranks.calls == [] + + +def test_ranks_build_the_same_code_or_all_fail(ranks): + assert ranks("require", Target, [(_TARGET, None)]) is Target + assert ranks("auto", None, [(None, None)]) is None with pytest.raises(RuntimeError, match="ranks disagree.*rank 1: built-in"): - modeling_v2_resolve_on_all_ranks(_config(), _args("auto")) - - -def test_another_ranks_error_fails_this_rank(monkeypatch, dist): - dist.others = [(None, "NativeLoadError: libth_modeling_v2_x.so cannot be loaded")] - _route_to(monkeypatch, target=Target) + ranks("auto", Target, [(None, None)]) with pytest.raises(RuntimeError, match="rank 1: NativeLoadError"): - modeling_v2_resolve_on_all_ranks(_config(), _args("auto")) - - -def test_this_ranks_error_is_raised_after_the_exchange(monkeypatch, dist): - """Raising before the exchange would leave the other ranks waiting in it.""" - dist.others = [(_TARGET, None)] - _route_to(monkeypatch, error=ValueError("modeling_v2='require', but no target fits")) + ranks("auto", Target, [(None, "NativeLoadError: libth_modeling_v2_x.so cannot be loaded")]) + # This rank's own error is raised after the exchange, not before it, which + # would leave the other ranks waiting in it. with pytest.raises(ValueError, match="no target fits"): - modeling_v2_resolve_on_all_ranks(_config(), _args("require")) - assert dist.calls == 1 + ranks("require", ValueError("modeling_v2='require', but no target fits"), [(_TARGET, None)]) + assert len(ranks.calls) == 1 diff --git a/tests/unittest/scripts/cbts/test_modeling_v2_rule.py b/tests/unittest/scripts/cbts/test_modeling_v2_rule.py index 5348193d5cb8..548eb7e980b2 100644 --- a/tests/unittest/scripts/cbts/test_modeling_v2_rule.py +++ b/tests/unittest/scripts/cbts/test_modeling_v2_rule.py @@ -83,93 +83,61 @@ def _selected(result) -> set[str]: } -def test_claims_python_and_native_sources_but_not_docs_or_stock(): - assert _is_mv2_claim(f"{_PY}/models/gpt_oss/gpt_oss_120b__sm_10x__default/__init__.py") - assert _is_mv2_claim( - f"{_PY}/models/gpt_oss/gpt_oss_120b__sm_10x__default/csrc/swizzleBlockScales.cu" - ) - assert _is_mv2_claim(f"{_PY}/models/gpt_oss/csrc/CMakeLists.txt") - assert _is_mv2_claim(f"{_CPP}/common/namespace.h") - assert _is_mv2_claim("cpp/cmake/modules/modeling_v2.cmake") - assert not _is_mv2_claim(f"{_CPP}/README.md") - assert not _is_mv2_claim("cpp/tensorrt_llm/thop/attentionOp.cpp") - assert not _is_mv2_claim("tensorrt_llm/_torch/pyexecutor/model_loader.py") - - -def test_family_of_reads_the_family_directory_native_code_included(rule): - assert rule.family_of(f"{_PY}/models/minimax_m3/routing.py") == "minimax_m3" - assert rule.family_of(f"{_PY}/models/minimax_m3/csrc/attnMega.cu") == "minimax_m3" - assert ( - rule.family_of(f"{_PY}/models/minimax_m3/m3__sm_107__tp2/csrc/draftStep.cu") == "minimax_m3" - ) - assert rule.family_of(f"{_PY}/catalog/_op.py") is None - assert rule.family_of(f"{_PY}/models/__init__.py") is None - assert rule.family_of(f"{_CPP}/common/torchLibrary.h") is None - assert rule.family_of(f"{_CPP}/CMakeLists.txt") is None +def test_claims_and_families(): + """Python and native modeling_v2 sources are claimed; docs and stock are not. - -@pytest.mark.parametrize( - "changed", - [ - f"{_PY}/models/gpt_oss/gpt_oss_120b__sm_10x__default/weights.py", + A path under models// belongs to that family; anything else claimed is + shared by every family. + """ + for path in ( f"{_PY}/models/gpt_oss/gpt_oss_120b__sm_10x__default/csrc/swizzleBlockScales.cu", f"{_PY}/models/gpt_oss/csrc/CMakeLists.txt", - ], -) -def test_a_family_change_selects_what_runs_that_family_and_the_checks(rule, changed): - """The directory entry collects every family's unit tests; comm/ is --ignored.""" - result = _apply(rule, changed) - assert result.scope == "modelingv2only" - assert _selected(result) == {_UNIT, _GPT_OSS_ACCURACY, _CLAIMS} - assert result.sanity_relevant is False - - -def test_a_family_keeps_its_own_entries_on_other_gpus(rule): - result = _apply(rule, f"{_PY}/models/kimi_k3_vl/csrc/k3AttnResDecode.cu") - assert _selected(result) == {_UNIT, _KIMI_UNIT, _CLAIMS} - - -def test_two_families_union(rule): - result = _apply( - rule, - f"{_PY}/models/deepseek_v3/routing.py", - f"{_PY}/models/gpt_oss/csrc/CMakeLists.txt", + f"{_CPP}/common/namespace.h", + "cpp/cmake/modules/modeling_v2.cmake", + ): + assert _is_mv2_claim(path), path + for path in ( + f"{_CPP}/README.md", + "cpp/tensorrt_llm/thop/attentionOp.cpp", + "tensorrt_llm/_torch/pyexecutor/model_loader.py", + ): + assert not _is_mv2_claim(path), path + assert ModelingV2Rule.family_of(f"{_PY}/models/minimax_m3/csrc/attnMega.cu") == "minimax_m3" + assert ModelingV2Rule.family_of(f"{_PY}/models/minimax_m3/m3__sm_107__tp2/csrc/d.cu") == ( + "minimax_m3" ) - assert _selected(result) == {_UNIT, _DEEPSEEK_ACCURACY, _GPT_OSS_ACCURACY, _CLAIMS} + for path in (f"{_PY}/models/__init__.py", f"{_PY}/catalog/_op.py", f"{_CPP}/CMakeLists.txt"): + assert ModelingV2Rule.family_of(path) is None, path -@pytest.mark.parametrize( - ("entry", "runs"), - [ - ("unittest/_torch/modeling_v2", True), - ("unittest/_torch/modeling_v2/ --ignore=unittest/_torch/modeling_v2/comm", True), - ("unittest/_torch/modeling_v2/models", True), +def test_entry_runs_family(): + for entry, runs in ( + ("unittest/_torch/modeling_v2 --ignore=unittest/_torch/modeling_v2/comm", True), ("unittest/_torch/modeling_v2/models/gpt_oss/test_x.py::test_y", True), + ("accuracy/modeling_v2/test_gpt_oss.py::TestGptOss", True), ("unittest/_torch/modeling_v2 --ignore=unittest/_torch/modeling_v2/models", False), - ("unittest/_torch/modeling_v2 --ignore=unittest/_torch/modeling_v2/models/gpt_oss/", False), ("unittest/_torch/modeling_v2/models/gpt_oss_mini", False), - ("unittest/_torch/modeling_v2/comm", False), - ("accuracy/modeling_v2/test_gpt_oss.py::TestGptOss", True), ("accuracy/modeling_v2/test_deepseek_v3.py", False), - ], -) -def test_entry_runs_family(entry, runs): - assert ModelingV2Rule._entry_runs_family(entry, _entry_target(entry), "gpt_oss") is runs - - -@pytest.mark.parametrize( - "changed", - [ - f"{_PY}/catalog/_op.py", - f"{_PY}/_native.py", - f"{_CPP}/common/torchLibrary.h", - f"{_CPP}/CMakeLists.txt", - "cpp/cmake/modules/modeling_v2.cmake", - ], -) -def test_shared_modeling_v2_code_selects_everything_of_modeling_v2(rule, changed): - result = _apply(rule, changed) - assert result.scope == "modelingv2only" + ): + assert ModelingV2Rule._entry_runs_family(entry, _entry_target(entry), "gpt_oss") is runs, ( + entry + ) + + +def test_scope(rule): + """A family change selects what runs that family's tests, plus the CPU checks. + + Families union; shared code selects every modeling_v2 entry; a family no entry + runs falls back to the full suite. + """ + result = _apply(rule, f"{_PY}/models/gpt_oss/gpt_oss_120b__sm_10x__default/csrc/k.cu") + assert result.scope == "modelingv2only" and result.sanity_relevant is False + assert _selected(result) == {_UNIT, _GPT_OSS_ACCURACY, _CLAIMS} + result = _apply( + rule, f"{_PY}/models/deepseek_v3/routing.py", f"{_PY}/models/kimi_k3_vl/csrc/k.cu" + ) + assert _selected(result) == {_UNIT, _DEEPSEEK_ACCURACY, _KIMI_UNIT, _CLAIMS} + result = _apply(rule, f"{_CPP}/common/torchLibrary.h") assert _selected(result) == { _UNIT, _COMM, @@ -178,14 +146,7 @@ def test_shared_modeling_v2_code_selects_everything_of_modeling_v2(rule, changed _GPT_OSS_ACCURACY, _CLAIMS, } - - -def test_a_family_no_entry_runs_falls_back(rule): + assert _apply(rule, "tensorrt_llm/_torch/pyexecutor/model_loader.py") is None rule.yaml_index.blocks[0].tests.remove(_UNIT) result = _apply(rule, f"{_PY}/models/minimax_m3/csrc/attnMega.cu") - assert result.scope is None - assert "minimax_m3" in result.reason - - -def test_stock_changes_are_not_claimed(rule): - assert _apply(rule, "tensorrt_llm/_torch/pyexecutor/model_loader.py") is None + assert result.scope is None and "minimax_m3" in result.reason diff --git a/tests/unittest/scripts/test_build_wheel_modeling_v2.py b/tests/unittest/scripts/test_build_wheel_modeling_v2.py index 8a527f0b6410..9f416d1cfba5 100644 --- a/tests/unittest/scripts/test_build_wheel_modeling_v2.py +++ b/tests/unittest/scripts/test_build_wheel_modeling_v2.py @@ -1,71 +1,48 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""CPU-only checks of how build_wheel.py installs the modeling_v2 family libraries.""" +"""CPU-only check of how build_wheel.py installs the modeling_v2 family libraries.""" import importlib.util import shutil from pathlib import Path -from types import ModuleType import pytest pytestmark = pytest.mark.cpu_only -@pytest.fixture(scope="module") -def build_wheel() -> ModuleType: - script = Path(__file__).resolve().parents[3] / "scripts" / "build_wheel.py" - spec = importlib.util.spec_from_file_location("build_wheel", script) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - - -def _family(build_dir: Path, directory: str, archs: str, libraries=()) -> None: - family_dir = build_dir / directory +def _family(build_dir: Path, family: str, archs: str, library: bool) -> None: + family_dir = build_dir / family family_dir.mkdir(parents=True) (family_dir / "cuda_architectures.txt").write_text(archs) - for library in libraries: - (family_dir / library).write_bytes(b"\x7fELF") + if library: + (family_dir / f"libth_modeling_v2_{family}.so").write_bytes(b"\x7fELF") -@pytest.fixture -def dirs(tmp_path: Path): - build_dir = tmp_path / "build" / "tensorrt_llm" / "modeling_v2" - lib_dir = tmp_path / "tensorrt_llm" / "libs" - build_dir.mkdir(parents=True) - lib_dir.mkdir(parents=True) - return build_dir, lib_dir +def test_install_modeling_v2_libs(tmp_path: Path) -> None: + """A family library built for some architecture is installed. + One this build did not produce is removed. + """ + script = Path(__file__).resolve().parents[3] / "scripts" / "build_wheel.py" + spec = importlib.util.spec_from_file_location("build_wheel", script) + build_wheel = importlib.util.module_from_spec(spec) + spec.loader.exec_module(build_wheel) -def test_installs_only_families_with_architectures(build_wheel, dirs) -> None: - build_dir, lib_dir = dirs - _family(build_dir, "fixture", "103a-real", ["libth_modeling_v2_fixture.so"]) - _family(build_dir, "glmMoeDsa", "") + build_dir, lib_dir = tmp_path / "build", tmp_path / "libs" + lib_dir.mkdir() + _family(build_dir, "gpt_oss", "100f-real", library=True) + _family(build_dir, "glm_moe_dsa", "", library=True) # left by an earlier build (lib_dir / "libth_common.so").write_bytes(b"stock") + (lib_dir / "libth_modeling_v2_glm_moe_dsa.so").write_bytes(b"earlier build") + (lib_dir / "libth_modeling_v2_removed.so").symlink_to(lib_dir / "gone.so") build_wheel.install_modeling_v2_libs(build_dir, lib_dir, shutil.copy) - assert sorted(p.name for p in lib_dir.iterdir()) == [ "libth_common.so", - "libth_modeling_v2_fixture.so", + "libth_modeling_v2_gpt_oss.so", ] - -def test_removes_a_library_this_build_did_not_produce(build_wheel, dirs) -> None: - build_dir, lib_dir = dirs - _family(build_dir, "fixture", "", ["libth_modeling_v2_fixture.so"]) - (lib_dir / "libth_modeling_v2_fixture.so").write_bytes(b"earlier build") - (lib_dir / "libth_modeling_v2_removed_family.so").symlink_to(lib_dir / "gone.so") - - build_wheel.install_modeling_v2_libs(build_dir, lib_dir, shutil.copy) - - assert list(lib_dir.iterdir()) == [] - - -def test_a_family_with_architectures_must_have_one_library(build_wheel, dirs) -> None: - build_dir, lib_dir = dirs - _family(build_dir, "fixture", "103a-real") - + _family(build_dir, "qwen4_exp", "103a-real", library=False) with pytest.raises(AssertionError, match="Exactly one modeling_v2 library"): build_wheel.install_modeling_v2_libs(build_dir, lib_dir, shutil.copy) From 8e25d347732236430ac782782148800e200b3185 Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Mon, 26 Oct 2026 21:44:17 -0700 Subject: [PATCH 17/20] [None][fix] modeling_v2: keep gpt_oss's host scale swizzle The gpt_oss weight loader's scale swizzle runs the target's kernel on CUDA tensors only; host tensors take the torch permutation again, which the CPU shard test (test_modeling_v2_gpt_oss_default_shards.py) drives. That test skips when this installation does not carry the target, since importing the target loads its family library, as routing does. The swizzle kernel's test compares the kernel against the host path of the same function, and checks the row and column rejections. The rank agreement test checks that `off` exchanges nothing right after the `off` call. Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- .../gpt_oss_120b__sm_10x__default/weights.py | 12 +++++-- ...odeling_v2_gpt_oss_swizzle_block_scales.py | 35 ++++++++++--------- ...test_modeling_v2_gpt_oss_default_shards.py | 12 +++++-- .../test_modeling_v2_rank_agreement.py | 1 + 4 files changed, 37 insertions(+), 23 deletions(-) diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/models/gpt_oss/gpt_oss_120b__sm_10x__default/weights.py b/tensorrt_llm/_torch/_experimental/modeling_v2/models/gpt_oss/gpt_oss_120b__sm_10x__default/weights.py index 44781c025096..48ed6f1a1813 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/models/gpt_oss/gpt_oss_120b__sm_10x__default/weights.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/models/gpt_oss/gpt_oss_120b__sm_10x__default/weights.py @@ -186,9 +186,15 @@ def _fc1_perm(rows: int, device) -> torch.Tensor: def _swizzle_scales(s: torch.Tensor) -> torch.Tensor: """Per-expert 128x4 block-scale swizzle, viewed back as `[E, M, C]`: the byte at `(m, c)` moves to flat offset `(m//128)*512*(C//4) + (c//4)*512 - + (m%32)*16 + ((m%128)//32)*4 + (c%4)`. This target's own kernel - (csrc/swizzleBlockScales.cu) moves it in one pass on the device.""" - return native_ops.swizzle_block_scales(s) + + (m%32)*16 + ((m%128)//32)*4 + (c%4)`. On the device this target's own + kernel (csrc/swizzleBlockScales.cu) moves it in one pass; on the host, + a torch permutation does.""" + if s.is_cuda: + return native_ops.swizzle_block_scales(s) + e, m, c = s.shape + assert m % 128 == 0 and c % 4 == 0, (m, c) + v = s.reshape(e, m // 128, 4, 32, c // 4, 4) # (e, m/128, (m%128)/32, m%32, c/4, c%4) + return v.permute(0, 1, 4, 3, 2, 5).reshape(e, m, c).contiguous() def _experts(t: torch.Tensor, d) -> torch.Tensor: diff --git a/tests/unittest/_torch/modeling_v2/models/gpt_oss/gpt_oss_120b__sm_10x__default/test_modeling_v2_gpt_oss_swizzle_block_scales.py b/tests/unittest/_torch/modeling_v2/models/gpt_oss/gpt_oss_120b__sm_10x__default/test_modeling_v2_gpt_oss_swizzle_block_scales.py index 53ba2a743735..ab41f587f048 100644 --- a/tests/unittest/_torch/modeling_v2/models/gpt_oss/gpt_oss_120b__sm_10x__default/test_modeling_v2_gpt_oss_swizzle_block_scales.py +++ b/tests/unittest/_torch/modeling_v2/models/gpt_oss/gpt_oss_120b__sm_10x__default/test_modeling_v2_gpt_oss_swizzle_block_scales.py @@ -5,8 +5,8 @@ Importing the target package loads ``libth_modeling_v2_gpt_oss.so`` through ``_native.load_target_ops``, as routing does: a pass here means the family library was built for this GPU, packaged, opened, carries the target, and registered the -fake. The kernel only moves bytes, so it must match the plain-torch swizzle it -replaces bit for bit. +fake. The kernel only moves bytes, so it must match the torch swizzle the loader +runs on the host bit for bit. """ import pytest @@ -24,26 +24,27 @@ ) -def _reference(s: torch.Tensor) -> torch.Tensor: - """The plain-torch swizzle the loader used before the kernel.""" - e, m, c = s.shape - v = s.reshape(e, m // 128, 4, 32, c // 4, 4) - return v.permute(0, 1, 4, 3, 2, 5).reshape(e, m, c).contiguous() - - def test_swizzle_block_scales(): g = torch.Generator(device="cuda").manual_seed(0) - # gpt-oss-120b's FC1 and FC2 scales (FC2's 90 scales per row padded to 96), and no experts. + # gpt-oss-120b's FC1 and FC2 scales (FC2's 90 scales per row padded to 96), and no experts: + # the kernel, on the device, against the torch permutation the loader runs on the host. for shape in ((4, 5888, 92), (3, 384, 96), (0, 128, 4)): s = torch.randint(0, 256, shape, generator=g, device="cuda", dtype=torch.uint8) - assert torch.equal(weights._swizzle_scales(s), _reference(s)), shape + assert torch.equal(weights._swizzle_scales(s).cpu(), weights._swizzle_scales(s.cpu())), ( + shape + ) - strided = torch.zeros(2, 128, 8, device="cuda", dtype=torch.uint8)[:, :, :4] - with pytest.raises(RuntimeError, match="contiguous"): - native_ops.swizzle_block_scales(strided) + def scales(rows, cols): + return torch.zeros(2, rows, cols, device="cuda", dtype=torch.uint8) + + for bad, match in ( + (scales(127, 4), "got rows 127, cols 4"), + (scales(128, 6), "got rows 128, cols 6"), + (scales(128, 8)[:, :, :4], "contiguous"), + ): + with pytest.raises(RuntimeError, match=match): + native_ops.swizzle_block_scales(bad) with FakeTensorMode(): - out = native_ops.swizzle_block_scales( - torch.empty(2, 256, 8, device="cuda", dtype=torch.uint8) - ) + out = native_ops.swizzle_block_scales(scales(256, 8)) assert out.shape == (2, 256, 8) and out.dtype == torch.uint8 diff --git a/tests/unittest/_torch/modeling_v2/test_modeling_v2_gpt_oss_default_shards.py b/tests/unittest/_torch/modeling_v2/test_modeling_v2_gpt_oss_default_shards.py index 226dbc8232b2..0529d8b775ef 100644 --- a/tests/unittest/_torch/modeling_v2/test_modeling_v2_gpt_oss_default_shards.py +++ b/tests/unittest/_torch/modeling_v2/test_modeling_v2_gpt_oss_default_shards.py @@ -31,16 +31,22 @@ import pytest import torch +from tensorrt_llm._torch._experimental.modeling_v2._router_index import NoTarget from tensorrt_llm._torch._experimental.modeling_v2.models.gpt_oss import routing -from tensorrt_llm._torch._experimental.modeling_v2.models.gpt_oss.gpt_oss_120b__sm_10x__default import ( # noqa: E501 - weights as D, -) from tensorrt_llm._torch.moe.fused_moe.quantization import ( _get_weight_alignment, _requires_rank_local_tp_padding, ) from tensorrt_llm.mapping import Mapping +# Importing the target loads its family library, as routing does. +try: + from tensorrt_llm._torch._experimental.modeling_v2.models.gpt_oss.gpt_oss_120b__sm_10x__default import ( # noqa: E501 + weights as D, + ) +except NoTarget as no_target: + pytest.skip(str(no_target), allow_module_level=True) + pytestmark = pytest.mark.cpu_only _INTER = 2880 diff --git a/tests/unittest/_torch/modeling_v2/test_modeling_v2_rank_agreement.py b/tests/unittest/_torch/modeling_v2/test_modeling_v2_rank_agreement.py index 25480462b733..290ea9a8e1ab 100644 --- a/tests/unittest/_torch/modeling_v2/test_modeling_v2_rank_agreement.py +++ b/tests/unittest/_torch/modeling_v2/test_modeling_v2_rank_agreement.py @@ -64,6 +64,7 @@ def allgather(outcome): def test_off_and_one_rank_exchange_nothing(ranks): assert ranks("off", AssertionError("routing ran under off"), []) is None + assert ranks.calls == [] assert ranks("auto", Target, [], world_size=1) is Target assert ranks.calls == [] From 29a844865e8399641ff743cc4c28bdb91554107f Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Mon, 26 Oct 2026 22:16:28 -0700 Subject: [PATCH 18/20] [None][chore] modeling_v2: always build the family libraries; name the not-built case The family libraries are always built: BUILD_MODELING_V2 and the Windows branch of the discovery are removed, as is the loader's Windows check. TRTLLM_MODELING_V2_TARGETS still narrows a local build. A target whose native code this installation did not build raises NativeCodeNotBuilt, a NoTarget subclass: NoTarget alone says that no target fits the configuration, while this says the target exists but its library or its code is missing. As a NoTarget it keeps the fallback under `auto` and the error under `require`. Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- cpp/CMakeLists.txt | 4 --- cpp/tensorrt_llm/modeling_v2/CMakeLists.txt | 14 -------- .../_experimental/modeling_v2/README.md | 7 ++-- .../_experimental/modeling_v2/_native.py | 35 ++++++++++++------- ...test_modeling_v2_gpt_oss_default_shards.py | 6 ++-- .../test_modeling_v2_native_loader.py | 19 +++++----- 6 files changed, 37 insertions(+), 48 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 064f80b4f5e3..9c68ecd60f6c 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -49,10 +49,6 @@ option(BUILD_DEEP_EP "Build the Deep EP module" ON) option(BUILD_NCCL_EXTENSIONS "Build the nccl-extensions Python wheel" ON) option(BUILD_DEEP_GEMM "Build the DeepGEMM module" ON) option(BUILD_FLASH_MLA "Build the FlashMLA module" ON) -option( - BUILD_MODELING_V2 - "Build the modeling_v2 per-family torch-op libraries (libth_modeling_v2_*)" - ON) set(TRTLLM_MODELING_V2_TARGETS "all" CACHE diff --git a/cpp/tensorrt_llm/modeling_v2/CMakeLists.txt b/cpp/tensorrt_llm/modeling_v2/CMakeLists.txt index 0f84199709ba..3785811be8cf 100644 --- a/cpp/tensorrt_llm/modeling_v2/CMakeLists.txt +++ b/cpp/tensorrt_llm/modeling_v2/CMakeLists.txt @@ -58,20 +58,6 @@ foreach(arch_file IN LISTS previous_arch_files) endif() endforeach() -if(WIN32 OR NOT BUILD_MODELING_V2) - # As deep_ep does on a build without a matching architecture: an empty - # cuda_architectures.txt per family, so build_wheel.py installs nothing, - # including a library left from an earlier configuration. - foreach(family_list IN LISTS family_lists) - get_filename_component(csrc_dir "${family_list}" DIRECTORY) - get_filename_component(family_dir "${csrc_dir}" DIRECTORY) - get_filename_component(family "${family_dir}" NAME) - file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/${family}/cuda_architectures.txt" - "") - endforeach() - return() -endif() - foreach(family_list IN LISTS family_lists) get_filename_component(csrc_dir "${family_list}" DIRECTORY) get_filename_component(family_dir "${csrc_dir}" DIRECTORY) diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/README.md b/tensorrt_llm/_torch/_experimental/modeling_v2/README.md index fe0fa24e2b0b..9d1926e7cb5c 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/README.md +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/README.md @@ -217,12 +217,13 @@ reached with `load_family_ops(__name__)` and faked in `models/foo/fake_ops.py`. with no code of its own declares no `SOURCES`. - **`ARCHS`** (`NN`, `NNa`, `NNf`) is intersected with the build's architectures; a target - with none is not built, and is a `NoTarget` at run time. `REQUIRES` (`MULTI_DEVICE`, + with none is not built, and is a `NativeCodeNotBuilt` at run time. `REQUIRES` (`MULTI_DEVICE`, `NVSHMEM`) and `DEPS` (a closed list) are the other keywords; a per-file compile option goes on the file with `set_source_files_properties`. - **Loading**: a library this installation does not carry, or a target it did not build, - is a `NoTarget` (`auto` keeps the built-in model); one that is installed but fails to - load, or lacks an op a `fake_ops.py` declares, is a `NativeLoadError`. + is a `NativeCodeNotBuilt`, a `NoTarget` (`auto` keeps the built-in model); one that is + installed but fails to load, or lacks an op a `fake_ops.py` declares, is a + `NativeLoadError`. - **Building only your family**: `build_wheel.py -D "TRTLLM_MODELING_V2_TARGETS=foo/*"`, and after an edit `cmake --build cpp/build --target th_modeling_v2_foo`. - `test_modeling_v2_native_claims.py` checks these rules on CPU; run it before sending the PR. diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/_native.py b/tensorrt_llm/_torch/_experimental/modeling_v2/_native.py index c76fb1b5caba..9091a66155c2 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/_native.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/_native.py @@ -7,7 +7,7 @@ (a target's own), and builds into one torch-op library, ``libth_modeling_v2_.so``, installed next to ``libth_common.so``. Nothing of it is in the stock libraries, and nothing loads it until routing picks a target of -the family. The API is four names: +the family. The API is five names: * :func:`load_target_ops` -- a target package with native code starts with ``native_ops = load_target_ops(__name__)``; @@ -15,15 +15,17 @@ from any module of the family; * :func:`namespace_of` -- the torch namespace a ``fake_ops.py`` registers its fakes in, ``namespace_of(__name__)``; +* :class:`NativeCodeNotBuilt` -- a target whose native code this installation did + not build; * :class:`NativeLoadError` -- a library that is installed but broken. ``route`` imports a target only on the branch that returns it, so loading is part of routing, and it follows how the stock code treats its optional native modules (``flash_mla``, ``deep_ep``): a library this installation does not have, or a target it did not build (it answers ``torch.ops.mv2_.built_targets()``), is a -:class:`NoTarget`, so ``auto`` keeps the built-in model; a library that fails to -load, or lacks an op a ``fake_ops.py`` declares, is a :class:`NativeLoadError`, -never a fallback. The family's ``fake_ops.py`` registers when the library opens, a +:class:`NativeCodeNotBuilt`, so ``auto`` keeps the built-in model; a library that +fails to load, or lacks an op a ``fake_ops.py`` declares, is a +:class:`NativeLoadError`, never a fallback. The family's ``fake_ops.py`` registers when the library opens, a target's when the target is found in it. The "Native code" section of the modeling_v2 README is the guide for authors. @@ -34,7 +36,6 @@ import importlib import importlib.util import re -import sys from pathlib import Path from typing import Any, Dict, FrozenSet, Optional, Tuple @@ -50,6 +51,17 @@ _family_ops: Dict[str, Any] = {} +class NativeCodeNotBuilt(NoTarget): + """The target exists, but this installation did not build its native code. + + Its family's library is not installed, or was built without the target: its + ``ARCHS`` miss the build's, a ``REQUIRES`` is unmet, or + ``TRTLLM_MODELING_V2_TARGETS`` left it out. It is a :class:`NoTarget`, so + ``auto`` keeps the built-in model and ``require`` reports why, as for a target + that does not fit. + """ + + class NativeLoadError(RuntimeError): """A family library is installed but cannot be used. @@ -125,14 +137,11 @@ def _load_family_library(family: str) -> FrozenSet[str]: """Open ``libth_modeling_v2_.so`` once and return the targets it carries.""" if family in _built_targets: return _built_targets[family] - if sys.platform == "win32": - raise NoTarget("modeling_v2 native code is not built on Windows") path = _LIBS_DIR / _library_name(family) if not path.is_file(): - raise NoTarget( + raise NativeCodeNotBuilt( f"this installation has no {path.name}: the build compiles a family's library " - "only when its CUDA architectures include one of the family's targets, and " - "only with BUILD_MODELING_V2=ON" + "only when its CUDA architectures include one of the family's targets" ) try: built = _open_library(path, family) @@ -148,8 +157,8 @@ def load_target_ops(package: str) -> Any: namespace, ``torch.ops.mv2__``. Call it from the target's ``__init__.py`` with ``__name__``. Raises - :class:`NoTarget` when this installation does not carry the target and - :class:`NativeLoadError` when its library is broken. + :class:`NativeCodeNotBuilt` when this installation does not carry the target + and :class:`NativeLoadError` when its library is broken. """ family, target = _split(package) if target is None or package != f"{_MODELS_PACKAGE}.{family}.{target}": @@ -157,7 +166,7 @@ def load_target_ops(package: str) -> Any: built = _load_family_library(family) if target not in built: carried = ", ".join(sorted(built)) or "none" - raise NoTarget( + raise NativeCodeNotBuilt( f"{_library_name(family)} does not carry {target}: this build did not compile " f"it (its ARCHS miss the build's, a REQUIRES is unmet, or " f"TRTLLM_MODELING_V2_TARGETS excluded it); it carries {carried}" diff --git a/tests/unittest/_torch/modeling_v2/test_modeling_v2_gpt_oss_default_shards.py b/tests/unittest/_torch/modeling_v2/test_modeling_v2_gpt_oss_default_shards.py index 0529d8b775ef..7930ce8bfb78 100644 --- a/tests/unittest/_torch/modeling_v2/test_modeling_v2_gpt_oss_default_shards.py +++ b/tests/unittest/_torch/modeling_v2/test_modeling_v2_gpt_oss_default_shards.py @@ -31,7 +31,7 @@ import pytest import torch -from tensorrt_llm._torch._experimental.modeling_v2._router_index import NoTarget +from tensorrt_llm._torch._experimental.modeling_v2._native import NativeCodeNotBuilt from tensorrt_llm._torch._experimental.modeling_v2.models.gpt_oss import routing from tensorrt_llm._torch.moe.fused_moe.quantization import ( _get_weight_alignment, @@ -44,8 +44,8 @@ from tensorrt_llm._torch._experimental.modeling_v2.models.gpt_oss.gpt_oss_120b__sm_10x__default import ( # noqa: E501 weights as D, ) -except NoTarget as no_target: - pytest.skip(str(no_target), allow_module_level=True) +except NativeCodeNotBuilt as not_built: + pytest.skip(str(not_built), allow_module_level=True) pytestmark = pytest.mark.cpu_only diff --git a/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_loader.py b/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_loader.py index 3934771eeadb..e0c3f73ef13a 100644 --- a/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_loader.py +++ b/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_loader.py @@ -3,9 +3,9 @@ """What ``_native`` decides, and that the installed family libraries load. The first tests drive the loader's decisions without opening anything: which -outcomes are ``NoTarget`` (this installation does not carry the target, so -``auto`` keeps the built-in model) and which are ``NativeLoadError`` (an -installed library is broken, which is never a fallback). +outcomes are ``NativeCodeNotBuilt`` (this installation does not carry the +target; a ``NoTarget``, so ``auto`` keeps the built-in model) and which are +``NativeLoadError`` (an installed library is broken, which is never a fallback). The last is the load smoke test of the installed wheel: every family library in ``tensorrt_llm/libs`` opens in a fresh process, names the targets it carries, and @@ -27,6 +27,7 @@ from tensorrt_llm._torch._experimental.modeling_v2 import _native from tensorrt_llm._torch._experimental.modeling_v2._native import ( + NativeCodeNotBuilt, NativeLoadError, load_family_ops, load_target_ops, @@ -68,8 +69,9 @@ def test_names(): load_target_ops(f"{_TARGET}.weights") -def test_what_is_no_target_and_what_is_broken(libs, monkeypatch): - with pytest.raises(NoTarget, match="has no libth_modeling_v2_gpt_oss.so"): +def test_what_is_not_built_and_what_is_broken(libs): + assert issubclass(NativeCodeNotBuilt, NoTarget) # what makes `auto` fall back + with pytest.raises(NativeCodeNotBuilt, match="has no libth_modeling_v2_gpt_oss.so"): load_target_ops(_TARGET) def fail(): @@ -80,12 +82,7 @@ def fail(): load_target_ops(_TARGET) libs.install(lambda: frozenset({"other__sm_90__default"})) - with pytest.raises(NoTarget, match="does not carry gpt_oss_120b__sm_10x__default"): - load_target_ops(_TARGET) - - monkeypatch.setattr(sys, "platform", "win32") - monkeypatch.setattr(_native, "_built_targets", {}) - with pytest.raises(NoTarget, match="Windows"): + with pytest.raises(NativeCodeNotBuilt, match="does not carry gpt_oss_120b__sm_10x__default"): load_target_ops(_TARGET) From 1412f285009ab14fe4ad469fa372860050650ba3 Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Mon, 26 Oct 2026 22:46:30 -0700 Subject: [PATCH 19/20] [None][chore] modeling_v2: reuse NoTarget for missing native code Keep the loader API at four names. Missing native code means no usable target in this installation; auto falls back, require errors, and an installed but broken library still raises NativeLoadError. Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- .../_experimental/modeling_v2/README.md | 7 +++-- .../_experimental/modeling_v2/_native.py | 27 +++++-------------- ...test_modeling_v2_gpt_oss_default_shards.py | 6 ++--- .../test_modeling_v2_native_loader.py | 12 ++++----- 4 files changed, 18 insertions(+), 34 deletions(-) diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/README.md b/tensorrt_llm/_torch/_experimental/modeling_v2/README.md index 9d1926e7cb5c..fe0fa24e2b0b 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/README.md +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/README.md @@ -217,13 +217,12 @@ reached with `load_family_ops(__name__)` and faked in `models/foo/fake_ops.py`. with no code of its own declares no `SOURCES`. - **`ARCHS`** (`NN`, `NNa`, `NNf`) is intersected with the build's architectures; a target - with none is not built, and is a `NativeCodeNotBuilt` at run time. `REQUIRES` (`MULTI_DEVICE`, + with none is not built, and is a `NoTarget` at run time. `REQUIRES` (`MULTI_DEVICE`, `NVSHMEM`) and `DEPS` (a closed list) are the other keywords; a per-file compile option goes on the file with `set_source_files_properties`. - **Loading**: a library this installation does not carry, or a target it did not build, - is a `NativeCodeNotBuilt`, a `NoTarget` (`auto` keeps the built-in model); one that is - installed but fails to load, or lacks an op a `fake_ops.py` declares, is a - `NativeLoadError`. + is a `NoTarget` (`auto` keeps the built-in model); one that is installed but fails to + load, or lacks an op a `fake_ops.py` declares, is a `NativeLoadError`. - **Building only your family**: `build_wheel.py -D "TRTLLM_MODELING_V2_TARGETS=foo/*"`, and after an edit `cmake --build cpp/build --target th_modeling_v2_foo`. - `test_modeling_v2_native_claims.py` checks these rules on CPU; run it before sending the PR. diff --git a/tensorrt_llm/_torch/_experimental/modeling_v2/_native.py b/tensorrt_llm/_torch/_experimental/modeling_v2/_native.py index 9091a66155c2..a96e8f52edda 100644 --- a/tensorrt_llm/_torch/_experimental/modeling_v2/_native.py +++ b/tensorrt_llm/_torch/_experimental/modeling_v2/_native.py @@ -7,7 +7,7 @@ (a target's own), and builds into one torch-op library, ``libth_modeling_v2_.so``, installed next to ``libth_common.so``. Nothing of it is in the stock libraries, and nothing loads it until routing picks a target of -the family. The API is five names: +the family. The API is four names: * :func:`load_target_ops` -- a target package with native code starts with ``native_ops = load_target_ops(__name__)``; @@ -15,17 +15,15 @@ from any module of the family; * :func:`namespace_of` -- the torch namespace a ``fake_ops.py`` registers its fakes in, ``namespace_of(__name__)``; -* :class:`NativeCodeNotBuilt` -- a target whose native code this installation did - not build; * :class:`NativeLoadError` -- a library that is installed but broken. ``route`` imports a target only on the branch that returns it, so loading is part of routing, and it follows how the stock code treats its optional native modules (``flash_mla``, ``deep_ep``): a library this installation does not have, or a target it did not build (it answers ``torch.ops.mv2_.built_targets()``), is a -:class:`NativeCodeNotBuilt`, so ``auto`` keeps the built-in model; a library that -fails to load, or lacks an op a ``fake_ops.py`` declares, is a -:class:`NativeLoadError`, never a fallback. The family's ``fake_ops.py`` registers when the library opens, a +:class:`NoTarget`, so ``auto`` keeps the built-in model; a library that fails to +load, or lacks an op a ``fake_ops.py`` declares, is a :class:`NativeLoadError`, +never a fallback. The family's ``fake_ops.py`` registers when the library opens, a target's when the target is found in it. The "Native code" section of the modeling_v2 README is the guide for authors. @@ -51,17 +49,6 @@ _family_ops: Dict[str, Any] = {} -class NativeCodeNotBuilt(NoTarget): - """The target exists, but this installation did not build its native code. - - Its family's library is not installed, or was built without the target: its - ``ARCHS`` miss the build's, a ``REQUIRES`` is unmet, or - ``TRTLLM_MODELING_V2_TARGETS`` left it out. It is a :class:`NoTarget`, so - ``auto`` keeps the built-in model and ``require`` reports why, as for a target - that does not fit. - """ - - class NativeLoadError(RuntimeError): """A family library is installed but cannot be used. @@ -139,7 +126,7 @@ def _load_family_library(family: str) -> FrozenSet[str]: return _built_targets[family] path = _LIBS_DIR / _library_name(family) if not path.is_file(): - raise NativeCodeNotBuilt( + raise NoTarget( f"this installation has no {path.name}: the build compiles a family's library " "only when its CUDA architectures include one of the family's targets" ) @@ -157,7 +144,7 @@ def load_target_ops(package: str) -> Any: namespace, ``torch.ops.mv2__``. Call it from the target's ``__init__.py`` with ``__name__``. Raises - :class:`NativeCodeNotBuilt` when this installation does not carry the target + :class:`NoTarget` when this installation does not carry the target and :class:`NativeLoadError` when its library is broken. """ family, target = _split(package) @@ -166,7 +153,7 @@ def load_target_ops(package: str) -> Any: built = _load_family_library(family) if target not in built: carried = ", ".join(sorted(built)) or "none" - raise NativeCodeNotBuilt( + raise NoTarget( f"{_library_name(family)} does not carry {target}: this build did not compile " f"it (its ARCHS miss the build's, a REQUIRES is unmet, or " f"TRTLLM_MODELING_V2_TARGETS excluded it); it carries {carried}" diff --git a/tests/unittest/_torch/modeling_v2/test_modeling_v2_gpt_oss_default_shards.py b/tests/unittest/_torch/modeling_v2/test_modeling_v2_gpt_oss_default_shards.py index 7930ce8bfb78..0529d8b775ef 100644 --- a/tests/unittest/_torch/modeling_v2/test_modeling_v2_gpt_oss_default_shards.py +++ b/tests/unittest/_torch/modeling_v2/test_modeling_v2_gpt_oss_default_shards.py @@ -31,7 +31,7 @@ import pytest import torch -from tensorrt_llm._torch._experimental.modeling_v2._native import NativeCodeNotBuilt +from tensorrt_llm._torch._experimental.modeling_v2._router_index import NoTarget from tensorrt_llm._torch._experimental.modeling_v2.models.gpt_oss import routing from tensorrt_llm._torch.moe.fused_moe.quantization import ( _get_weight_alignment, @@ -44,8 +44,8 @@ from tensorrt_llm._torch._experimental.modeling_v2.models.gpt_oss.gpt_oss_120b__sm_10x__default import ( # noqa: E501 weights as D, ) -except NativeCodeNotBuilt as not_built: - pytest.skip(str(not_built), allow_module_level=True) +except NoTarget as no_target: + pytest.skip(str(no_target), allow_module_level=True) pytestmark = pytest.mark.cpu_only diff --git a/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_loader.py b/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_loader.py index e0c3f73ef13a..eb969fa92536 100644 --- a/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_loader.py +++ b/tests/unittest/_torch/modeling_v2/test_modeling_v2_native_loader.py @@ -3,9 +3,9 @@ """What ``_native`` decides, and that the installed family libraries load. The first tests drive the loader's decisions without opening anything: which -outcomes are ``NativeCodeNotBuilt`` (this installation does not carry the -target; a ``NoTarget``, so ``auto`` keeps the built-in model) and which are -``NativeLoadError`` (an installed library is broken, which is never a fallback). +outcomes are ``NoTarget`` (this installation does not carry the target, so +``auto`` keeps the built-in model) and which are ``NativeLoadError`` (an +installed library is broken, which is never a fallback). The last is the load smoke test of the installed wheel: every family library in ``tensorrt_llm/libs`` opens in a fresh process, names the targets it carries, and @@ -27,7 +27,6 @@ from tensorrt_llm._torch._experimental.modeling_v2 import _native from tensorrt_llm._torch._experimental.modeling_v2._native import ( - NativeCodeNotBuilt, NativeLoadError, load_family_ops, load_target_ops, @@ -70,8 +69,7 @@ def test_names(): def test_what_is_not_built_and_what_is_broken(libs): - assert issubclass(NativeCodeNotBuilt, NoTarget) # what makes `auto` fall back - with pytest.raises(NativeCodeNotBuilt, match="has no libth_modeling_v2_gpt_oss.so"): + with pytest.raises(NoTarget, match="has no libth_modeling_v2_gpt_oss.so"): load_target_ops(_TARGET) def fail(): @@ -82,7 +80,7 @@ def fail(): load_target_ops(_TARGET) libs.install(lambda: frozenset({"other__sm_90__default"})) - with pytest.raises(NativeCodeNotBuilt, match="does not carry gpt_oss_120b__sm_10x__default"): + with pytest.raises(NoTarget, match="does not carry gpt_oss_120b__sm_10x__default"): load_target_ops(_TARGET) From 5cf96b9deb6e0ce1fe92916333466e35dbebe374 Mon Sep 17 00:00:00 2001 From: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> Date: Tue, 27 Oct 2026 19:07:33 -0700 Subject: [PATCH 20/20] [None][fix] modeling_v2: retain upstream node-scoped CBTS after rebase MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Preserve main’s hierarchical test selection and repo_root integration while claiming native infrastructure and supporting covering directory entries with ignore options. Keep the B300 directory entry and native loader behavior. Signed-off-by: Wanli Jiang <35160485+Wanli-Jiang@users.noreply.github.com> --- jenkins/scripts/cbts/README.md | 4 +- jenkins/scripts/cbts/cbts/command/main.py | 2 +- jenkins/scripts/cbts/cbts/rules/README.md | 19 +- .../scripts/cbts/cbts/rules/modeling_v2.py | 317 +++++++++++------- .../scripts/cbts/test_modeling_v2_rule.py | 38 ++- 5 files changed, 228 insertions(+), 152 deletions(-) diff --git a/jenkins/scripts/cbts/README.md b/jenkins/scripts/cbts/README.md index e1a45484f168..20453820c954 100644 --- a/jenkins/scripts/cbts/README.md +++ b/jenkins/scripts/cbts/README.md @@ -50,7 +50,7 @@ Nine rules, registered in `cbts/command/main.py::RULE_CLASSES`: | `TestListRule` | `testlistonly` | `tests/integration/test_lists/test-db/*.yml` | | `VisualGenRule` | `visualgenonly` | `examples/visual_gen/**`, `scripts/visualgen_eval/**`, `tensorrt_llm/_torch/visual_gen/**`, `tensorrt_llm/media/**`, `tensorrt_llm/visual_gen/**` (excl. docs; reference images such as `cat_piano.png` ARE test fixtures and stay claimed; outward-facing files force fallback) | | `SpecDecRule` | `specdeconly` | `tensorrt_llm/_torch/speculative/**`, `tensorrt_llm/models/{eagle,medusa,redrafter}/**`, `examples/{eagle,medusa,redrafter,draft_target_model,ngram}/**`, `examples/llm-api/llm_speculative_decoding.py` (excl. docs; other suffixes incl. images kept as potential test fixtures) | -| `ModelingV2Rule` | `modelingv2only` | `tensorrt_llm/_torch/_experimental/modeling_v2/**`, `cpp/tensorrt_llm/modeling_v2/**`, `cpp/cmake/modules/modeling_v2.cmake` (excl. docs); per family when only one family changed | +| `ModelingV2Rule` | `modelingv2only` | `tensorrt_llm/_torch/_experimental/modeling_v2/**`, `cpp/tensorrt_llm/modeling_v2/**`, `cpp/cmake/modules/modeling_v2.cmake` (excl. docs); per model node for changes under `models/` | | `AgentFlowRule` | `agentflowonly` | `agent-flow/**` (excl. docs and out-of-scope metadata) | | `OpenEngineRule` | `openengineonly` | `tensorrt_llm/grpc/openengine/**` (excl. docs) | | `DocsRule` | `docsonly` | Outside noop paths: `docs/**` and verified Python docstring edits → `CPU-Build_Docs` plus complete `l0_cpu`; other `**/*.md` / `**/*.rst` → `CPU-Build_Docs` only | @@ -67,7 +67,7 @@ See `cbts/rules/README.md` for per-rule logic. | `testlistonly` | `TestListRule` fired solo: PR only adds entries under `tests/integration/test_lists/test-db/*.yml`. | | `visualgenonly` | `VisualGenRule` fired solo: PR only touches VisualGen internal source paths (`examples/visual_gen/**`, `scripts/visualgen_eval/**`, `tensorrt_llm/_torch/visual_gen/**`; excl. docs; image fixtures like `cat_piano.png` are claimed). Narrows to blocks containing VG test entries. Outward-facing files under `tensorrt_llm/visual_gen/**` and `tensorrt_llm/media/**` (eagerly imported by `trtllm-serve`) force `null` fallback. | | `specdeconly` | `SpecDecRule` fired solo: PR only touches speculative-decoding source paths (`tensorrt_llm/_torch/speculative/**`, `tensorrt_llm/models/{eagle,medusa,redrafter}/**`, `examples/{eagle,medusa,redrafter,draft_target_model,ngram}/**`, `examples/llm-api/llm_speculative_decoding.py`; excl. docs). Narrows to blocks containing spec-dec test entries (eagle / medusa / redrafter / ngram / draft-target-model / MTP). | -| `modelingv2only` | `ModelingV2Rule` fired solo: PR only touches modeling_v2 code -- the subtree (`tensorrt_llm/_torch/_experimental/modeling_v2/**`, each family's native code in its `csrc/` included: one torch-op library per model family) or what the family libraries share (`cpp/tensorrt_llm/modeling_v2/**`, `cpp/cmake/modules/modeling_v2.cmake`); excl. docs, which are a fifth of the Python subtree — every catalog entry carries a contract document. A change confined to one family's directory narrows to the entries that run that family's tests (`accuracy/modeling_v2/test_.py`, and the unit entries that collect `unittest/_torch/modeling_v2/models/`) plus the modeling_v2 CPU checks; any other modeling_v2 change narrows to every modeling_v2 entry. No outward-facing fallback is needed: nothing beyond the resolver module is imported, and no family library is loaded, unless `LLM(modeling_v2=...)` is on; the CPU checks that keep this true run in every scope. The callers outside the subtree (`_torch/pyexecutor/model_loader.py`, `_torch/models/modeling_auto.py`) are left unclaimed, so touching them falls back to baseline. | +| `modelingv2only` | `ModelingV2Rule` fired solo: PR only touches modeling_v2 code -- the subtree (`tensorrt_llm/_torch/_experimental/modeling_v2/**`, each family's native code in its `csrc/` included: one torch-op library per model family) or what the family libraries share (`cpp/tensorrt_llm/modeling_v2/**`, `cpp/cmake/modules/modeling_v2.cmake`); excl. docs, which are a fifth of the Python subtree — every catalog entry carries a contract document. A change under `models/` selects its model node's unit tests and target accuracy gates, plus whole-tree checks; shared code or an unreadable routing tree selects every modeling_v2 entry. No outward-facing fallback is needed: nothing beyond the resolver module is imported, and no family library is loaded, unless `LLM(modeling_v2=...)` is on; the CPU checks that keep this true run in every scope. The callers outside the subtree (`_torch/pyexecutor/model_loader.py`, `_torch/models/modeling_auto.py`) are left unclaimed, so touching them falls back to baseline. | | `agentflowonly` | `AgentFlowRule` fired solo: PR only touches `agent-flow/**` source or test files (excl. docs). Runs `CPU-AgentFlow-UnitTest`. | | `openengineonly` | `OpenEngineRule` fired solo: PR only touches `tensorrt_llm/grpc/openengine/**` source files (excl. docs). Narrows to the registered OpenEngine unit tests: the stub-based ones on the always-run `CPU-Generic-*` stages, plus `test_capability_conformance.py` on `A10-PyTorch-*`, which needs a GPU. | | `docsonly` | `DocsRule` fired solo: all documentation changes run `CPU-Build_Docs`; changes under `docs/**` also run the complete `l0_cpu` suite on matching `CPU-Generic-*` stages. Docs-only orchestration skips the SBSA track, so its resolved ARM stage is not launched. | diff --git a/jenkins/scripts/cbts/cbts/command/main.py b/jenkins/scripts/cbts/cbts/command/main.py index 17ca3e11079c..2c1b045a4c23 100644 --- a/jenkins/scripts/cbts/cbts/command/main.py +++ b/jenkins/scripts/cbts/cbts/command/main.py @@ -101,7 +101,7 @@ def build_rules( TestListRule(yaml_index, stages, repo_root=repo_root), VisualGenRule(yaml_index, stages), SpecDecRule(yaml_index, stages), - ModelingV2Rule(yaml_index, stages), + ModelingV2Rule(yaml_index, stages, repo_root=repo_root), AgentFlowRule(yaml_index, stages), OpenEngineRule(yaml_index, stages), DocsRule(yaml_index, stages), diff --git a/jenkins/scripts/cbts/cbts/rules/README.md b/jenkins/scripts/cbts/cbts/rules/README.md index 7e31df823161..e3605f493dec 100644 --- a/jenkins/scripts/cbts/cbts/rules/README.md +++ b/jenkins/scripts/cbts/cbts/rules/README.md @@ -16,7 +16,7 @@ for the overall CBTS architecture. | `test_list.py` | `TestListRule` | `testlistonly` | `tests/integration/test_lists/test-db/*.yml` | | `visual_gen.py` | `VisualGenRule` | `visualgenonly` | `examples/visual_gen/**`, `scripts/visualgen_eval/**`, `tensorrt_llm/_torch/visual_gen/**`, `tensorrt_llm/media/**`, `tensorrt_llm/visual_gen/**` (each excl. docs) | | `spec_dec.py` | `SpecDecRule` | `specdeconly` | `tensorrt_llm/_torch/speculative/**`, `tensorrt_llm/models/{eagle,medusa,redrafter}/**`, `examples/{eagle,medusa,redrafter,draft_target_model,ngram}/**`, `examples/llm-api/llm_speculative_decoding.py` (each excl. docs) | -| `modeling_v2.py` | `ModelingV2Rule` | `modelingv2only` | `tensorrt_llm/_torch/_experimental/modeling_v2/**`, `cpp/tensorrt_llm/modeling_v2/**`, `cpp/cmake/modules/modeling_v2.cmake` (excl. docs); per family when only one family changed | +| `modeling_v2.py` | `ModelingV2Rule` | `modelingv2only` | `tensorrt_llm/_torch/_experimental/modeling_v2/**`, `cpp/tensorrt_llm/modeling_v2/**`, `cpp/cmake/modules/modeling_v2.cmake` (excl. docs); per model node for changes under `models/` | | `agent_flow.py` | `AgentFlowRule` | `agentflowonly` | `agent-flow/**` (excl. docs and out-of-scope metadata) → the single `CPU-AgentFlow-UnitTest` stage; not test-db-driven | | `openengine.py` | `OpenEngineRule` | `openengineonly` | `tensorrt_llm/grpc/openengine/**` (excl. docs) → the `l0_cpu` block containing `unittest/grpc/openengine/` | | `docs.py` | `DocsRule` | `docsonly` | Outside noop paths: `docs/**` and verified Python docstring edits → `CPU-Build_Docs` plus complete `l0_cpu`; other `**/*.md` / `**/*.rst` → `CPU-Build_Docs` only | @@ -309,15 +309,14 @@ per model family, loaded only when routing picks one of the family's targets), plus what every family library shares: `cpp/tensorrt_llm/modeling_v2/` and the native build module `cpp/cmake/modules/modeling_v2.cmake`. -Scope — per family, else all of modeling_v2: a change confined to one -family's directory (`models//`, its native code included) keeps the -entries that run that family's tests -- `accuracy/modeling_v2/test_.py`, -and any unit entry that collects `unittest/_torch/modeling_v2/models/` -(the modeling_v2 directory entry, which does not `--ignore` it, or an entry -under it) -- plus the modeling_v2 CPU checks -(`unittest/_torch/modeling_v2/test_modeling_v2_*.py`). Several families union. -Any other modeling_v2 change keeps every modeling_v2 entry. A claimed family -no entry runs falls back to baseline. +Scope — per model node, else all of modeling_v2: source changes under +`models/` (native `csrc/` included) select the nearest model node's unit tests +and the accuracy gates of its targets. Family-shared code selects the whole +family. A covering directory entry is retained when no more specific entry +exists, unless it ignores that node. Whole-tree unit checks are always retained. +Shared Python or native infrastructure selects every modeling_v2 entry, as does +an unreadable routing tree. If no modeling_v2 entry matches, the rule falls back +to baseline. `.md` exclusion carries more weight here than elsewhere: every catalog entry ships a contract document, so roughly a fifth of the subtree is diff --git a/jenkins/scripts/cbts/cbts/rules/modeling_v2.py b/jenkins/scripts/cbts/cbts/rules/modeling_v2.py index 152b2179bca6..fee9b67e260b 100644 --- a/jenkins/scripts/cbts/cbts/rules/modeling_v2.py +++ b/jenkins/scripts/cbts/cbts/rules/modeling_v2.py @@ -11,54 +11,82 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""ModelingV2Rule — narrows CI when only modeling_v2 code changes. - -modeling_v2 is a second modeling path: one self-contained forward per -(checkpoint, GPU arch, parallel topology), assembled from a catalog of op -wrappers. It lives under `tensorrt_llm/_torch/_experimental/modeling_v2/`, a -family's native code included (`models//csrc/`, -`models///csrc/`), which builds one torch-op library per model -family, loaded only when routing picks one of the family's targets. What every -family library shares is in `cpp/tensorrt_llm/modeling_v2/` and -`cpp/cmake/modules/modeling_v2.cmake`. - -Scope — per family, else all of modeling_v2: -A change confined to one family's directory (`models//`, its native -code included) selects the entries that run that family's tests -- -`accuracy/modeling_v2/test_.py`, and any unit entry that collects -`unittest/_torch/modeling_v2/models//` (the modeling_v2 directory entry, -or one under the family's directory) -- plus the modeling_v2 CPU checks. Any -other modeling_v2 change (catalog, resolver, loader, the native build module, its -discovery and common headers) selects every modeling_v2 entry. Several families -in one PR union their entries. - -Why no outward fallback is needed: nothing beyond the resolver is imported, and -no family library is loaded, unless `LLM(modeling_v2=...)` is on -- the model -loader calls the resolver, which returns immediately when the switch is off, -and a target's library opens only when routing imports that target. Family -libraries are separate modules, never linked into libtensorrt_llm.so or -libth_common.so. The claims tests under `tests/unittest/_torch/modeling_v2/` -keep that true (stock never references a target, targets never reach a -sibling, `modeling_v2` defaults to `off`); they run in every scope this rule -selects. The callers outside the subtree (`_torch/pyexecutor/model_loader.py`, -`_torch/models/modeling_auto.py`) stay unclaimed: a change to them runs the -baseline. - -`.md` exclusion matters here: every catalog entry carries a contract document, -so a fifth of the Python subtree is Markdown; claiming it would let a +"""ModelingV2Rule — narrows CI when the modeling_v2 subtree changes. + +modeling_v2 is a second modeling path living entirely under +`tensorrt_llm/_torch/_experimental/modeling_v2/`: one self-contained forward per +(checkpoint, GPU arch, parallel topology), assembled from a catalog of +op wrappers. Native code lives beside each model in `csrc/`; shared build +infrastructure lives under `cpp/tensorrt_llm/modeling_v2/` and +`cpp/cmake/modules/modeling_v2.cmake` and selects every modeling_v2 entry. + +Block selection: a block belongs to modeling_v2 iff one of its `tests:` +entries matches a marker in `_MV2_ENTRY_PATTERNS`. Those markers are exact +by construction rather than by luck: every unit test file in the subtree is +named `test_modeling_v2_*` precisely so it cannot collide with the upstream +test of the same op, and the accuracy gates live in their own +`accuracy/modeling_v2/` directory. + +Entry selection: each changed file maps to the tests it can affect. + +- `models/` is a tree of nodes. A directory is a node when it, or a + directory below it, holds a target's `modeling.py`; a family directory + (`models//`) is always one. Other directories -- `ops/` and the + like -- serve the node above them. A changed file belongs to the nearest + node at or above it, and selects that node's unit tests + (`unittest/_torch/modeling_v2/models/`) plus the accuracy gate of + every target under it. A target's gate is `Test` in + `accuracy/modeling_v2/test_.py`, the class name read from the + import line in the family's `routing.py` that routes to it. +- Anything else -- the catalog, the resolver, `models/__init__.py` -- can + reach every target and selects every modeling_v2 entry. + +The tests directly under `unittest/_torch/modeling_v2/` check the tree as a +whole (claims, routing, contracts), so any change keeps them. + +A unit test prefix that no entry of a block sits at or under keeps the +longest entry of that block that covers it instead: a family is one +directory entry, and a change to one of its targets runs the family's +unit tests. A test file listed at a directory above a node -- a family's +own test file -- belongs to every node under that directory, so a change +to any of them keeps it. Whenever the tree on disk cannot answer -- a node directory +deleted with its `routing.py` -- the file falls back to the wider +selection. So does a node holding a `modeling.py` that its family's +`routing.py` does not import, such as a target nested below the family's +first level: its accuracy gate cannot be named. + +Outward fallback: not needed, and that is a property of the design +rather than an accident. Nothing beyond the resolver module is imported +unless `LLM(modeling_v2=...)` is on: the model loader calls +`modeling_v2_resolve`, which returns immediately when the switch is off, +and the routing modules are imported lazily behind it. The callers +outside the subtree are `tensorrt_llm/_torch/pyexecutor/model_loader.py` +(decides) and `tensorrt_llm/_torch/models/modeling_auto.py` (reads the +decision off `ModelConfig`), neither of which this rule claims -- a PR +touching them falls back to baseline, which is what a change to the +shared resolver deserves. + +`.md` exclusion matters more here than for most rules: the catalog +carries a contract document per entry, so roughly a fifth of the files +in the subtree are Markdown. Claiming them would make a documentation-only PR pull in multi-GPU GB300 stages. -PerfSanity policy: `perfsanity_relevant` is True only when a selected block -lives in a `*_perf_sanity*` yaml -- same as VisualGenRule / SpecDecRule. +PerfSanity policy: `perfsanity_relevant` is dynamic, True only when a +matched block lives in a `*_perf_sanity*` yaml -- same as VisualGenRule +/ SpecDecRule. modeling_v2 has no perf-sanity entries +today, so this aggregates to False and Groovy Layer 2 drops the +force-keep of `*-PerfSanity-*` stages. -Sanity policy: `sanity_relevant=False`. Nothing here is imported by -`trtllm-serve` or `import tensorrt_llm`, and the family libraries are not -loaded by either, so nothing here is what PackageSanityCheck verifies. +Sanity policy: `sanity_relevant=False`. The subtree ships no +user-facing entry point and is not imported by `trtllm-serve` or by +`import tensorrt_llm`, so nothing it contains is what PackageSanityCheck +verifies about the wheel. """ from __future__ import annotations import re +from pathlib import Path from typing import Optional from cbts.blocks import Stage, YAMLIndex, _entry_target, _target_in_filter_subtree @@ -67,36 +95,33 @@ from .base import PRInputs, Rule, RuleResult from .docs import is_docs_path -_PY_ROOT = "tensorrt_llm/_torch/_experimental/modeling_v2/" -_PY_MODELS = _PY_ROOT + "models/" -# What every family library shares: the discovery and the common/ macros. -_CPP_ROOT = "cpp/tensorrt_llm/modeling_v2/" -# The native build module: shared by every family library. +# Source-path prefixes the rule may claim. Tests under tests/** are left +# to TestsDefRule; the two scopes combine via _TESTSONLY_FAMILY. +_MV2_SRC_PREFIXES: tuple[str, ...] = ( + "tensorrt_llm/_torch/_experimental/modeling_v2/", + "cpp/tensorrt_llm/modeling_v2/", +) _CPP_BUILD_FILES = frozenset({"cpp/cmake/modules/modeling_v2.cmake"}) - -# Source-path prefixes the rule may claim. Tests under tests/** are left to -# TestsDefRule; the two scopes combine via _TESTSONLY_FAMILY. -_MV2_SRC_PREFIXES: tuple[str, ...] = (_PY_ROOT, _CPP_ROOT) +_IGNORE = re.compile(r"--ignore=(\S+)") _UNIT_ROOT = "unittest/_torch/modeling_v2" _ACCURACY_ROOT = "accuracy/modeling_v2" # Substrings that mark a test entry as modeling_v2. All are unambiguous: -# - "unittest/_torch/modeling_v2/" → the op-level catalog matrix and the -# per-family unit tests (`.../models/`) +# - "unittest/_torch/modeling_v2/" → the op and target unit tests # - "accuracy/modeling_v2/" → the per-model accuracy gates -# - "test_modeling_v2_" → any unit file named by the subtree's own convention +# - "test_modeling_v2_" → any future unit file named by the subtree's +# own convention _MV2_ENTRY_PATTERNS: tuple[str, ...] = ( f"{_UNIT_ROOT}/", f"{_ACCURACY_ROOT}/", "test_modeling_v2_", ) -# The modeling_v2 CPU checks (claims, routing, native loader): selected in -# every scope, because they are what keeps the narrowing sound. -_MV2_CHECK_ENTRY = re.compile(r"^unittest/_torch/modeling_v2/test_modeling_v2_[\w]+\.py$") - -_IGNORE = re.compile(r"--ignore=(\S+)") +# A family's `routing.py` imports each target it routes to by this line, from +# the target directory directly under the family. The claims test reads the +# same line with the same pattern. +_TARGET_IMPORT = re.compile(r"^\s*from \.(\w+)\.modeling import (\w+)$", re.MULTILINE) def _is_mv2_claim(path: str) -> bool: @@ -110,11 +135,39 @@ def _is_mv2_claim(path: str) -> bool: """ if not (path.startswith(_MV2_SRC_PREFIXES) or path in _CPP_BUILD_FILES): return False - return not is_docs_path(path) + if is_docs_path(path): + return False + return True def _entry_is_mv2(entry: str) -> bool: - return any(p in entry for p in _MV2_ENTRY_PATTERNS) + return entry.split()[0].rstrip("/") == _UNIT_ROOT or any( + p in entry for p in _MV2_ENTRY_PATTERNS + ) + + +def _mv2_entries(block) -> list[str]: + return [t for t in block.tests if _entry_is_mv2(t)] + + +def _is_whole_tree_test(target: str) -> bool: + """A test directly under the unit root, which checks the tree as a whole.""" + rest = target.removeprefix(f"{_UNIT_ROOT}/").split("::")[0] + return target.startswith(f"{_UNIT_ROOT}/") and "/" not in rest and rest.endswith(".py") + + +def _is_ancestor_file(target: str, prefix: str) -> bool: + """A test file in a directory below the models root that holds `prefix`. + + A test file at a family (or an intermediate) directory belongs to every + node under it, so a change to any of those nodes runs it. + """ + parent, _, name = target.split("::")[0].rpartition("/") + return ( + name.endswith(".py") + and parent.startswith(f"{_UNIT_ROOT}/models/") + and _target_in_filter_subtree(prefix, parent) + ) def _is_perf_sanity_stem(stem: str) -> bool: @@ -130,12 +183,20 @@ def _kept_entries(entries: list[str], prefixes: set[str]) -> list[str]: file in a directory above a prefix is kept as well; it does not count as sitting under the prefix, so it never displaces the covering entry. """ - targets = {entry: _entry_target(entry) for entry in entries} + targets = {entry: _entry_target(entry).split()[0] for entry in entries} kept = {e for e, t in targets.items() if _is_whole_tree_test(t)} for prefix in prefixes: under = {e for e, t in targets.items() if _target_in_filter_subtree(t, prefix)} if not under: - covering = [e for e, t in targets.items() if _target_in_filter_subtree(prefix, t)] + covering = [ + e + for e, t in targets.items() + if _target_in_filter_subtree(prefix, t) + and not any( + _target_in_filter_subtree(prefix, ignored.rstrip("/")) + for ignored in _IGNORE.findall(e) + ) + ] under = {max(covering, key=lambda e: len(targets[e]))} if covering else set() kept |= under kept |= {e for e, t in targets.items() if _is_ancestor_file(t, prefix)} @@ -146,90 +207,99 @@ class ModelingV2Rule(Rule): name = "modelingv2" needs_diff_for: tuple[str, ...] = () - def __init__(self, yaml_index: YAMLIndex, stages: dict[str, Stage]) -> None: + def __init__(self, yaml_index: YAMLIndex, stages: dict[str, Stage], repo_root: Path) -> None: self.yaml_index = yaml_index self._stages_by_yaml = stages_by_yaml_stem(stages) + self._src_root = Path(repo_root) / _MV2_SRC_PREFIXES[0] + self._tests_root = Path(repo_root) / "tests" - @staticmethod - def family_of(path: str) -> Optional[str]: - """The family a claimed path belongs to, or None for shared modeling_v2 code.""" - if path.startswith(_PY_MODELS): - parts = path[len(_PY_MODELS) :].split("/") - return parts[0] if len(parts) > 1 else None - return None + def _is_node(self, rel_dir: Path) -> bool: + """Whether `models/` is a node: a family, or holds a target at or below it.""" + path = self._src_root / "models" / rel_dir + return len(rel_dir.parts) == 1 or (path.is_dir() and any(path.rglob("modeling.py"))) - @staticmethod - def _entry_runs_family(entry: str, target: str, family: str) -> bool: - """Whether a test-db entry runs tests of `family`. + def _gates_under(self, node: Path) -> Optional[set[str]]: + """Accuracy-gate prefixes of every target at or under `node`. - Its accuracy gates, or unit tests under its directory, collected by the - entry itself or by a directory entry above it that does not `--ignore` it. + None if unreadable: no `routing.py`, or a `modeling.py` under `node` + that no import line in it routes to, whose gate cannot be named. """ - path = target.split()[0] # the target keeps long options such as --ignore - accuracy = f"accuracy/modeling_v2/test_{family}.py" - if path == accuracy or path.startswith(accuracy + "::"): - return True - unit = f"unittest/_torch/modeling_v2/models/{family}" - if path == unit or path.startswith(unit + "/"): - return True - if not unit.startswith(path.rstrip("/") + "/"): - return False - ignored = (path.rstrip("/") for path in _IGNORE.findall(entry)) - return not any(unit == path or unit.startswith(path + "/") for path in ignored) - - def _select(self, families: Optional[set[str]]): - """Select the modeling_v2 entries a change runs. - - Every modeling_v2 entry when `families` is None, else the entries of - `families` plus the CPU checks. Also returns the families that matched - no entry. - """ - block_filters: dict[tuple[str, int], dict[str, set[str]]] = {} - matched_families: set[str] = set() - for block in self.yaml_index.blocks: - for entry in block.tests: - if not _entry_is_mv2(entry): - continue - target = _entry_target(entry) - if not target: - continue - if families is not None: - owners = {f for f in families if self._entry_runs_family(entry, target, f)} - if not owners and not _MV2_CHECK_ENTRY.match(target): - continue - matched_families |= owners - key = (block.yaml_stem, block.block_index) - block_filters.setdefault(key, {}).setdefault(target, set()).add(entry) - unmatched = set() if families is None else families - matched_families - return block_filters, unmatched + models = self._src_root / "models" + family = node.parts[0] + routing = models / family / "routing.py" + if not routing.is_file(): + return None + routed, gates = set(), set() + for target, target_cls in _TARGET_IMPORT.findall(routing.read_text(encoding="utf-8")): + leaf = Path(family, target) + routed.add(leaf) + if leaf == node or node in leaf.parents: + gates.add(f"{_ACCURACY_ROOT}/test_{family}.py::Test{target_cls}") + if any( + m.parent.relative_to(models) not in routed for m in (models / node).rglob("modeling.py") + ): + return None + return gates + + def _prefixes_for(self, path: str) -> Optional[set[str]]: + """Filter prefixes a changed file selects; None selects every modeling_v2 entry.""" + if not path.startswith(_MV2_SRC_PREFIXES[0]): + return None + rel = Path(path[len(_MV2_SRC_PREFIXES[0]) :]) + if rel.parts[0] != "models" or len(rel.parts) < 3: + return None + node = Path(*rel.parts[1:-1]) + while not self._is_node(node): + node = node.parent + gates = self._gates_under(node) + if gates is None: + return None + return {f"{_UNIT_ROOT}/models/{node.as_posix()}"} | gates def apply(self, pr: PRInputs) -> Optional[RuleResult]: claimed = {f for f in pr.changed_files if _is_mv2_claim(f)} if not claimed: return None - owners = {self.family_of(f) for f in claimed} - families = None if None in owners else set(owners) - block_filters, unmatched = self._select(families) + prefixes: Optional[set[str]] = set() + for path in sorted(claimed): + selected = self._prefixes_for(path) + if selected is None: + prefixes = None + break + prefixes |= selected - if not block_filters or unmatched: - # Do not fabricate stages -- fall back to baseline so the change - # still gets coverage. Reachable if no entry runs a family's - # tests, or the subtree's entries leave the test-db. - missing = f" for {sorted(unmatched)}" if unmatched else "" + block_filters: dict[tuple[str, int], dict[str, set[str]]] = {} + for block in self.yaml_index.blocks: + entries = _mv2_entries(block) + if prefixes is not None: + entries = _kept_entries(entries, prefixes) + if not entries: + continue + prefix_dict = block_filters.setdefault((block.yaml_stem, block.block_index), {}) + for entry in entries: + target = _entry_target(entry) + if target: + prefix_dict.setdefault(target, set()).add(entry) + + if not block_filters: + # Defensive: modeling_v2 source changed but no modeling_v2 block + # exists in any yaml. Do not fabricate stages -- fall back to + # baseline so the change still gets coverage. Reachable if the + # subtree's entries are ever removed from the test-db without + # the subtree going with them. return RuleResult( handled_files=claimed, affected_stages=set(), scope=None, reason=( f"modelingv2: {len(claimed)} modeling_v2 source file(s); " - f"no modeling_v2 test entry{missing} in any test-db yaml — fallback" + "no modeling_v2 block matched in any test-db yaml — fallback" ), ) affected = resolve_affected_stages(block_filters, self.yaml_index, self._stages_by_yaml) perfsanity_relevant = any(_is_perf_sanity_stem(stem) for stem, _ in block_filters) - scope_text = "all modeling_v2" if families is None else f"families {sorted(families)}" return RuleResult( handled_files=claimed, @@ -239,7 +309,8 @@ def apply(self, pr: PRInputs) -> Optional[RuleResult]: sanity_relevant=False, perfsanity_relevant=perfsanity_relevant, reason=( - f"modelingv2: {len(claimed)} modeling_v2 source file(s), {scope_text} → " - f"{len(block_filters)} modeling_v2 block(s), {len(affected)} stage(s)" + f"modelingv2: {len(claimed)} modeling_v2 source file(s) → " + f"{'every modeling_v2 entry' if prefixes is None else f'{len(prefixes)} test prefix(es)'}, " + f"{len(block_filters)} block(s), {len(affected)} stage(s)" ), ) diff --git a/tests/unittest/scripts/cbts/test_modeling_v2_rule.py b/tests/unittest/scripts/cbts/test_modeling_v2_rule.py index 548eb7e980b2..88026448641b 100644 --- a/tests/unittest/scripts/cbts/test_modeling_v2_rule.py +++ b/tests/unittest/scripts/cbts/test_modeling_v2_rule.py @@ -20,9 +20,9 @@ import pytest __extra_import_path__ = ["~/jenkins/scripts/cbts"] -from cbts.blocks import Block, Stage, YAMLIndex, _entry_target +from cbts.blocks import Block, Stage, YAMLIndex from cbts.rules.base import PRInputs -from cbts.rules.modeling_v2 import ModelingV2Rule, _is_mv2_claim +from cbts.rules.modeling_v2 import ModelingV2Rule, _is_mv2_claim, _kept_entries pytestmark = pytest.mark.cpu_only @@ -37,7 +37,7 @@ @pytest.fixture -def rule() -> ModelingV2Rule: +def rule(tmp_path) -> ModelingV2Rule: yaml_index = YAMLIndex() yaml_index.blocks = [ Block( @@ -67,7 +67,16 @@ def rule() -> ModelingV2Rule: "B200-PyTorch-1": Stage("B200-PyTorch-1", "l0_b200", "x86_64", 1, 1), "CPU-Generic-x86-1": Stage("CPU-Generic-x86-1", "l0_cpu", "x86_64", 1, 1), } - return ModelingV2Rule(yaml_index, stages) + for family, target, cls in ( + ("gpt_oss", "gpt_oss_120b__sm_10x__default", "GptOss"), + ("deepseek_v3", "target", "Deepseek"), + ("kimi_k3_vl", "target", "Kimi"), + ): + root = tmp_path / _PY / "models" / family + (root / target).mkdir(parents=True) + (root / target / "modeling.py").touch() + (root / "routing.py").write_text(f"from .{target}.modeling import {cls}\n") + return ModelingV2Rule(yaml_index, stages, repo_root=tmp_path) def _apply(rule: ModelingV2Rule, *files: str): @@ -102,12 +111,6 @@ def test_claims_and_families(): "tensorrt_llm/_torch/pyexecutor/model_loader.py", ): assert not _is_mv2_claim(path), path - assert ModelingV2Rule.family_of(f"{_PY}/models/minimax_m3/csrc/attnMega.cu") == "minimax_m3" - assert ModelingV2Rule.family_of(f"{_PY}/models/minimax_m3/m3__sm_107__tp2/csrc/d.cu") == ( - "minimax_m3" - ) - for path in (f"{_PY}/models/__init__.py", f"{_PY}/catalog/_op.py", f"{_CPP}/CMakeLists.txt"): - assert ModelingV2Rule.family_of(path) is None, path def test_entry_runs_family(): @@ -119,16 +122,18 @@ def test_entry_runs_family(): ("unittest/_torch/modeling_v2/models/gpt_oss_mini", False), ("accuracy/modeling_v2/test_deepseek_v3.py", False), ): - assert ModelingV2Rule._entry_runs_family(entry, _entry_target(entry), "gpt_oss") is runs, ( - entry - ) + prefixes = { + "unittest/_torch/modeling_v2/models/gpt_oss", + "accuracy/modeling_v2/test_gpt_oss.py", + } + assert bool(_kept_entries([entry], prefixes)) is runs, entry def test_scope(rule): """A family change selects what runs that family's tests, plus the CPU checks. - Families union; shared code selects every modeling_v2 entry; a family no entry - runs falls back to the full suite. + Families union; shared code selects every modeling_v2 entry; an unreadable family + selects all modeling_v2 entries. """ result = _apply(rule, f"{_PY}/models/gpt_oss/gpt_oss_120b__sm_10x__default/csrc/k.cu") assert result.scope == "modelingv2only" and result.sanity_relevant is False @@ -149,4 +154,5 @@ def test_scope(rule): assert _apply(rule, "tensorrt_llm/_torch/pyexecutor/model_loader.py") is None rule.yaml_index.blocks[0].tests.remove(_UNIT) result = _apply(rule, f"{_PY}/models/minimax_m3/csrc/attnMega.cu") - assert result.scope is None and "minimax_m3" in result.reason + assert result.scope == "modelingv2only" + assert _COMM in _selected(result) # unreadable family falls back to all modeling_v2