Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 2 additions & 4 deletions .github/instructions/cpp-ci-test-policy.instructions.md
Original file line number Diff line number Diff line change
Expand Up @@ -34,11 +34,9 @@ unload, not CUDA kernel correctness or trained-model quality.

The packaging pipeline includes the unconditional `cpp_test_engine` stage from
`.pipelines/v2/templates/stages-test-engine.yml`. It runs on the existing `onnxruntime-Ubuntu2404-AMD-CPU` pool image
without a custom container and uses standard CPU NuGet packages with a test-only Engine-capable GenAI pin.
`FOUNDRY_LOCAL_REQUIRE_DYNAMIC_ENGINE_TESTS=ON` rejects packages that cannot compile the suite. The XML gate rejects
without a custom container and uses the same stable ORT and GenAI packages as the SDK build. The XML gate rejects
missing lifecycle cases, skipped/disabled/unexecuted tests, and failures. Do not bypass failures with skips or
`continueOnError`. Its binaries are not published as SDK artifacts, and shipping/release dependency pins are unchanged.
Generator-only builds can still exclude the suite; the unconditional Engine lane supplies the required coverage.
`continueOnError`. Its binaries are not published as SDK artifacts.

## Debugging skips in CI

Expand Down
15 changes: 5 additions & 10 deletions .pipelines/foundry-local-packaging.yml
Original file line number Diff line number Diff line change
Expand Up @@ -53,17 +53,12 @@ parameters:

variables:
- group: FoundryLocal-ESRP-Signing
# C++ SDK (sdk_v2/cpp) native dependency versions. Release builds use the
# stable pins from sdk_v2/deps_versions.json; non-release CI validates the
# selected GenAI nightly before it becomes a stable SDK default.
# C++ SDK (sdk_v2/cpp) native dependency versions. Must match the stable pins
# in sdk_v2/deps_versions.json.
- name: cppOrtVersion
value: '1.28.0'
- ${{ if eq(parameters.isRelease, true) }}:
- name: cppGenaiVersion
value: '0.15.2'
- ${{ else }}:
- name: cppGenaiVersion
value: '0.16.0-dev1001400138'
value: '1.30.0'
- name: cppGenaiVersion
value: '0.16.0'
- name: cppWinmlVersion
value: '2.1.70'
- name: cppBuildConfig
Expand Down
5 changes: 2 additions & 3 deletions .pipelines/v2/sdk_v2-pipeline-plan.md
Original file line number Diff line number Diff line change
Expand Up @@ -284,9 +284,8 @@ purposes:

Versions are pipeline-level variables, currently:

* `ortVersion` `1.28.0` (`Microsoft.ML.OnnxRuntime`)
* `genaiVersion` `0.15.2` for releases; selected ORT-Nightly version for non-release CI
(`Microsoft.ML.OnnxRuntimeGenAI.Foundry`)
* `ortVersion` `1.30.0` (`Microsoft.ML.OnnxRuntime`)
* `genaiVersion` `0.16.0` (`Microsoft.ML.OnnxRuntimeGenAI.Foundry`)
* `winmlVersion` `2.1.70` (`Microsoft.Windows.AI.MachineLearning`, WinML 2.x reg-free)

These must be kept in sync with the cmake defaults and with
Expand Down
3 changes: 2 additions & 1 deletion .pipelines/v2/templates/stages-sdk-v2.yml
Original file line number Diff line number Diff line change
Expand Up @@ -33,11 +33,12 @@ stages:
genaiVersion: ${{ parameters.genaiVersion }}
winmlVersion: ${{ parameters.winmlVersion }}

# Test-only CUDA runtime; never replaces the native packaging artifacts.
# Dedicated Engine lifecycle test stage.
- template: stages-test-engine.yml
parameters:
buildConfig: ${{ parameters.buildConfig }}
ortVersion: ${{ parameters.ortVersion }}
genaiVersion: ${{ parameters.genaiVersion }}

# ── C# SDK ──
- template: stages-cs.yml
Expand Down
6 changes: 2 additions & 4 deletions .pipelines/v2/templates/stages-test-engine.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# CPU paged-attention Engine gate, independent of the Generator/release packaging matrix.
# CPU paged-attention Engine gate.
# Model assets are checked in and staged by the normal C++ testdata build rule.
parameters:
- name: buildConfig
Expand All @@ -7,7 +7,6 @@ parameters:
type: string
- name: genaiVersion
type: string
default: '0.16.0-dev1001407373'

stages:
- stage: cpp_test_engine
Expand Down Expand Up @@ -48,8 +47,7 @@ stages:
--cmake_extra_defines \
"GENAI_FETCH_URL=$NUGET_CACHE/genai.zip" \
"ORT_GENAI_VERSION=$ENGINE_GENAI_VERSION" \
"ORT_FETCH_URL=$NUGET_CACHE/ort.zip" \
"FOUNDRY_LOCAL_REQUIRE_DYNAMIC_ENGINE_TESTS=ON"
"ORT_FETCH_URL=$NUGET_CACHE/ort.zip"
build="$PWD/build/Linux/$BUILD_CONFIG"
bin="$build/bin"
export LD_LIBRARY_PATH="$bin:${LD_LIBRARY_PATH:-}"
Expand Down
61 changes: 4 additions & 57 deletions sdk_v2/cpp/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -50,12 +50,6 @@ option(FOUNDRY_LOCAL_BUILD_TESTS "Build unit tests" ON)
option(FOUNDRY_LOCAL_BUILD_EXAMPLES "Build example programs" ON)
option(FOUNDRY_LOCAL_BUILD_SERVICE "Build web service support (requires oat++)" ON)
option(FOUNDRY_LOCAL_ENABLE_ASAN "Enable AddressSanitizer + UndefinedBehaviorSanitizer (Linux only)" OFF)
option(FOUNDRY_LOCAL_REQUIRE_DYNAMIC_ENGINE_TESTS
"Fail configuration unless the ORT GenAI dynamic Engine tests can be built"
OFF)
if(FOUNDRY_LOCAL_REQUIRE_DYNAMIC_ENGINE_TESTS AND NOT FOUNDRY_LOCAL_BUILD_TESTS)
message(FATAL_ERROR "FOUNDRY_LOCAL_REQUIRE_DYNAMIC_ENGINE_TESTS requires FOUNDRY_LOCAL_BUILD_TESTS=ON")
endif()

# Optional 1DS ingestion token override. Override only via environment so it does
# not appear in CMake cache files.
Expand Down Expand Up @@ -115,50 +109,6 @@ endif()
# ORT and ORT GenAI — acquired via FetchContent from nuget.org.
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
find_package(OnnxRuntimeGenAI REQUIRED)

include(CheckCXXSourceCompiles)

set(CMAKE_REQUIRED_LIBRARIES OnnxRuntimeGenAI::OnnxRuntimeGenAI)
unset(FOUNDRY_LOCAL_OGA_HAS_DYNAMIC_ENGINE CACHE)
check_cxx_source_compiles("
#include <cstddef>
#include <cstdint>
#include <memory>

#include <ort_genai.h>

int main() {
auto* model = reinterpret_cast<OgaModel*>(1);
auto engine = OgaEngine::Create(*model);
auto events = engine->CreateEventBuffer(1);
auto request_options = OgaRequestOptions::Create();
auto request = engine->CreateRequest(request_options.get());
auto turn_options = request->CreateTurnOptions();
turn_options->SetMaxGeneratedTokens(static_cast<uint64_t>(1));
turn_options->SetDoSample(true);
turn_options->SetTemperature(1.0f);
turn_options->SetTopP(1.0f);
turn_options->SetTopK(1);
turn_options->SetSeed(static_cast<uint64_t>(1));
const char* values[] = {\"stop\"};
auto stop_strings = OgaStringArray::Create(values, 1);
turn_options->SetStopStrings(*stop_strings);
turn_options->SetGuidance(\"\", \"\");
const auto finish_reason = OgaFinishReason_StopString;
return events && request && turn_options && finish_reason == OgaFinishReason_Eos ? 0 : 1;
}
" FOUNDRY_LOCAL_OGA_HAS_DYNAMIC_ENGINE)
unset(CMAKE_REQUIRED_LIBRARIES)

if(FOUNDRY_LOCAL_OGA_HAS_DYNAMIC_ENGINE)
message(STATUS "ORT GenAI dynamic Engine API: enabled")
else()
if(FOUNDRY_LOCAL_REQUIRE_DYNAMIC_ENGINE_TESTS)
message(FATAL_ERROR
"FOUNDRY_LOCAL_REQUIRE_DYNAMIC_ENGINE_TESTS requires an ORT GenAI package with the dynamic Engine API")
endif()
message(STATUS "ORT GenAI dynamic Engine API: unavailable; using Generator backend")
endif()
find_package(OnnxRuntime REQUIRED)

# WinML EP Catalog — Windows-only, for hardware EP discovery and download. The
Expand Down Expand Up @@ -331,12 +281,10 @@ set(FOUNDRY_LOCAL_SOURCES
${FOUNDRY_LOCAL_INTERNAL_HEADERS}
)

if(FOUNDRY_LOCAL_OGA_HAS_DYNAMIC_ENGINE)
list(APPEND FOUNDRY_LOCAL_SOURCES
src/inferencing/generative/chat/onnx_chat_engine.cc
src/inferencing/generative/chat/onnx_engine_chat_stream.cc
)
endif()
list(APPEND FOUNDRY_LOCAL_SOURCES
src/inferencing/generative/chat/onnx_chat_engine.cc
src/inferencing/generative/chat/onnx_engine_chat_stream.cc
)
Comment thread
baijumeswani marked this conversation as resolved.

# 1DS bridge — always compiled for Foundry Local Core.
list(APPEND FOUNDRY_LOCAL_SOURCES src/telemetry/one_ds_telemetry.cc)
Expand Down Expand Up @@ -412,7 +360,6 @@ function(foundry_local_configure_target TARGET LINK_SCOPE)
${TARGET}
PRIVATE
FOUNDRY_LOCAL_HAS_EP_BOOTSTRAPPERS=$<BOOL:${FOUNDRY_LOCAL_HAS_EP_BOOTSTRAPPERS}>
FOUNDRY_LOCAL_OGA_HAS_DYNAMIC_ENGINE=$<BOOL:${FOUNDRY_LOCAL_OGA_HAS_DYNAMIC_ENGINE}>
)

if(FOUNDRY_LOCAL_BUILD_SERVICE)
Expand Down
30 changes: 15 additions & 15 deletions sdk_v2/cpp/src/ep_detection/cuda_ep_manifest.cc
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ EpBundleArtifact Archive(std::string id, std::string filename, std::string sha25

EpBundleManifest WindowsX64Manifest() {
return EpBundleManifest{
.bundle_id = "cuda-ep-win-x64-cuda-12.8.4-ort-1.28.0-genai-0.15.2-20260806-182620",
.bundle_id = "cuda-ep-win-x64-cuda-12.8.4-ort-1.30.0-genai-0.16.0-20260913-071149",
.artifacts =
{
Archive(
Expand Down Expand Up @@ -66,13 +66,13 @@ EpBundleManifest WindowsX64Manifest() {
.sha256 = "49487537744256a3d4365c4792b03bf31130ad1faea0a13eafa219620941d837"},
}),
Archive(
"cuda-ep", "cuda-ep-bins-win-x64-20260806-182620.zip",
"e62938987e848a0fbb3d215dfefaed40307d2446393909927ba0345eaaf3d263", 256 * kMiB,
"cuda-ep", "cuda-ep-bins-win-x64-20260913-071149.zip",
"5d1c38eb4058b6898d78ae5e7881ac07dda8ef37156be232f0f63846a1405f85", 256 * kMiB,
{
{.relative_path = "onnxruntime-genai-cuda.dll",
.sha256 = "7894fb5efaad4a663e834f20b912b44cc383629b24ffe8bbc6382786a7326dbc"},
.sha256 = "132d28f988ee8bc1ce050e8dcdda7036ab436616d827508af8e0a4f6ec21f2da"},
{.relative_path = "onnxruntime_providers_cuda.dll",
.sha256 = "60f1aeef7ebe27f7e659cb88f597005ca5a5e75832b85dcef3eef02b9322df9a"},
.sha256 = "7d014892b64d03092c01e99ae5786c6080c5239eeee30e6fb902fd2a800fa95e"},
}),
},
.provider_relative_path = "onnxruntime_providers_cuda.dll",
Expand All @@ -81,7 +81,7 @@ EpBundleManifest WindowsX64Manifest() {

EpBundleManifest WindowsArm64Manifest() {
return EpBundleManifest{
.bundle_id = "cuda-ep-win-arm64-cuda-13.4.1-ort-1.28.0-genai-0.15.2-20260806-182803",
.bundle_id = "cuda-ep-win-arm64-cuda-13.4.1-ort-1.30.0-genai-0.16.0-20260913-071350",
.artifacts =
{
Archive(
Expand Down Expand Up @@ -115,13 +115,13 @@ EpBundleManifest WindowsArm64Manifest() {
.sha256 = "c9e0ec0e0a4e659393e15897ed1f6e5bac677e0c0fe7e12290f0386f19477b6b"},
}),
Archive(
"cuda-ep", "cuda-ep-bins-win-arm64-20260806-182803.zip",
"212e670c61b3292d4a7d98f16fc2cf61f7b080604e0c145e81c39ec81e7b3259", 96 * kMiB,
"cuda-ep", "cuda-ep-bins-win-arm64-20260913-071350.zip",
"fc8f0a01daafedc82aa57072f023b11cf801dd94ed0a449ddc8ac4698e43e519", 96 * kMiB,
{
{.relative_path = "onnxruntime-genai-cuda.dll",
.sha256 = "ab61145f4bc6284286e663586f634b973072d58ced20c497c7e5259f2ef3fc08"},
.sha256 = "d27a02b8a83d0aaac47904d1baff4f7954aa25dcc130dba539fe9aafc53eab30"},
{.relative_path = "onnxruntime_providers_cuda.dll",
.sha256 = "d92ffbd23a84f91b976baed9031de267efe1dc892d85c09d0979d25b89f5d1a0"},
.sha256 = "82a3887c64791fc7131a0f705130f02f456952915ecea71372236656c59a47b2"},
}),
},
.provider_relative_path = "onnxruntime_providers_cuda.dll",
Expand All @@ -130,17 +130,17 @@ EpBundleManifest WindowsArm64Manifest() {

EpBundleManifest LinuxX64Manifest() {
return EpBundleManifest{
.bundle_id = "cuda-ep-linux-x64-ort-1.28.0-genai-0.15.2-20260806-182830",
.bundle_id = "cuda-ep-linux-x64-ort-1.30.0-genai-0.16.0-20260913-071417",
.artifacts =
{
Archive(
"cuda-ep", "cuda-ep-linux-x64-20260806-182830.zip",
"abf347e7234d7434105efde12a2e0609fdd1d8828167b9873f4463926f1206e6", 448 * kMiB,
"cuda-ep", "cuda-ep-linux-x64-20260913-071417.zip",
"482e615e4c66b3f14a980af00e330e28fa5897a34161ec2972f75289685546be", 448 * kMiB,
{
{.relative_path = "libonnxruntime-genai-cuda.so",
.sha256 = "d5300fc4413d9e74bd8dfceb5233fca6fcfa1d5ddc247081365fdb5f143091e6"},
.sha256 = "3c95bb3b92f560718482dd3638388b6fd7374fb5ba6d9d5f4c4ca22c095e7681"},
{.relative_path = "libonnxruntime_providers_cuda.so",
.sha256 = "b88d7b7f4b2e81d3eff41663fc70f4ae9e03dee9e2301cb53dc250e5a96d7f7a"},
.sha256 = "8589ae31ed4941e72693a6a2e9dd8b8e721c888cfd9e6ffcb26da4e6e58ce720"},
}),
},
.provider_relative_path = "libonnxruntime_providers_cuda.so",
Expand Down
7 changes: 0 additions & 7 deletions sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,7 @@
#include "inferencing/generative/chat/media_input.h"
#include "inferencing/generative/chat/onnx_chat_engine.h"
#include "inferencing/generative/chat/onnx_chat_generator.h"
#if FOUNDRY_LOCAL_OGA_HAS_DYNAMIC_ENGINE
#include "inferencing/generative/chat/onnx_engine_chat_stream.h"
#endif
#include "inferencing/generative/chat/reasoning_stream_splitter.h"
#include "inferencing/generative/chat/stop_strings.h"
#include "inferencing/generative/genai_model_instance.h"
Expand Down Expand Up @@ -61,12 +59,7 @@ std::unique_ptr<ChatGenerator> CreateTextChatGenerator(const std::vector<Transcr
const ToolCallContext& tool_ctx,
bool use_full_context) {
if (model.GetGenAIConfig().GetChatBackendKind() != ChatBackendKind::kGenerator) {
#if FOUNDRY_LOCAL_OGA_HAS_DYNAMIC_ENGINE
return OnnxEngineChatStream::Create(messages, options, model, tool_ctx);
#else
FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL,
"model requires the ORT GenAI dynamic Engine API, but this build does not provide it");
#endif
}

return OnnxChatGenerator::Create(messages, options, model, tool_ctx, use_full_context);
Expand Down
6 changes: 0 additions & 6 deletions sdk_v2/cpp/src/inferencing/generative/genai_model_instance.cc
Original file line number Diff line number Diff line change
Expand Up @@ -78,18 +78,12 @@ GenAIModelInstance::GenAIModelInstance(std::string model_id,
}

if (genai_config_.GetChatBackendKind() == ChatBackendKind::kEngine) {
#if FOUNDRY_LOCAL_OGA_HAS_DYNAMIC_ENGINE
try {
chat_engine_ = std::make_unique<OnnxChatEngine>(*this);
} catch (const std::runtime_error& e) {
FL_LOG_AND_THROW(logger, FOUNDRY_LOCAL_ERROR_INTERNAL,
"failed to create chat engine for model ", model_id_, ": ", e.what());
}
#else
FL_LOG_AND_THROW(logger, FOUNDRY_LOCAL_ERROR_INTERNAL,
"model ", model_id_,
" requires the ORT GenAI dynamic Engine API, but this build does not provide it");
#endif
}
}

Expand Down
6 changes: 0 additions & 6 deletions sdk_v2/cpp/src/inferencing/generative/genai_model_instance.h
Original file line number Diff line number Diff line change
Expand Up @@ -62,11 +62,7 @@ class GenAIModelInstance {
/// Access the underlying OGA objects.
OgaModel& GetOgaModel();
Preprocessor& GetPreprocessor();
#if FOUNDRY_LOCAL_OGA_HAS_DYNAMIC_ENGINE
OnnxChatEngine* GetChatEngine() { return chat_engine_.get(); }
#else
OnnxChatEngine* GetChatEngine() { return nullptr; }
#endif

/// Get the last-activity timestamp.
std::chrono::steady_clock::time_point LastActivity() const { return last_activity_; }
Expand All @@ -93,9 +89,7 @@ class GenAIModelInstance {
ExecutionProvider ep_;
std::unique_ptr<OgaModel> oga_model_;
std::unique_ptr<Preprocessor> preprocessor_;
#if FOUNDRY_LOCAL_OGA_HAS_DYNAMIC_ENGINE
std::unique_ptr<OnnxChatEngine> chat_engine_;
#endif
TagInfo tag_info_;
std::once_flag tag_info_init_flag_;
std::chrono::steady_clock::time_point last_activity_;
Expand Down
12 changes: 3 additions & 9 deletions sdk_v2/cpp/test/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -95,17 +95,11 @@ if(FOUNDRY_LOCAL_HAS_EP_BOOTSTRAPPERS)
)
endif()

if(FOUNDRY_LOCAL_OGA_HAS_DYNAMIC_ENGINE)
target_sources(foundry_local_tests PRIVATE
internal_api/chat/dynamic_engine_chat_test.cc
)
endif()
target_sources(foundry_local_tests PRIVATE
internal_api/chat/dynamic_engine_chat_test.cc
)

target_compile_options(foundry_local_tests PRIVATE ${FOUNDRY_LOCAL_COMPILE_OPTIONS})
target_compile_definitions(
foundry_local_tests
PRIVATE FOUNDRY_LOCAL_OGA_HAS_DYNAMIC_ENGINE=$<BOOL:${FOUNDRY_LOCAL_OGA_HAS_DYNAMIC_ENGINE}>
)

target_link_libraries(foundry_local_tests
PRIVATE
Expand Down
Loading