From 4b43d9e3d76f30fa3cd5176e6ba95e77be2ee6ab Mon Sep 17 00:00:00 2001 From: Richard Loyer Date: Sun, 31 May 2026 19:19:28 +0200 Subject: [PATCH] =?UTF-8?q?chantier=20B=20STT=20#1=20:=20S1=20unification?= =?UTF-8?q?=20engine=20(mel=20partag=C3=A9=20+=20skeleton=20stt=20+=20VAD?= =?UTF-8?q?=20C++)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit S1.1 mel extractor unifié (kazeia_mel.{h,cpp}) : - Paramétrable : window (sym/periodic), compression (log/log10), post-norm (none/whisper), FFT auto radix-2 (zero-pad N_FFT pas puissance de 2) - Configs pré-faites : config_qwen3_tts_speaker() + config_whisper() - speaker_encoder.cpp migré dessus : régression damien cos=0.9997 et bit-exact (max abs diff = 0) vs avant. E2E clonage in-process intact (RTF inchangé). S1.2 audit ORT C++ : - libonnxruntime.so + headers C++ trouvés dans cache Gradle (AAR Maven onnxruntime-android-qnn:1.24.3). API_VERSION=24, QNN EP intégré. - headers/ copiés dans dist/include/onnxruntime/ (versionné, ~1 MB) - libonnxruntime.so copiée dans dist/lib/ (gitignore, 20 MB, README pour reproduire l'extraction). S1.3 skeleton stt_engine : - stt_engine.{h,cpp} : API parallèle à tts_engine.h (SttEngineLoadCfg / SttTranscribeCfg / SttTranscribeResult) - VAD RMS C++ complet (port de VadStage.kt prod : frame=1600, seuil=150, 3 frames speech, 8 frames silence) - stt_engine_load + stt_engine_transcribe : stubs err=-99 jusqu'au port ORT QNN en S2 (encoder NPU + decoder loop KV-cache + tokenizer BPE) - Binaire kazeia_stt_test (1.2 MB statique) valide mel Whisper [80,3000] + VAD sur audio FR 5s, plage de valeurs cohérente. Reste S2 : port encoder/decoder NPU via Ort::Session + QNN EP, port tokenizer BPE byte-level, override transcribe. Quand S2 done : S3 = JNI + façade Kotlin + suppression libmel_extractor.so / VadStage.kt / WhisperHybridEngine.kt. --- .gitignore | 1 + dist/build_speaker_encoder.sh | 1 + dist/build_stt_test.sh | 28 + dist/build_tts_pipeline.sh | 1 + .../onnxruntime/cpu_provider_factory.h | 19 + dist/include/onnxruntime/onnxruntime_c_api.h | 8289 +++++++++++++++++ .../include/onnxruntime/onnxruntime_cxx_api.h | 3534 +++++++ .../onnxruntime/onnxruntime_cxx_inline.h | 3896 ++++++++ .../onnxruntime/onnxruntime_env_config_keys.h | 24 + .../onnxruntime/onnxruntime_ep_c_api.h | 2112 +++++ .../onnxruntime_ep_device_ep_metadata_keys.h | 28 + .../include/onnxruntime/onnxruntime_float16.h | 537 ++ .../onnxruntime/onnxruntime_lite_custom_op.h | 1099 +++ .../onnxruntime_run_options_config_keys.h | 54 + .../onnxruntime_session_options_config_keys.h | 436 + dist/jni/kazeia_mel.cpp | 236 + dist/jni/kazeia_mel.h | 63 + dist/jni/speaker_encoder.cpp | 108 +- dist/jni/stt_engine.cpp | 167 + dist/jni/stt_engine.h | 103 + dist/jni/stt_test.cpp | 113 + dist/jni/tts_pipeline.cpp | 8 +- dist/lib/README_ORT.md | 15 + 23 files changed, 20768 insertions(+), 104 deletions(-) create mode 100644 dist/build_stt_test.sh create mode 100644 dist/include/onnxruntime/cpu_provider_factory.h create mode 100644 dist/include/onnxruntime/onnxruntime_c_api.h create mode 100644 dist/include/onnxruntime/onnxruntime_cxx_api.h create mode 100644 dist/include/onnxruntime/onnxruntime_cxx_inline.h create mode 100644 dist/include/onnxruntime/onnxruntime_env_config_keys.h create mode 100644 dist/include/onnxruntime/onnxruntime_ep_c_api.h create mode 100644 dist/include/onnxruntime/onnxruntime_ep_device_ep_metadata_keys.h create mode 100644 dist/include/onnxruntime/onnxruntime_float16.h create mode 100644 dist/include/onnxruntime/onnxruntime_lite_custom_op.h create mode 100644 dist/include/onnxruntime/onnxruntime_run_options_config_keys.h create mode 100644 dist/include/onnxruntime/onnxruntime_session_options_config_keys.h create mode 100644 dist/jni/kazeia_mel.cpp create mode 100644 dist/jni/kazeia_mel.h create mode 100644 dist/jni/stt_engine.cpp create mode 100644 dist/jni/stt_engine.h create mode 100644 dist/jni/stt_test.cpp create mode 100644 dist/lib/README_ORT.md diff --git a/.gitignore b/.gitignore index 83251a2..02288f8 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ eval/out/ dist/b-vulkan/ dist/lib-chraac/ models/ +dist/lib/libonnxruntime.so diff --git a/dist/build_speaker_encoder.sh b/dist/build_speaker_encoder.sh index 2b60b61..fec1987 100755 --- a/dist/build_speaker_encoder.sh +++ b/dist/build_speaker_encoder.sh @@ -21,6 +21,7 @@ CFLAGS=( INCS=( -I"$INC" ) LIBS=( "$JNI/speaker_encoder.cpp" + "$JNI/kazeia_mel.cpp" -L"$LIB" -lggml -lggml-base -lggml-cpu -Wl,-rpath,'$ORIGIN' -llog -ldl -lm diff --git a/dist/build_stt_test.sh b/dist/build_stt_test.sh new file mode 100644 index 0000000..c1ad8d1 --- /dev/null +++ b/dist/build_stt_test.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Build kazeia_stt_test : binaire smoke-test S1.3 (mel Whisper + VAD RMS). +# Pas encore de dépendance ORT/QNN (ça vient en S2). +set -euo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +NDK="${ANDROID_NDK_ROOT:-/opt/Kazeia/android-ndk-r27d}" +CXX="$NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android31-clang++" + +JNI="$HERE/jni" +OUT="$HERE/b-jni/kazeia_stt_test" +mkdir -p "$HERE/b-jni" + +CFLAGS=( + -std=c++17 -O3 -fPIE + -march=armv8.6-a+i8mm+bf16+dotprod+fp16 + -Wno-unused-parameter -Wno-unused-variable -Wno-sign-compare + -static-libstdc++ +) +LIBS=( + "$JNI/stt_test.cpp" + "$JNI/stt_engine.cpp" + "$JNI/kazeia_mel.cpp" + -llog -ldl -lm +) + +echo "== building $OUT ==" +"$CXX" "${CFLAGS[@]}" "${LIBS[@]}" -o "$OUT" +ls -la "$OUT" diff --git a/dist/build_tts_pipeline.sh b/dist/build_tts_pipeline.sh index 097bd09..a79d803 100755 --- a/dist/build_tts_pipeline.sh +++ b/dist/build_tts_pipeline.sh @@ -37,6 +37,7 @@ LIBS=( "$JNI/sampler.cpp" "$JNI/kazeia_text_tokenizer.cpp" "$JNI/speaker_encoder.cpp" + "$JNI/kazeia_mel.cpp" "$DECODER_A" -L"$LIB" -lllama -lggml -lggml-base -lggml-cpu -Wl,-rpath,'$ORIGIN' diff --git a/dist/include/onnxruntime/cpu_provider_factory.h b/dist/include/onnxruntime/cpu_provider_factory.h new file mode 100644 index 0000000..2926786 --- /dev/null +++ b/dist/include/onnxruntime/cpu_provider_factory.h @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "onnxruntime_c_api.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * \param use_arena zero: false. non-zero: true. + */ +ORT_EXPORT +ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_CPU, _In_ OrtSessionOptions* options, int use_arena) +ORT_ALL_ARGS_NONNULL; + +#ifdef __cplusplus +} +#endif diff --git a/dist/include/onnxruntime/onnxruntime_c_api.h b/dist/include/onnxruntime/onnxruntime_c_api.h new file mode 100644 index 0000000..6ae1539 --- /dev/null +++ b/dist/include/onnxruntime/onnxruntime_c_api.h @@ -0,0 +1,8289 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// See docs\c_cxx\README.md on generating the Doxygen documentation from this file + +/** \mainpage ONNX Runtime + * + * ONNX Runtime is a high-performance inference and training graph execution engine for deep learning models. + * + * ONNX Runtime's C, C++ APIs offer an easy to use interface to onboard and execute onnx models. + * - \subpage c_cpp_api "Core C, C++ APIs" + * - \subpage training_c_cpp_api "Training C, C++ APIs for on-device training" + * + * \page c_cpp_api Core C, C++ APIs + *

C

+ * + * ::OrtApi - Click here to go to the structure with all C API functions. + * + *

C++

+ * + * ::Ort - Click here to go to the namespace holding all of the C++ wrapper classes + * + * It is a set of header only wrapper classes around the C API. The goal is to turn the C style return value error codes into C++ exceptions, and to + * automate memory management through standard C++ RAII principles. + * + * \addtogroup Global + * ONNX Runtime C API + * @{ + */ + +#pragma once +#include +#include +#include +#include + +/** \brief The API version defined in this header + * + * This value is used by some API functions to behave as this version of the header expects. + */ +#define ORT_API_VERSION 24 + +#ifdef __cplusplus +extern "C" { +#endif + +//! @} +// SAL2 Definitions +#ifndef _MSC_VER +#define _In_ +#define _In_z_ +#define _In_opt_ +#define _In_opt_z_ +#define _Out_ +#define _Out_opt_ +#define _Outptr_ +#define _Outptr_opt_ +#define _Inout_ +#define _Inout_opt_ +#define _Frees_ptr_opt_ +#define _Ret_maybenull_ +#define _Ret_notnull_ +#define _Check_return_ +#define _Outptr_result_maybenull_ +#define _Outptr_result_maybenull_z_ +#define _In_reads_(X) +#define _In_reads_opt_ +#define _Inout_updates_(X) +#define _Out_writes_(X) +#define _Out_writes_opt_(X) +#define _Inout_updates_all_(X) +#define _Out_writes_bytes_all_(X) +#define _Out_writes_all_(X) +#define _Success_(X) +#define _Outptr_result_buffer_maybenull_(X) +#define ORT_ALL_ARGS_NONNULL __attribute__((nonnull)) +#else +#include +#define ORT_ALL_ARGS_NONNULL +#endif + +#ifdef _WIN32 +// Define ORT_DLL_IMPORT if your program is dynamically linked to Ort. +// dllexport is not used, we use a .def file. +#ifdef ORT_DLL_IMPORT +#define ORT_EXPORT __declspec(dllimport) +#else +#define ORT_EXPORT +#endif +#define ORT_API_CALL __stdcall +#define ORT_MUST_USE_RESULT +#define ORTCHAR_T wchar_t +#else +// To make symbols visible on macOS/iOS +#ifdef __APPLE__ +#define ORT_EXPORT __attribute__((visibility("default"))) +#else +#define ORT_EXPORT +#endif +#define ORT_API_CALL +#define ORT_MUST_USE_RESULT __attribute__((warn_unused_result)) +#define ORTCHAR_T char +#endif + +/// ORTCHAR_T, ORT_TSTR are reserved specifically for path handling. +/// All other strings are UTF-8 encoded, use char and std::string +#ifndef ORT_TSTR +#ifdef _WIN32 +#define ORT_TSTR(X) L##X +// When X is a macro, L##X is not defined. In this case, we need to use ORT_TSTR_ON_MACRO. +#define ORT_TSTR_ON_MACRO(X) L"" X +#else +#define ORT_TSTR(X) X +#define ORT_TSTR_ON_MACRO(X) X +#endif +#endif + +// On Windows, ORT_FILE is a wchar_t version of the __FILE__ macro. +// Otherwise, ORT_FILE is equivalent to __FILE__. +#ifndef ORT_FILE +#define ORT_FILE_INTERNAL(x) ORT_TSTR(x) +#define ORT_FILE ORT_FILE_INTERNAL(__FILE__) +#endif + +// Any pointer marked with _In_ or _Out_, cannot be NULL. + +// Windows users should use unicode paths when possible to bypass the MAX_PATH limitation +// Every pointer marked with _In_ or _Out_, cannot be NULL. Caller should ensure that. +// for ReleaseXXX(...) functions, they can accept NULL pointer. + +#ifdef __cplusplus +// For any compiler with C++11 support, MSVC 2015 and greater, or Clang version supporting noexcept. +// Such complex condition is needed because compilers set __cplusplus value differently. +#ifndef __has_feature +#define __has_feature(x) 0 +#endif +#if ((__cplusplus >= 201103L) || (_MSC_VER >= 1900) || (defined(__has_feature) && __has_feature(cxx_noexcept))) +#define NO_EXCEPTION noexcept +#else +#define NO_EXCEPTION throw() +#endif +#else +#define NO_EXCEPTION +#endif + +// __VA_ARGS__ on Windows and Linux are different +#define ORT_API(RETURN_TYPE, NAME, ...) RETURN_TYPE ORT_API_CALL NAME(__VA_ARGS__) NO_EXCEPTION + +#define ORT_API_T(RETURN_TYPE, NAME, ...) \ + RETURN_TYPE(ORT_API_CALL* NAME)(__VA_ARGS__) NO_EXCEPTION + +#define ORT_API_STATUS(NAME, ...) \ + _Success_(return == 0) _Check_return_ _Ret_maybenull_ OrtStatusPtr ORT_API_CALL NAME(__VA_ARGS__) \ + NO_EXCEPTION ORT_MUST_USE_RESULT + +// XXX: Unfortunately, SAL annotations are known to not work with function pointers +#define ORT_API2_STATUS(NAME, ...) \ + _Check_return_ _Ret_maybenull_ OrtStatusPtr(ORT_API_CALL* NAME)(__VA_ARGS__) NO_EXCEPTION ORT_MUST_USE_RESULT + +// Used in *.cc files. Almost as same as ORT_API_STATUS, except without ORT_MUST_USE_RESULT and ORT_EXPORT +#define ORT_API_STATUS_IMPL(NAME, ...) \ + _Success_(return == 0) _Check_return_ _Ret_maybenull_ OrtStatusPtr ORT_API_CALL NAME(__VA_ARGS__) NO_EXCEPTION + +#define ORT_CLASS_RELEASE(X) void(ORT_API_CALL * Release##X)(_Frees_ptr_opt_ Ort##X * input) + +#ifdef __DOXYGEN__ +#undef ORT_API_STATUS +#define ORT_API_STATUS(NAME, ...) OrtStatus* NAME(__VA_ARGS__) +#undef ORT_API2_STATUS +#define ORT_API2_STATUS(NAME, ...) OrtStatus* NAME(__VA_ARGS__) +#undef ORT_CLASS_RELEASE +#define ORT_CLASS_RELEASE(X) void Release##X(Ort##X* input) +#undef NO_EXCEPTION +#define NO_EXCEPTION +#endif +/** \addtogroup Global + * ONNX Runtime C API + * @{ + */ + +/** Copied from TensorProto::DataType + * Currently, Ort doesn't support complex64, complex128 + */ +typedef enum ONNXTensorElementDataType { + ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED, + ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT, // maps to c type float + ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8, // maps to c type uint8_t + ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8, // maps to c type int8_t + ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16, // maps to c type uint16_t + ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16, // maps to c type int16_t + ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32, // maps to c type int32_t + ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64, // maps to c type int64_t + ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING, // maps to c++ type std::string + ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL, + ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16, + ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE, // maps to c type double + ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32, // maps to c type uint32_t + ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64, // maps to c type uint64_t + ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX64, // complex with float32 real and imaginary components + ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX128, // complex with float64 real and imaginary components + ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16, // Non-IEEE floating-point format based on IEEE754 single-precision + // float 8 types were introduced in onnx 1.14, see https://onnx.ai/onnx/technical/float8.html + ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E4M3FN, // Non-IEEE floating-point format based on IEEE754 single-precision + ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E4M3FNUZ, // Non-IEEE floating-point format based on IEEE754 single-precision + ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E5M2, // Non-IEEE floating-point format based on IEEE754 single-precision + ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E5M2FNUZ, // Non-IEEE floating-point format based on IEEE754 single-precision + // Int4 types were introduced in ONNX 1.16. See https://onnx.ai/onnx/technical/int4.html + ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT4, // maps to a pair of packed uint4 values (size == 1 byte) + ONNX_TENSOR_ELEMENT_DATA_TYPE_INT4, // maps to a pair of packed int4 values (size == 1 byte) + // Float4 types were introduced in ONNX 1.18. See https://onnx.ai/onnx/technical/float4.html + ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT4E2M1, // maps to a pair of packed float4 values (size == 1 byte) + // Int2 types were introduced in ONNX 1.20. See https://onnx.ai/onnx/technical/int2.html + ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT2, // maps to 4 packed uint2 values (size == 1 byte) + ONNX_TENSOR_ELEMENT_DATA_TYPE_INT2, // maps to 4 packed int2 values (size == 1 byte) +} ONNXTensorElementDataType; + +// Synced with onnx TypeProto oneof +typedef enum ONNXType { + ONNX_TYPE_UNKNOWN, + ONNX_TYPE_TENSOR, + ONNX_TYPE_SEQUENCE, + ONNX_TYPE_MAP, + ONNX_TYPE_OPAQUE, + ONNX_TYPE_SPARSETENSOR, + ONNX_TYPE_OPTIONAL +} ONNXType; + +// These types are synced with internal +// SparseFormatFlags +typedef enum OrtSparseFormat { + ORT_SPARSE_UNDEFINED = 0, + ORT_SPARSE_COO = 0x1, + ORT_SPARSE_CSRC = 0x2, + ORT_SPARSE_BLOCK_SPARSE = 0x4 +} OrtSparseFormat; + +// Enum allows to query sparse tensor indices +enum OrtSparseIndicesFormat { + ORT_SPARSE_COO_INDICES, + ORT_SPARSE_CSR_INNER_INDICES, + ORT_SPARSE_CSR_OUTER_INDICES, + ORT_SPARSE_BLOCK_SPARSE_INDICES +}; + +/** \brief Logging severity levels + * + * In typical API usage, specifying a logging severity level specifies the minimum severity of log messages to show. + */ +typedef enum OrtLoggingLevel { + ORT_LOGGING_LEVEL_VERBOSE, ///< Verbose informational messages (least severe). + ORT_LOGGING_LEVEL_INFO, ///< Informational messages. + ORT_LOGGING_LEVEL_WARNING, ///< Warning messages. + ORT_LOGGING_LEVEL_ERROR, ///< Error messages. + ORT_LOGGING_LEVEL_FATAL, ///< Fatal error messages (most severe). +} OrtLoggingLevel; + +typedef enum OrtErrorCode { + ORT_OK, + ORT_FAIL, + ORT_INVALID_ARGUMENT, + ORT_NO_SUCHFILE, + ORT_NO_MODEL, + ORT_ENGINE_ERROR, + ORT_RUNTIME_EXCEPTION, + ORT_INVALID_PROTOBUF, + ORT_MODEL_LOADED, + ORT_NOT_IMPLEMENTED, + ORT_INVALID_GRAPH, + ORT_EP_FAIL, + ORT_MODEL_LOAD_CANCELED, + ORT_MODEL_REQUIRES_COMPILATION, + ORT_NOT_FOUND, +} OrtErrorCode; + +typedef enum OrtOpAttrType { + ORT_OP_ATTR_UNDEFINED = 0, + ORT_OP_ATTR_INT, + ORT_OP_ATTR_INTS, + ORT_OP_ATTR_FLOAT, + ORT_OP_ATTR_FLOATS, + ORT_OP_ATTR_STRING, + ORT_OP_ATTR_STRINGS, + ORT_OP_ATTR_GRAPH, + ORT_OP_ATTR_TENSOR, +} OrtOpAttrType; + +//! @} +#define ORT_RUNTIME_CLASS(X) \ + struct Ort##X; \ + typedef struct Ort##X Ort##X + +/** \addtogroup Global + * ONNX Runtime C API + * @{ + */ +// The actual types defined have an Ort prefix +ORT_RUNTIME_CLASS(Env); +ORT_RUNTIME_CLASS(Status); // nullptr for Status* indicates success +ORT_RUNTIME_CLASS(MemoryInfo); +ORT_RUNTIME_CLASS(IoBinding); +ORT_RUNTIME_CLASS(Session); // Don't call ReleaseSession from Dllmain (because session owns a thread pool) +ORT_RUNTIME_CLASS(Value); +ORT_RUNTIME_CLASS(RunOptions); +ORT_RUNTIME_CLASS(TypeInfo); +ORT_RUNTIME_CLASS(TensorTypeAndShapeInfo); +ORT_RUNTIME_CLASS(MapTypeInfo); +ORT_RUNTIME_CLASS(SequenceTypeInfo); +ORT_RUNTIME_CLASS(OptionalTypeInfo); +ORT_RUNTIME_CLASS(SessionOptions); +ORT_RUNTIME_CLASS(CustomOpDomain); +ORT_RUNTIME_CLASS(ModelMetadata); +ORT_RUNTIME_CLASS(ThreadPoolParams); +ORT_RUNTIME_CLASS(ThreadingOptions); +ORT_RUNTIME_CLASS(ArenaCfg); +ORT_RUNTIME_CLASS(PrepackedWeightsContainer); +ORT_RUNTIME_CLASS(TensorRTProviderOptionsV2); +ORT_RUNTIME_CLASS(NvTensorRtRtxProviderOptions); +ORT_RUNTIME_CLASS(CUDAProviderOptionsV2); +ORT_RUNTIME_CLASS(CANNProviderOptions); +ORT_RUNTIME_CLASS(DnnlProviderOptions); +ORT_RUNTIME_CLASS(Op); +ORT_RUNTIME_CLASS(OpAttr); +ORT_RUNTIME_CLASS(Logger); +ORT_RUNTIME_CLASS(ShapeInferContext); +ORT_RUNTIME_CLASS(LoraAdapter); +ORT_RUNTIME_CLASS(ValueInfo); +ORT_RUNTIME_CLASS(Node); +ORT_RUNTIME_CLASS(Graph); +ORT_RUNTIME_CLASS(Model); +ORT_RUNTIME_CLASS(ModelCompilationOptions); +ORT_RUNTIME_CLASS(HardwareDevice); +ORT_RUNTIME_CLASS(EpDevice); +ORT_RUNTIME_CLASS(KeyValuePairs); +ORT_RUNTIME_CLASS(SyncStream); // Opaque class to create an onnxruntime::Stream. +ORT_RUNTIME_CLASS(ExternalInitializerInfo); +ORT_RUNTIME_CLASS(ExternalResourceImporter); // Capability object for external resource import +ORT_RUNTIME_CLASS(ExternalMemoryHandle); // EP-imported view of shared external allocation +ORT_RUNTIME_CLASS(ExternalSemaphoreHandle); // EP-imported view of shared external semaphore +ORT_RUNTIME_CLASS(DeviceEpIncompatibilityDetails); +ORT_RUNTIME_CLASS(EpAssignedSubgraph); +ORT_RUNTIME_CLASS(EpAssignedNode); + +#ifdef _MSC_VER +typedef _Return_type_success_(return == 0) OrtStatus* OrtStatusPtr; +#else +typedef OrtStatus* OrtStatusPtr; +#endif + +/** \brief Memory allocation interface + * + * Structure of function pointers that defines a memory allocator. This can be created and filled in by the user for custom allocators. + * + * When an allocator is passed to any function, be sure that the allocator object is not destroyed until the last allocated object using it is freed. + */ +typedef struct OrtAllocator { + uint32_t version; ///< Must be initialized to ORT_API_VERSION + + /// Returns a pointer to an allocated block of `size` bytes + void*(ORT_API_CALL* Alloc)(struct OrtAllocator* this_, size_t size); + + /// Free a block of memory previously allocated with OrtAllocator::Alloc + void(ORT_API_CALL* Free)(struct OrtAllocator* this_, void* p); + + /// Return a pointer to an ::OrtMemoryInfo that describes this allocator + const struct OrtMemoryInfo*(ORT_API_CALL* Info)(const struct OrtAllocator* this_); + /** + * @brief Optional allocation function to use for memory allocations made during session initialization. + * Use this function if you want to separate allocations made by ORT during Run() calls from + * those made during session initialization. This allows for separate memory management strategies for these + * allocations. + * + * \return pointer to an allocated block of `size` bytes. nullptr if size was 0 or allocation failed. + * + * \since 1.18 + */ + void*(ORT_API_CALL* Reserve)(struct OrtAllocator* this_, size_t size); + + /** + * @brief Function used to get the statistics of the allocator. + * + * Return a pointer to the OrtKeyValuePairs structure that contains the statistics of the allocator. + * The user should call OrtApi::ReleaseKeyValuePairs when done. + * + * Current known keys are: + * - Limit: Bytes limit of the allocator. -1 if no limit is set. + * - InUse: Number of bytes in use. + * - TotalAllocated: The total number of allocated bytes by the allocator. + * - MaxInUse: The maximum bytes in use. + * - NumAllocs: Number of allocations. + * - NumReserves: Number of reserves. (Number of calls to Reserve() in arena-based allocators) + * - NumArenaExtensions: Number of arena extensions (Relevant only for arena based allocators) + * - NumArenaShrinkages: Number of arena shrinkages (Relevant only for arena based allocators) + * - MaxAllocSize: The max single allocation seen. + * + * The allocator is free to add other entries as appropriate. + * + * \note Implementation of this function is optional and GetStats may be set to a nullptr. + * If the OrtAllocator is wrapping an internal ORT allocator that does not implement GetStats + * the returned OrtKeyValuePairs instance will be empty. + * + * \since 1.23 + */ + ORT_API2_STATUS(GetStats, _In_ const struct OrtAllocator* this_, _Outptr_ OrtKeyValuePairs** out); + + /** \brief Allocate using a stream. + * + * If the allocator is stream aware this performs allocation using a stream. + * + * Alloc will be used if this is nullptr. + * + * \param[in] this_ OrtAllocator instance + * \param[in] size Size of the allocation in bytes. nullptr if size was 0 or allocation failed. + * \param[in] stream The stream to allocate on. + * + * \return pointer to an allocated block of `size` bytes + * + * \note Implementation of this function is optional and AllocOnStream may be set to a nullptr. + * \since 1.23 + */ + void*(ORT_API_CALL* AllocOnStream)(struct OrtAllocator* this_, size_t size, OrtSyncStream* stream); +} OrtAllocator; + +typedef void(ORT_API_CALL* OrtLoggingFunction)( + void* param, OrtLoggingLevel severity, const char* category, const char* logid, const char* code_location, + const char* message); + +/** \brief Graph optimization level + * + * Refer to https://www.onnxruntime.ai/docs/performance/graph-optimizations.html#graph-optimization-levels + * for an in-depth understanding of the Graph Optimization Levels. + */ +typedef enum GraphOptimizationLevel { + ORT_DISABLE_ALL = 0, + ORT_ENABLE_BASIC = 1, + ORT_ENABLE_EXTENDED = 2, + ORT_ENABLE_LAYOUT = 3, + ORT_ENABLE_ALL = 99 +} GraphOptimizationLevel; + +typedef enum ExecutionMode { + ORT_SEQUENTIAL = 0, + ORT_PARALLEL = 1, +} ExecutionMode; + +/** \brief Language projection identifiers + * /see OrtApi::SetLanguageProjection + */ +typedef enum OrtLanguageProjection { + ORT_PROJECTION_C = 0, + ORT_PROJECTION_CPLUSPLUS = 1, + ORT_PROJECTION_CSHARP = 2, + ORT_PROJECTION_PYTHON = 3, + ORT_PROJECTION_JAVA = 4, + ORT_PROJECTION_WINML = 5, + ORT_PROJECTION_NODEJS = 6, +} OrtLanguageProjection; + +struct OrtKernelInfo; +typedef struct OrtKernelInfo OrtKernelInfo; +struct OrtKernelContext; +typedef struct OrtKernelContext OrtKernelContext; +struct OrtCustomOp; +typedef struct OrtCustomOp OrtCustomOp; + +typedef enum OrtAllocatorType { + OrtInvalidAllocator = -1, + OrtDeviceAllocator = 0, + OrtArenaAllocator = 1, + OrtReadOnlyAllocator = 2, +} OrtAllocatorType; + +/** \brief Memory types for allocated memory, execution provider specific types should be extended in each provider. + */ +// Whenever this struct is updated, please also update the MakeKey function in onnxruntime / core / framework / execution_provider.cc +typedef enum OrtMemType { + /// Any CPU memory used by non-CPU execution provider + OrtMemTypeCPUInput = -2, + /// CPU accessible memory outputted by non-CPU execution provider, i.e. HOST_ACCESSIBLE + OrtMemTypeCPUOutput = -1, + /// CPU accessible memory allocated by non-CPU execution provider, i.e. HOST_ACCESSIBLE + OrtMemTypeCPU = OrtMemTypeCPUOutput, + /// The default allocator for execution provider + OrtMemTypeDefault = 0, +} OrtMemType; + +/** \brief This matches OrtDevice::MemoryType values */ +typedef enum OrtDeviceMemoryType { + OrtDeviceMemoryType_DEFAULT = 0, ///< Device memory + OrtDeviceMemoryType_HOST_ACCESSIBLE = 5, ///< Shared/pinned memory for transferring between CPU and the device +} OrtDeviceMemoryType; + +/** \brief This mimics OrtDevice type constants so they can be returned in the API + */ +typedef enum OrtMemoryInfoDeviceType { + OrtMemoryInfoDeviceType_CPU = 0, + OrtMemoryInfoDeviceType_GPU = 1, + OrtMemoryInfoDeviceType_FPGA = 2, + OrtMemoryInfoDeviceType_NPU = 3, +} OrtMemoryInfoDeviceType; + +typedef enum OrtHardwareDeviceType { + OrtHardwareDeviceType_CPU, + OrtHardwareDeviceType_GPU, + OrtHardwareDeviceType_NPU +} OrtHardwareDeviceType; + +/** \brief These are the default EP selection policies used by ORT when doing automatic EP selection. + */ +typedef enum OrtExecutionProviderDevicePolicy { + OrtExecutionProviderDevicePolicy_DEFAULT, + OrtExecutionProviderDevicePolicy_PREFER_CPU, + OrtExecutionProviderDevicePolicy_PREFER_NPU, + OrtExecutionProviderDevicePolicy_PREFER_GPU, + OrtExecutionProviderDevicePolicy_MAX_PERFORMANCE, + OrtExecutionProviderDevicePolicy_MAX_EFFICIENCY, + OrtExecutionProviderDevicePolicy_MIN_OVERALL_POWER, +} OrtExecutionProviderDevicePolicy; + +/** \brief Reasons why an execution provider might not be compatible with a device + */ +typedef enum OrtDeviceEpIncompatibilityReason { + OrtDeviceEpIncompatibility_NONE = 0, + OrtDeviceEpIncompatibility_DRIVER_INCOMPATIBLE = 1 << 0, + OrtDeviceEpIncompatibility_DEVICE_INCOMPATIBLE = 1 << 1, + OrtDeviceEpIncompatibility_MISSING_DEPENDENCY = 1 << 2, + OrtDeviceEpIncompatibility_UNKNOWN = 1 << 31 +} OrtDeviceEpIncompatibilityReason; + +/** \brief Delegate to allow providing custom OrtEpDevice selection logic + * + * This delegate is called by the EP selection code to allow the user to provide custom device selection logic. + * The user can use this to select OrtEpDevice instances from the list of available devices. + * + * \param ep_devices The list of available devices. + * \param num_devices The number of available devices. + * \param model_metadata The model metadata. + * \param runtime_metadata The runtime metadata. May be nullptr. + * \param selected Pre-allocated array to populate with selected OrtEpDevice pointers from ep_devices. + * \param max_selected The maximum number of devices that can be selected in the pre-allocated array. + Currently the maximum is 8. + * \param num_selected The number of selected devices. + * \param state Opaque pointer. Required to use the delegate from other languages like C# and python. + * + * \return OrtStatus* Selection status. Return nullptr on success. + * Use CreateStatus to provide error info. Use ORT_FAIL as the error code. + * ORT will release the OrtStatus* if not null. + */ +typedef OrtStatus*(ORT_API_CALL* EpSelectionDelegate)(_In_ const OrtEpDevice** ep_devices, + _In_ size_t num_devices, + _In_ const OrtKeyValuePairs* model_metadata, + _In_opt_ const OrtKeyValuePairs* runtime_metadata, + _Inout_ const OrtEpDevice** selected, + _In_ size_t max_selected, + _Out_ size_t* num_selected, + _In_ void* state); + +/** \brief Function called by ORT to write a buffer to a custom destination (e.g., file, stream, etc.). + * + * \param state Opaque pointer holding the user's state. + * \param buffer The buffer to write. + * \param buffer_num_bytes The size of the buffer in bytes. + * + * \return OrtStatus* Write status. Return nullptr on success. + * Use CreateStatus to provide error info. Use ORT_FAIL as the error code. + * ORT will release the OrtStatus* if not null. + */ +typedef OrtStatus*(ORT_API_CALL* OrtWriteBufferFunc)(_In_ void* state, + _In_ const void* buffer, + _In_ size_t buffer_num_bytes); + +/** \brief Function called by ORT to allow user to specify how an initializer should be saved, that is, either + * written to an external file or stored within the model. ORT calls this function for every initializer when + * generating a model. + * + * If the function implementation sets the `new_external_info` output parameter to NULL, ORT stores the initializer data + * within the generated model. + * + * Otherwise, if the function implementation sets `new_external_info` to a valid OrtExternalInitializerInfo instance, + * ORT assumes that this function stores the initializer data in a file. In this case, ORT configures the model's + * initializer to point to the location specified by the `new_external_info` output parameter. + * + * \param[in] state Opaque pointer holding the user's state. + * \param[in] initializer_name The initializer's name as a null-terminated string. + * \param[in] initializer_value OrtValue containing the initializer's data, type, and shape. + * \param[in] external_info If the initializer is originally stored in an external file, `external_info` contains + * the file path, file offset, and the data's byte size within the file. Otherwise, + * `external_info` is NULL if the initializer is not originally stored in a file. + * \param[out] new_external_info Output parameter set to a new OrtExternalInitializerInfo instance indicating the + * location where the function implementation stored the initializer data. + * The function implementation must use `OrtApi::CreateExternalInitializerInfo()` to + * create the instance. + * If the function implementation sets `new_external_info` to NULL, + * ORT stores the initializers within the model. + * + * \note ORT takes ownership of the `new_external_info` output parameter. + * + * \return OrtStatus* Write status. Return nullptr on success. + * Use CreateStatus to provide error info. Use ORT_FAIL as the error code. + * ORT will release the OrtStatus* if not null. + */ +typedef OrtStatus*(ORT_API_CALL* OrtGetInitializerLocationFunc)( + _In_ void* state, + _In_ const char* initializer_name, + _In_ const OrtValue* initializer_value, + _In_opt_ const OrtExternalInitializerInfo* external_info, + _Outptr_result_maybenull_ OrtExternalInitializerInfo** new_external_info); + +/** \brief Algorithm to use for cuDNN Convolution Op + */ +typedef enum OrtCudnnConvAlgoSearch { + OrtCudnnConvAlgoSearchExhaustive, // expensive exhaustive benchmarking using cudnnFindConvolutionForwardAlgorithmEx + OrtCudnnConvAlgoSearchHeuristic, // lightweight heuristic based search using cudnnGetConvolutionForwardAlgorithm_v7 + OrtCudnnConvAlgoSearchDefault, // default algorithm using CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_PRECOMP_GEMM +} OrtCudnnConvAlgoSearch; + +/** \brief CUDA Provider Options + * + * \see OrtApi::SessionOptionsAppendExecutionProvider_CUDA + */ +typedef struct OrtCUDAProviderOptions { +#ifdef __cplusplus + OrtCUDAProviderOptions() + : device_id{}, + cudnn_conv_algo_search{OrtCudnnConvAlgoSearchExhaustive}, + gpu_mem_limit{SIZE_MAX}, + arena_extend_strategy{}, + do_copy_in_default_stream{1}, + has_user_compute_stream{}, + user_compute_stream{}, + default_memory_arena_cfg{}, + tunable_op_enable{false}, + tunable_op_tuning_enable{false}, + tunable_op_max_tuning_duration_ms{} {} +#endif + + /** \brief CUDA device Id + * Defaults to 0. + */ + int device_id; + + /** \brief CUDA Convolution algorithm search configuration. + * See enum OrtCudnnConvAlgoSearch for more details. + * Defaults to OrtCudnnConvAlgoSearchExhaustive. + */ + OrtCudnnConvAlgoSearch cudnn_conv_algo_search; + + /** \brief CUDA memory limit (To use all possible memory pass in maximum size_t) + * Defaults to SIZE_MAX. + * \note If a ::OrtArenaCfg has been applied, it will override this field + */ + size_t gpu_mem_limit; + + /** \brief Strategy used to grow the memory arena + * 0 = kNextPowerOfTwo
+ * 1 = kSameAsRequested
+ * Defaults to 0. + * \note If a ::OrtArenaCfg has been applied, it will override this field + */ + int arena_extend_strategy; + + /** \brief Flag indicating if copying needs to take place on the same stream as the compute stream in the CUDA EP + * 0 = Use separate streams for copying and compute. + * 1 = Use the same stream for copying and compute. + * Defaults to 1. + * WARNING: Setting this to 0 may result in data races for some models. + * Please see issue #4829 for more details. + */ + int do_copy_in_default_stream; + + /** \brief Flag indicating if there is a user provided compute stream + * Defaults to 0. + */ + int has_user_compute_stream; + + /** \brief User provided compute stream. + * If provided, please set `has_user_compute_stream` to 1. + */ + void* user_compute_stream; + + /** \brief CUDA memory arena configuration parameters + */ + OrtArenaCfg* default_memory_arena_cfg; + + /** \brief Enable TunableOp for using. + * Set it to 1/0 to enable/disable TunableOp. Otherwise, it is disabled by default. + * This option can be overridden by environment variable ORT_CUDA_TUNABLE_OP_ENABLE. + */ + int tunable_op_enable; + + /** \brief Enable TunableOp for tuning. + * Set it to 1/0 to enable/disable TunableOp tuning. Otherwise, it is disabled by default. + * This option can be overridden by environment variable ORT_CUDA_TUNABLE_OP_TUNING_ENABLE. + */ + int tunable_op_tuning_enable; + + /** \brief Max tuning duration time limit for each instance of TunableOp. + * Defaults to 0 to disable the limit. + */ + int tunable_op_max_tuning_duration_ms; + +} OrtCUDAProviderOptions; + +/** \brief ROCM Provider Options + * + * \see OrtApi::SessionOptionsAppendExecutionProvider_ROCM + */ +typedef struct OrtROCMProviderOptions { +#ifdef __cplusplus + OrtROCMProviderOptions() + : device_id{}, + miopen_conv_exhaustive_search{0}, + gpu_mem_limit{SIZE_MAX}, + arena_extend_strategy{}, + do_copy_in_default_stream{1}, + has_user_compute_stream{}, + user_compute_stream{}, + default_memory_arena_cfg{}, + enable_hip_graph{false}, + tunable_op_enable{false}, + tunable_op_tuning_enable{false}, + tunable_op_max_tuning_duration_ms{} {} +#endif + + /** \brief ROCM device Id + * Defaults to 0. + */ + int device_id; + + /** \brief ROCM MIOpen Convolution algorithm exhaustive search option. + * Defaults to 0 (false). + */ + int miopen_conv_exhaustive_search; + + /** \brief ROCM memory limit (To use all possible memory pass in maximum size_t) + * Defaults to SIZE_MAX. + * \note If a ::OrtArenaCfg has been applied, it will override this field + */ + size_t gpu_mem_limit; + + /** \brief Strategy used to grow the memory arena + * 0 = kNextPowerOfTwo
+ * 1 = kSameAsRequested
+ * Defaults to 0. + * \note If a ::OrtArenaCfg has been applied, it will override this field + */ + int arena_extend_strategy; + + /** \brief Flag indicating if copying needs to take place on the same stream as the compute stream in the ROCM EP + * 0 = Use separate streams for copying and compute. + * 1 = Use the same stream for copying and compute. + * Defaults to 1. + * WARNING: Setting this to 0 may result in data races for some models. + * Please see issue #4829 for more details. + */ + int do_copy_in_default_stream; + + /** \brief Flag indicating if there is a user provided compute stream + * Defaults to 0. + */ + int has_user_compute_stream; + + /** \brief User provided compute stream. + * If provided, please set `has_user_compute_stream` to 1. + */ + void* user_compute_stream; + + /** \brief ROCM memory arena configuration parameters + */ + OrtArenaCfg* default_memory_arena_cfg; + + int enable_hip_graph; + + /** \brief Enable TunableOp for using. + * Set it to 1/0 to enable/disable TunableOp. Otherwise, it is disabled by default. + * This option can be overridden by environment variable ORT_ROCM_TUNABLE_OP_ENABLE. + */ + int tunable_op_enable; + + /** \brief Enable TunableOp for tuning. + * Set it to 1/0 to enable/disable TunableOp tuning. Otherwise, it is disabled by default. + * This option can be overridden by environment variable ORT_ROCM_TUNABLE_OP_TUNING_ENABLE. + */ + int tunable_op_tuning_enable; + + /** \brief Max tuning duration time limit for each instance of TunableOp. + * Defaults to 0 to disable the limit. + */ + int tunable_op_max_tuning_duration_ms; + +} OrtROCMProviderOptions; + +/** \brief TensorRT Provider Options + * + * \see OrtApi::SessionOptionsAppendExecutionProvider_TensorRT + */ +typedef struct OrtTensorRTProviderOptions { + int device_id; ///< CUDA device id (0 = default device) + int has_user_compute_stream; // indicator of user specified CUDA compute stream. + void* user_compute_stream; // user specified CUDA compute stream. + int trt_max_partition_iterations; // maximum iterations for TensorRT parser to get capability + int trt_min_subgraph_size; // minimum size of TensorRT subgraphs + size_t trt_max_workspace_size; // maximum workspace size for TensorRT. + int trt_fp16_enable; // enable TensorRT FP16 precision. Default 0 = false, nonzero = true + int trt_int8_enable; // enable TensorRT INT8 precision. Default 0 = false, nonzero = true + const char* trt_int8_calibration_table_name; // TensorRT INT8 calibration table name. + int trt_int8_use_native_calibration_table; // use native TensorRT generated calibration table. Default 0 = false, nonzero = true + int trt_dla_enable; // enable DLA. Default 0 = false, nonzero = true + int trt_dla_core; // DLA core number. Default 0 + int trt_dump_subgraphs; // dump TRT subgraph. Default 0 = false, nonzero = true + int trt_engine_cache_enable; // enable engine caching. Default 0 = false, nonzero = true + const char* trt_engine_cache_path; // specify engine cache path + int trt_engine_decryption_enable; // enable engine decryption. Default 0 = false, nonzero = true + const char* trt_engine_decryption_lib_path; // specify engine decryption library path + int trt_force_sequential_engine_build; // force building TensorRT engine sequentially. Default 0 = false, nonzero = true + // This is the legacy struct and don't add new fields here. + // For new field that can be represented by string, please add it in include/onnxruntime/core/providers/tensorrt/tensorrt_provider_options.h + // For non-string field, need to create a new separate api to handle it. +} OrtTensorRTProviderOptions; + +/** \brief MIGraphX Provider Options + * + * \see OrtApi::SessionOptionsAppendExecutionProvider_MIGraphX + */ +typedef struct OrtMIGraphXProviderOptions { + int device_id; // hip device id. + int migraphx_fp16_enable; // MIGraphX FP16 precision. Default 0 = false, nonzero = true + int migraphx_fp8_enable; // MIGraphX FP8 precision. Default 0 = false, nonzero = true + int migraphx_int8_enable; // MIGraphX INT8 precision. Default 0 = false, nonzero = true + int migraphx_use_native_calibration_table; // MIGraphx INT8 cal table. Default 0 = false, nonzero = true + const char* migraphx_int8_calibration_table_name; // MIGraphx INT8 calibration table name + int migraphx_save_compiled_model; // migraphx save compiled model. Default 0 = false, nonzero = true + const char* migraphx_save_model_path; // migraphx model path name + int migraphx_load_compiled_model; // migraphx int8 cal table. Default 0 = false, nonzero = true + const char* migraphx_load_model_path; // migraphx model path name + bool migraphx_exhaustive_tune; // MIGraphX tuned compile. Default = false, nonzero = true + + /** \brief MIGraphX memory limit (To use all possible memory pass in maximum size_t) + * Defaults to SIZE_MAX. + * \note If a ::OrtArenaCfg has been applied, it will override this field + */ + size_t migraphx_mem_limit; + + /** \brief Strategy used to grow the memory arena + * 0 = kNextPowerOfTwo
+ * 1 = kSameAsRequested
+ * Defaults to 0. + * \note If a ::OrtArenaCfg has been applied, it will override this field + */ + int migraphx_arena_extend_strategy; + + // This is the legacy struct and don't add new fields here. +} OrtMIGraphXProviderOptions; + +/** \brief OpenVINO Provider Options + * \brief This Struct is frozen since ORT 1.13.0. Its maintained part of Legacy API for compatibility. + * \brief For latest OpenVINO Provider Options update to the ProviderOptions map. + * \brief Latest OpenVINO Provider Options are listed in the + * \htmlonly + * onnxruntime document. + * \endhtmlonly + * \see OrtApi::SessionOptionsAppendExecutionProvider() + */ +typedef struct OrtOpenVINOProviderOptions { +#ifdef __cplusplus + OrtOpenVINOProviderOptions() : device_type{}, + enable_npu_fast_compile{}, + device_id{}, + num_of_threads{}, + cache_dir{}, + context{}, + enable_opencl_throttling{}, + enable_dynamic_shapes{} {} +#endif + /** \brief Device type string + * + * Valid settings are one of: "CPU_FP32", "CPU_FP16", "GPU_FP32", "GPU_FP16" + */ + const char* device_type; + unsigned char enable_npu_fast_compile; ///< 0 = disabled, nonzero = enabled + const char* device_id; + size_t num_of_threads; ///< 0 = Use default number of threads + const char* cache_dir; // path is set to empty by default + void* context; + unsigned char enable_opencl_throttling; ///< 0 = disabled, nonzero = enabled + unsigned char enable_dynamic_shapes; ///< 0 = disabled, nonzero = enabled +} OrtOpenVINOProviderOptions; + +struct OrtApi; +typedef struct OrtApi OrtApi; + +struct OrtTrainingApi; +typedef struct OrtTrainingApi OrtTrainingApi; + +struct OrtModelEditorApi; +typedef struct OrtModelEditorApi OrtModelEditorApi; + +struct OrtCompileApi; +typedef struct OrtCompileApi OrtCompileApi; + +struct OrtInteropApi; +typedef struct OrtInteropApi OrtInteropApi; + +struct OrtEpApi; +typedef struct OrtEpApi OrtEpApi; + +/** \brief The helper interface to get the right version of OrtApi + * + * Get a pointer to this structure through ::OrtGetApiBase + */ +struct OrtApiBase { + /** \brief Get a pointer to the requested version of the ::OrtApi + * + * \param[in] version Must be ::ORT_API_VERSION + * \return The ::OrtApi for the version requested, nullptr will be returned if this version is unsupported, for example when using a runtime + * older than the version created with this header file. + * + * One can call GetVersionString() to get the version of the Onnxruntime library for logging + * and error reporting purposes. + */ + const OrtApi*(ORT_API_CALL* GetApi)(uint32_t version)NO_EXCEPTION; + + /** \brief Returns a null terminated string of the version of the Onnxruntime library (eg: "1.8.1") + * + * \return UTF-8 encoded version string. Do not deallocate the returned buffer. + */ + const char*(ORT_API_CALL* GetVersionString)(void)NO_EXCEPTION; +}; + +typedef struct OrtApiBase OrtApiBase; + +/** \brief The Onnxruntime library's entry point to access the C API + * + * Call this to get the a pointer to an ::OrtApiBase + */ +ORT_EXPORT const OrtApiBase* ORT_API_CALL OrtGetApiBase(void) NO_EXCEPTION; + +/** \brief Thread work loop function + * + * Onnxruntime will provide the working loop on custom thread creation + * Argument is an onnxruntime built-in type which will be provided when thread pool calls OrtCustomCreateThreadFn + */ +typedef void (*OrtThreadWorkerFn)(void* ort_worker_fn_param); + +typedef const struct OrtCustomHandleType { + char __place_holder; +}* OrtCustomThreadHandle; + +/** \brief Ort custom thread creation function + * + * The function should return a thread handle to be used in onnxruntime thread pools + * Onnxruntime will throw exception on return value of nullptr or 0, indicating that the function failed to create a thread + */ +typedef OrtCustomThreadHandle (*OrtCustomCreateThreadFn)(void* ort_custom_thread_creation_options, OrtThreadWorkerFn ort_thread_worker_fn, void* ort_worker_fn_param); + +/** \brief Custom thread join function + * + * Onnxruntime thread pool destructor will call the function to join a custom thread. + * Argument ort_custom_thread_handle is the value returned by OrtCustomCreateThreadFn + */ +typedef void (*OrtCustomJoinThreadFn)(OrtCustomThreadHandle ort_custom_thread_handle); + +typedef OrtStatus*(ORT_API_CALL* RegisterCustomOpsFn)(OrtSessionOptions* options, const OrtApiBase* api); + +/** \brief Callback function for RunAsync + * + * \param[in] user_data User specific data that passed back to the callback + * \param[out] outputs On succeed, outputs host inference results, on error, the value will be nullptr + * \param[out] num_outputs Number of outputs, on error, the value will be zero + * \param[out] status On error, status will provide details + */ +typedef void (*RunAsyncCallbackFn)(void* user_data, OrtValue** outputs, size_t num_outputs, OrtStatusPtr status); + +/** \brief External memory handle type for importing GPU resources. + * + * \todo Add OPAQUE_WIN32 for Windows Vulkan-specific memory handles + * \todo Add POSIX file descriptor (OPAQUE_FD) for Linux Vulkan/CUDA/OpenCL interop + * \todo Add Linux DMA-BUF file descriptor for embedded GPU memory sharing + * + * \since Version 1.24. + */ +typedef enum OrtExternalMemoryHandleType { + ORT_EXTERNAL_MEMORY_HANDLE_TYPE_D3D12_RESOURCE = 0, /**< Shared HANDLE from ID3D12Device::CreateSharedHandle(resource) */ + ORT_EXTERNAL_MEMORY_HANDLE_TYPE_D3D12_HEAP = 1, /**< Shared HANDLE from ID3D12Device::CreateSharedHandle(heap) */ +} OrtExternalMemoryHandleType; + +/** \brief Descriptor for importing external memory. + * + * \note The version field must be set to ORT_API_VERSION. + * This ensures forward compatibility as fields may be added in future versions. + * + * \since Version 1.24. + */ +typedef struct OrtExternalMemoryDescriptor { + uint32_t version; /**< Must be ORT_API_VERSION */ + OrtExternalMemoryHandleType handle_type; /**< Type of the external memory handle */ + void* native_handle; /**< Platform-specific handle (e.g., Windows HANDLE) */ + size_t size_bytes; /**< Total size in bytes of the external allocation */ + size_t offset_bytes; /**< Offset in bytes into the allocation (default 0). + Base offset for the imported memory region. */ +} OrtExternalMemoryDescriptor; + +/** \brief External semaphore type for GPU synchronization. + * + * \since Version 1.24. + */ +typedef enum OrtExternalSemaphoreType { + ORT_EXTERNAL_SEMAPHORE_D3D12_FENCE = 0, /**< Shared HANDLE from ID3D12Device::CreateSharedHandle(fence) */ +} OrtExternalSemaphoreType; + +/** \brief Descriptor for importing external semaphores. + * + * \note The version field must be set to ORT_API_VERSION. + * This ensures forward compatibility as fields may be added in future versions. + * + * \since Version 1.24. + */ +typedef struct OrtExternalSemaphoreDescriptor { + uint32_t version; /**< Must be ORT_API_VERSION */ + OrtExternalSemaphoreType type; /**< Type of the external semaphore */ + void* native_handle; /**< Platform-specific handle (e.g., Windows HANDLE) */ +} OrtExternalSemaphoreDescriptor; + +/** \brief Descriptor for creating a tensor from imported external memory. + * + * \note The version field must be set to ORT_API_VERSION. + * This ensures forward compatibility as fields may be added in future versions. + * + * \since Version 1.24. + */ +typedef struct OrtExternalTensorDescriptor { + uint32_t version; /**< Must be ORT_API_VERSION */ + ONNXTensorElementDataType element_type; /**< Data type of tensor elements */ + const int64_t* shape; /**< Array of dimension sizes */ + size_t rank; /**< Number of dimensions */ + size_t offset_bytes; /**< Additional offset within imported memory (default 0). + Applied relative to OrtExternalMemoryDescriptor::offset_bytes. + Enables multiple tensors from the same imported memory handle. */ +} OrtExternalTensorDescriptor; + +/* + * Public enum for compiled model compatibility across EPs. + */ +typedef enum OrtCompiledModelCompatibility { + OrtCompiledModelCompatibility_EP_NOT_APPLICABLE = 0, + OrtCompiledModelCompatibility_EP_SUPPORTED_OPTIMAL, + OrtCompiledModelCompatibility_EP_SUPPORTED_PREFER_RECOMPILATION, + OrtCompiledModelCompatibility_EP_UNSUPPORTED, +} OrtCompiledModelCompatibility; + +/** \brief Configuration options for creating an OrtEnv. + * + * \note The version field must be set to ORT_API_VERSION. + * This ensures forward compatibility as fields may be added in future versions. + * + * \since Version 1.24. + */ +typedef struct OrtEnvCreationOptions { + uint32_t version; ///< Must be set to ORT_API_VERSION + + /** \brief The logging severity level for the environment. Must be set to a value from OrtLoggingLevel. + * + * \note Logging messages which are less severe than the `logging_severity_level` are not emitted. + * + * \note Serves as the default logging severity level for session creation and runs. + * Use OrtApi::SetSessionLogSeverityLevel to set a logging severity level for the creation of specific session. + * Use OrtApi::RunOptionsSetRunLogSeverityLevel to set a logging severity level for a specific session run. + * + * \since Version 1.24. + */ + int32_t logging_severity_level; + + /** \brief The log identifier. Must be set to a valid UTF-8 null-terminated string. + * + * \note This string identifier is copied by ORT. + * + * \since Version 1.24. + */ + const char* log_id; + + /** \brief Optional custom logging function. May be set to NULL. + * + * \note The OrtEnvCreationOptions::custom_logging_param is provided as the first argument to this logging function. + * This allows passing custom state into the logging function. + * + * \note This function is only called when a message's severity meets or exceeds the set logging severity level. + * + * \since Version 1.24. + */ + OrtLoggingFunction custom_logging_function; + + /** \brief Optional state to pass as the first argument to OrtEnvCreationOptions::custom_logger_function. + * May be set to NULL. + * + * \since Version 1.24. + */ + void* custom_logging_param; + + /** \brief Optional threading options for creating an environment with global thread pools shared across sessions. + * May be set to NULL. + * + * \note The OrtThreadingOptions instance is copied by ORT. + * + * \note Use OrtApi::CreateThreadingOptions() to create an instance of OrtThreadingOptions. + * + * \note Use this in conjunction with OrtApi::DisablePerSessionThreads or else the session will use its own + * thread pools. + * + * \since Version 1.24. + */ + const OrtThreadingOptions* threading_options; + + /** \brief Optional environment configuration entries represented as string key-value pairs. May be set to NULL. + * + * \note The OrtKeyValuePairs instance is copied by ORT. + * + * \note Refer to onnxruntime_env_config_keys.h for common config entry keys and their supported values. + * + * \note An application provides environment-level configuration options for execution provider libraries by + * using keys with the prefix 'ep_factory.\\.'. Ex: the key 'ep_factory.my_ep.some_ep_key' represents + * a key named 'some_ep_key' that is meant to be consumed by an execution provider named 'my_ep'. Refer to + * the specific execution provider's documentation for valid keys and values. + * + * \note An application may separately set session-level configuration options for execution providers via other APIs + * such as SessionOptionsAppendExecutionProvider_V2, which store configuration entries within OrtSessionOptions. + * If an environment-level configuration conflicts with a session-level configuration, then + * precedence is determined by the execution provider library itself. + * + * \since Version 1.24. + */ + const OrtKeyValuePairs* config_entries; + + // + // End of fields available in ORT 1.24 + // + +} OrtEnvCreationOptions; + +/** \brief The C API + * + * All C API functions are defined inside this structure as pointers to functions. + * Call OrtApiBase::GetApi to get a pointer to it + * + * \nosubgrouping + */ +struct OrtApi { + /// \name OrtStatus + /// @{ + + /** + * \brief Create an OrtStatus from a null terminated string + * + * \param[in] code + * \param[in] msg A null-terminated string. Its contents will be copied. + * \return A new OrtStatus object, must be destroyed with OrtApi::ReleaseStatus + */ + OrtStatus*(ORT_API_CALL* CreateStatus)(OrtErrorCode code, _In_ const char* msg)NO_EXCEPTION ORT_ALL_ARGS_NONNULL; + + /** \brief Get OrtErrorCode from OrtStatus + * + * \param[in] status + * \return OrtErrorCode that \p status was created with + */ + OrtErrorCode(ORT_API_CALL* GetErrorCode)(_In_ const OrtStatus* status) NO_EXCEPTION ORT_ALL_ARGS_NONNULL; + + /** \brief Get error string from OrtStatus + * + * \param[in] status + * \return The error message inside the `status`. Do not free the returned value. + */ + const char*(ORT_API_CALL* GetErrorMessage)(_In_ const OrtStatus* status)NO_EXCEPTION ORT_ALL_ARGS_NONNULL; + + /// @} + /// \name OrtEnv + /// @{ + + /** \brief Create an OrtEnv + * + * \note Invoking this function will return the same instance of the environment as that returned by a previous call + * to another env creation function; all arguments to this function will be ignored. + * \param[in] log_severity_level The log severity level. + * \param[in] logid The log identifier. + * \param[out] out Returned newly created OrtEnv. Must be freed with OrtApi::ReleaseEnv + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CreateEnv, OrtLoggingLevel log_severity_level, _In_ const char* logid, _Outptr_ OrtEnv** out); + + /** \brief Create an OrtEnv + * + * \note Invoking this function will return the same instance of the environment as that returned by a previous call + * to another env creation function; all arguments to this function will be ignored. If you want to provide your + * own logging function, consider setting it using the SetUserLoggingFunction API instead. + * \param[in] logging_function A pointer to a logging function. + * \param[in] logger_param A pointer to arbitrary data passed as the ::OrtLoggingFunction `param` parameter to + * `logging_function`. This parameter is optional. + * \param[in] log_severity_level The log severity level. + * \param[in] logid The log identifier. + * \param[out] out Returned newly created OrtEnv. Must be freed with OrtApi::ReleaseEnv + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CreateEnvWithCustomLogger, _In_ OrtLoggingFunction logging_function, _In_opt_ void* logger_param, + _In_ OrtLoggingLevel log_severity_level, _In_ const char* logid, _Outptr_ OrtEnv** out); + + /** \brief Enable Telemetry + * + * \note Telemetry events are on by default since they are lightweight + * \param[in] env + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(EnableTelemetryEvents, _In_ const OrtEnv* env); + /** \brief Disable Telemetry + * + * \see OrtApi::EnableTelemetryEvents + * \param[in] env + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(DisableTelemetryEvents, _In_ const OrtEnv* env); + + /// @} + /// \name OrtSession + /// @{ + + /** \brief Create an OrtSession from a model file + * + * \param[in] env + * \param[in] model_path + * \param[in] options + * \param[out] out Returned newly created OrtSession. Must be freed with OrtApi::ReleaseSession + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + // TODO: document the path separator convention? '/' vs '\' + // TODO: should specify the access characteristics of model_path. Is this read only during the + // execution of CreateSession, or does the OrtSession retain a handle to the file/directory + // and continue to access throughout the OrtSession lifetime? + // What sort of access is needed to model_path : read or read/write? + ORT_API2_STATUS(CreateSession, _In_ const OrtEnv* env, _In_ const ORTCHAR_T* model_path, + _In_ const OrtSessionOptions* options, _Outptr_ OrtSession** out); + + /** \brief Create an OrtSession from memory + * + * \param[in] env + * \param[in] model_data + * \param[in] model_data_length + * \param[in] options + * \param[out] out Returned newly created OrtSession. Must be freed with OrtApi::ReleaseSession + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CreateSessionFromArray, _In_ const OrtEnv* env, + _In_ const void* model_data, size_t model_data_length, + _In_ const OrtSessionOptions* options, _Outptr_ OrtSession** out); + + /** \brief Run the model in an ::OrtSession + * + * Will not return until the model run has completed. Multiple threads might be used to run the model based on + * the options in the ::OrtSession and settings used when creating the ::OrtEnv + * + * \param[in] session + * \param[in] run_options If nullptr, will use a default ::OrtRunOptions + * \param[in] input_names Array of null terminated UTF8 encoded strings of the input names + * \param[in] inputs Array of ::OrtValue%s of the input values + * \param[in] input_len Number of elements in the input_names and inputs arrays + * \param[in] output_names Array of null terminated UTF8 encoded strings of the output names + * \param[in] output_names_len Number of elements in the output_names and outputs array + * \param[out] outputs Array of ::OrtValue%s that the outputs are stored in. This can also be + * an array of nullptr values, in this case ::OrtValue objects will be allocated and pointers + * to them will be set into the `outputs` array. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(Run, _Inout_ OrtSession* session, _In_opt_ const OrtRunOptions* run_options, + _In_reads_(input_len) const char* const* input_names, + _In_reads_(input_len) const OrtValue* const* inputs, size_t input_len, + _In_reads_(output_names_len) const char* const* output_names, size_t output_names_len, + _Inout_updates_all_(output_names_len) OrtValue** outputs); + + /// @} + /// \name OrtSessionOptions + /// @{ + + /** \brief Create an ::OrtSessionOptions object + * + * To use additional providers, you must build ORT with the extra providers enabled. Then call one of these + * functions to enable them in the session:
+ * OrtSessionOptionsAppendExecutionProvider_CPU
+ * OrtSessionOptionsAppendExecutionProvider_CUDA
+ * OrtSessionOptionsAppendExecutionProvider_(remaining providers...)
+ * The order they are called indicates the preference order as well. In other words call this method + * on your most preferred execution provider first followed by the less preferred ones. + * If none are called Ort will use its internal CPU execution provider. + * + * \param[out] options The newly created OrtSessionOptions. Must be freed with OrtApi::ReleaseSessionOptions + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CreateSessionOptions, _Outptr_ OrtSessionOptions** options); + + /** \brief Set filepath to save optimized model after graph level transformations + * + * \param[in] options + * \param[in] optimized_model_filepath + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SetOptimizedModelFilePath, _Inout_ OrtSessionOptions* options, + _In_ const ORTCHAR_T* optimized_model_filepath); + + /** \brief Create a copy of an existing ::OrtSessionOptions + * + * \param[in] in_options OrtSessionOptions to copy + * \param[out] out_options Returned newly created ::OrtSessionOptions. Must be freed with OrtApi::ReleaseSessionOptions + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CloneSessionOptions, _In_ const OrtSessionOptions* in_options, + _Outptr_ OrtSessionOptions** out_options); + + /** \brief Set execution mode + * + * Controls whether you want to execute operators in your graph sequentially or in parallel. Usually when the model + * has many branches, setting this option to ExecutionMode.ORT_PARALLEL will give you better performance. + * See [docs/ONNX_Runtime_Perf_Tuning.md] for more details. + * + * \param[in] options + * \param[in] execution_mode + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SetSessionExecutionMode, _Inout_ OrtSessionOptions* options, ExecutionMode execution_mode); + + /** \brief Enable profiling for a session + * + * \param[in] options + * \param[in] profile_file_prefix + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(EnableProfiling, _Inout_ OrtSessionOptions* options, _In_ const ORTCHAR_T* profile_file_prefix); + + /** \brief Disable profiling for a session + * + * \param[in] options + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(DisableProfiling, _Inout_ OrtSessionOptions* options); + + /** \brief Enable the memory pattern optimization + * + * The idea is if the input shapes are the same, we could trace the internal memory allocation + * and generate a memory pattern for future request. So next time we could just do one allocation + * with a big chunk for all the internal memory allocation. + * \note Memory pattern optimization is only available when Sequential Execution mode is enabled (see OrtApi::SetSessionExecutionMode) + * + * \see OrtApi::DisableMemPattern + * + * \param[in] options + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(EnableMemPattern, _Inout_ OrtSessionOptions* options); + + /** \brief Disable the memory pattern optimization + * + * \see OrtApi::EnableMemPattern + * + * \param[in] options + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(DisableMemPattern, _Inout_ OrtSessionOptions* options); + + /** \brief Enable the memory arena on CPU + * + * Arena may pre-allocate memory for future usage. + * + * \param[in] options + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(EnableCpuMemArena, _Inout_ OrtSessionOptions* options); + + /** \brief Disable the memory arena on CPU + * + * \param[in] options + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(DisableCpuMemArena, _Inout_ OrtSessionOptions* options); + + /** \brief Set session log id + * + * \param[in] options + * \param[in] logid The log identifier. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SetSessionLogId, _Inout_ OrtSessionOptions* options, const char* logid); + + /** \brief Set session log verbosity level + * + * Applies to session load, initialization, etc + * + * \param[in] options + * \param[in] session_log_verbosity_level \snippet{doc} snippets.dox Log Verbosity Level + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SetSessionLogVerbosityLevel, _Inout_ OrtSessionOptions* options, int session_log_verbosity_level); + + /** \brief Set session log severity level + * + * \param[in] options + * \param[in] session_log_severity_level The log severity level (refer to ::OrtLoggingLevel for possible values). + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SetSessionLogSeverityLevel, _Inout_ OrtSessionOptions* options, int session_log_severity_level); + + /** \brief Set the optimization level to apply when loading a graph + * + * Please see https://onnxruntime.ai/docs/performance/model-optimizations/graph-optimizations.html for an in-depth explanation + * \param[in,out] options The session options object + * \param[in] graph_optimization_level The optimization level + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SetSessionGraphOptimizationLevel, _Inout_ OrtSessionOptions* options, + GraphOptimizationLevel graph_optimization_level); + + /** \brief Sets the number of threads used to parallelize the execution within nodes + * + * When running a single node operation, ex. add, this sets the maximum number of threads to use. + * + * \note If built with OpenMP, this has no effect on the number of threads used. In this case + * use the OpenMP env variables to configure the number of intra op num threads. + * + * \param[in] options + * \param[in] intra_op_num_threads Number of threads to use
+ * A value of 0 will use the default number of threads
+ * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SetIntraOpNumThreads, _Inout_ OrtSessionOptions* options, int intra_op_num_threads); + + /** \brief Sets the number of threads used to parallelize the execution of the graph + * + * If nodes can be run in parallel, this sets the maximum number of threads to use to run them in parallel. + * + * \note If sequential execution is enabled this value is ignored, it acts as if it was set to 1. + * + * \param[in] options + * \param[in] inter_op_num_threads Number of threads to use
+ * A value of 0 will use the default number of threads
+ * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SetInterOpNumThreads, _Inout_ OrtSessionOptions* options, int inter_op_num_threads); + + /// @} + /// \name OrtCustomOpDomain + /// @{ + + /** \brief Create a custom op domain + * + * \param[in] domain + * \param[out] out Newly created domain. Must be freed with OrtApi::ReleaseCustomOpDomain + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CreateCustomOpDomain, _In_ const char* domain, _Outptr_ OrtCustomOpDomain** out); + + /** \brief Add a custom op to a custom op domain + * + * \note The OrtCustomOp* pointer must remain valid until the ::OrtCustomOpDomain using it is released + * + * \param[in] custom_op_domain + * \param[in] op + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CustomOpDomain_Add, _Inout_ OrtCustomOpDomain* custom_op_domain, _In_ const OrtCustomOp* op); + + /// @} + /// \name OrtSessionOptions + /// @{ + + /** \brief Add custom op domain to a session options + * + * \note The OrtCustomOpDomain* must not be deleted until all sessions using it are released + * + * \param[in] options + * \param[in] custom_op_domain + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(AddCustomOpDomain, _Inout_ OrtSessionOptions* options, _In_ OrtCustomOpDomain* custom_op_domain); + + /** \deprecated Use OrtApi::RegisterCustomOpsLibrary_V2. + * + * Registers custom ops from a shared library. + * + * Loads a shared library (dll on windows, so on linux, etc) named 'library_path' and looks for this entry point: + * OrtStatus* RegisterCustomOps(OrtSessionOptions * options, const OrtApiBase* api); + * It then passes in the provided session options to this function along with the api base. + * The handle to the loaded library is returned in library_handle. It can be freed by the caller after all sessions using the passed in + * session options are destroyed, or if an error occurs and it is non null. + * + * \param[in] options + * \param[in] library_path + * \param[out] library_handle OS specific handle to the loaded library (Use FreeLibrary on Windows, dlclose on Linux, etc.. to unload) + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(RegisterCustomOpsLibrary, _Inout_ OrtSessionOptions* options, _In_ const char* library_path, _Outptr_ void** library_handle); + + /// @} + /// \name OrtSession + /// @{ + + /** \brief Get input count for a session + * + * This number must also match the number of inputs passed to OrtApi::Run + * + * \see OrtApi::SessionGetInputTypeInfo, OrtApi::SessionGetInputName, OrtApi::Session + * + * \param[in] session + * \param[out] out Number of inputs + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SessionGetInputCount, _In_ const OrtSession* session, _Out_ size_t* out); + + /** \brief Get output count for a session + * + * This number must also match the number of outputs returned by OrtApi::Run + * + * \see OrtApi::SessionGetOutputTypeInfo, OrtApi::SessionGetOutputName, OrtApi::Session + * + * \param[in] session + * \param[out] out Number of outputs + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SessionGetOutputCount, _In_ const OrtSession* session, _Out_ size_t* out); + + /** \brief Get overridable initializer count + * + * \see OrtApi::SessionGetOverridableInitializerTypeInfo, OrtApi::SessionGetOverridableInitializerName + * + * \param[in] session + * \param[in] out + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SessionGetOverridableInitializerCount, _In_ const OrtSession* session, _Out_ size_t* out); + + /** \brief Get input type information + * + * \param[in] session + * \param[in] index Must be between 0 (inclusive) and what OrtApi::SessionGetInputCount returns (exclusive) + * \param[out] type_info Must be freed with OrtApi::ReleaseTypeInfo + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SessionGetInputTypeInfo, _In_ const OrtSession* session, size_t index, _Outptr_ OrtTypeInfo** type_info); + + /** \brief Get output type information + * + * \param[in] session + * \param[in] index Must be between 0 (inclusive) and what OrtApi::SessionGetOutputCount returns (exclusive) + * \param[out] type_info Must be freed with OrtApi::ReleaseTypeInfo + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SessionGetOutputTypeInfo, _In_ const OrtSession* session, size_t index, _Outptr_ OrtTypeInfo** type_info); + + /** \brief Get overridable initializer type information + * + * \param[in] session + * \param[in] index Must be between 0 (inclusive) and what OrtApi::SessionGetOverridableInitializerCount returns (exclusive) + * \param[out] type_info Must be freed with OrtApi::ReleaseTypeInfo + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SessionGetOverridableInitializerTypeInfo, _In_ const OrtSession* session, size_t index, _Outptr_ OrtTypeInfo** type_info); + + /** \brief Get input name + * + * \param[in] session + * \param[in] index Must be between 0 (inclusive) and what OrtApi::SessionGetInputCount returns (exclusive) + * \param[in] allocator + * \param[out] value Set to a null terminated UTF-8 encoded string allocated using `allocator`. Must be freed using `allocator`. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SessionGetInputName, _In_ const OrtSession* session, size_t index, _Inout_ OrtAllocator* allocator, _Outptr_ char** value); + + /** \brief Get output name + * + * \param[in] session + * \param[in] index Must be between 0 (inclusive) and what OrtApi::SessionGetOutputCount returns (exclusive) + * \param[in] allocator + * \param[out] value Set to a null terminated UTF-8 encoded string allocated using `allocator`. Must be freed using `allocator`. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SessionGetOutputName, _In_ const OrtSession* session, size_t index, _Inout_ OrtAllocator* allocator, _Outptr_ char** value); + + /** \brief Get overridable initializer name + * + * \param[in] session + * \param[in] index Must be between 0 (inclusive) and what OrtApi::SessionGetOverridableInitializerCount returns (exclusive) + * \param[in] allocator + * \param[out] value Set to a null terminated UTF-8 encoded string allocated using `allocator`. Must be freed using `allocator`. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SessionGetOverridableInitializerName, _In_ const OrtSession* session, size_t index, + _Inout_ OrtAllocator* allocator, _Outptr_ char** value); + + /// @} + /// \name OrtRunOptions + /// @{ + + /** \brief Create an OrtRunOptions + * + * \param[out] out Returned newly created ::OrtRunOptions. Must be freed with OrtApi::ReleaseRunOptions + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CreateRunOptions, _Outptr_ OrtRunOptions** out); + + /** \brief Set per-run log verbosity level + * + * \see OrtApi::RunOptionsGetRunLogVerbosityLevel + * + * \param[in] options + * \param[in] log_verbosity_level \snippet{doc} snippets.dox Log Verbosity Level + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(RunOptionsSetRunLogVerbosityLevel, _Inout_ OrtRunOptions* options, int log_verbosity_level); + + /** \brief Set per-run log severity level + * + * \see OrtApi::RunOptionsGetRunLogSeverityLevel + * + * \param[in] options + * \param[in] log_severity_level The log severity level (refer to ::OrtLoggingLevel for possible values). + */ + ORT_API2_STATUS(RunOptionsSetRunLogSeverityLevel, _Inout_ OrtRunOptions* options, int log_severity_level); + + /** \brief Set per-run tag + * + * This is used in a per-run log identifier. + * + * \see OrtApi::RunOptionsGetRunTag + * + * \param[in] options + * \param[in] run_tag The run tag. + */ + ORT_API2_STATUS(RunOptionsSetRunTag, _Inout_ OrtRunOptions* options, _In_ const char* run_tag); + + /** \brief Get per-run log verbosity level + * + * \see OrtApi::RunOptionsSetRunLogVerbosityLevel + * + * \param[in] options + * \param[out] log_verbosity_level \snippet{doc} snippets.dox Log Verbosity Level + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(RunOptionsGetRunLogVerbosityLevel, _In_ const OrtRunOptions* options, + _Out_ int* log_verbosity_level); + + /** \brief Get per-run log severity level + * + * \see OrtApi::RunOptionsSetRunLogSeverityLevel + * + * \param[in] options + * \param[out] log_severity_level The log severity level (refer to ::OrtLoggingLevel for possible values). + */ + ORT_API2_STATUS(RunOptionsGetRunLogSeverityLevel, _In_ const OrtRunOptions* options, _Out_ int* log_severity_level); + + /** \brief Get per-run tag + * + * This is used in a per-run log identifier. + * + * \see OrtApi::RunOptionsSetRunTag + * + * \param[in] options + * \param[out] run_tag The run tag. + * Do not free this value, it is owned by `options`. It will be invalidated if the run tag + * changes (i.e., with OrtApi::RunOptionsSetRunTag) or `options` is freed. + */ + ORT_API2_STATUS(RunOptionsGetRunTag, _In_ const OrtRunOptions* options, _Out_ const char** run_tag); + + /** \brief Set terminate flag + * + * If a currently executing session needs to be force terminated, this can be called from another thread to force it to fail with an error. + * + * \param[in] options + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(RunOptionsSetTerminate, _Inout_ OrtRunOptions* options); + + /** \brief Clears the terminate flag + * + * Used so the OrtRunOptions instance can be used in a new OrtApi::Run call without it instantly terminating + * + * \param[in] options + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(RunOptionsUnsetTerminate, _Inout_ OrtRunOptions* options); + + /// @} + /// \name OrtValue + /// @{ + + /** \brief Create a tensor + * + * Create a tensor using a supplied ::OrtAllocator + * + * \param[in] allocator + * \param[in] shape Pointer to the tensor shape dimensions. + * \param[in] shape_len The number of tensor shape dimensions. + * \param[in] type + * \param[out] out Returns newly created ::OrtValue. Must be freed with OrtApi::ReleaseValue + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CreateTensorAsOrtValue, _Inout_ OrtAllocator* allocator, _In_ const int64_t* shape, size_t shape_len, + ONNXTensorElementDataType type, _Outptr_ OrtValue** out); + + /** \brief Create a tensor backed by a user supplied buffer + * + * Create a tensor with user's buffer. You can fill the buffer either before calling this function or after. + * p_data is owned by caller. ReleaseValue won't release p_data. + * + * If you wish to transfer ownership of p_data to ORT use CreateTensorWithDataAndDeleterAsOrtValue. + * + * \param[in] info Memory description of where the p_data buffer resides (CPU vs GPU etc). + * \param[in] p_data Pointer to the data buffer. + * \param[in] p_data_len The number of bytes in the data buffer. + * \param[in] shape Pointer to the tensor shape dimensions. + * \param[in] shape_len The number of tensor shape dimensions. + * \param[in] type The data type. + * \param[out] out Returns newly created ::OrtValue. Must be freed with OrtApi::ReleaseValue + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CreateTensorWithDataAsOrtValue, _In_ const OrtMemoryInfo* info, _Inout_ void* p_data, + size_t p_data_len, _In_ const int64_t* shape, size_t shape_len, ONNXTensorElementDataType type, + _Outptr_ OrtValue** out); + + /** \brief Return if an ::OrtValue is a tensor type + * + * \param[in] value A tensor type (string tensors are not supported) + * \param[out] out Set to 1 iff ::OrtValue is a tensor, 0 otherwise + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(IsTensor, _In_ const OrtValue* value, _Out_ int* out); + + /** \brief Get a pointer to the raw data inside a tensor + * + * Used to read/write/modify the internal tensor data directly. + * \note The returned pointer is valid until the \p value is destroyed. + * + * \param[in] value A tensor type (string tensors are not supported) + * \param[out] out Filled in with a pointer to the internal storage + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetTensorMutableData, _In_ OrtValue* value, _Outptr_ void** out); + + /** \brief Set all strings at once in a string tensor + * + * \param[in,out] value A tensor of type ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING + * \param[in] s An array of strings. Each string in this array must be null terminated. + * \param[in] s_len Count of strings in s (Must match the size of \p value's tensor shape) + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(FillStringTensor, _Inout_ OrtValue* value, _In_ const char* const* s, size_t s_len); + + /** \brief Get total byte length for all strings in a string tensor + * + * Typically used with OrtApi::GetStringTensorContent + * + * \param[in] value A tensor of type ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING + * \param[out] len Total byte length of all strings (does not include trailing nulls) + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetStringTensorDataLength, _In_ const OrtValue* value, _Out_ size_t* len); + + /** \brief Get all strings from a string tensor + * + * An example of the results:
+ * Given \p value is a string tensor with the strings { "This" "is" "a" "test" }
+ * \p s must have a size of 11 bytes
+ * \p offsets must have 4 elements
+ * After the call, these values will be filled in:
+ * \p s will contain "Thisisatest"
+ * \p offsets will contain { 0, 4, 6, 7 }
+ * The length of the last string is just s_len - offsets[last] + * + * \param[in] value A tensor of type ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING + * \param[in] s Buffer to sequentially write all tensor strings to. Each string is NOT null-terminated. + * \param[in] s_len Number of bytes of buffer pointed to by \p s (Get it from OrtApi::GetStringTensorDataLength) + * \param[out] offsets Array of start offsets into the strings written to \p s + * \param[in] offsets_len Number of elements in offsets + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetStringTensorContent, _In_ const OrtValue* value, _Out_writes_bytes_all_(s_len) void* s, + size_t s_len, _Out_writes_all_(offsets_len) size_t* offsets, size_t offsets_len); + + /// @} + /// \name OrtTypeInfo + /// @{ + + /** \brief Get ::OrtTensorTypeAndShapeInfo from an ::OrtTypeInfo + * + * \param[in] type_info + * \param[out] out Do not free this value, it will be valid until type_info is freed. + * If type_info does not represent tensor, this value will be set to nullptr. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CastTypeInfoToTensorInfo, _In_ const OrtTypeInfo* type_info, + _Outptr_result_maybenull_ const OrtTensorTypeAndShapeInfo** out); + + /** \brief Get ::ONNXType from ::OrtTypeInfo + * + * \param[in] type_info + * \param[out] out + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetOnnxTypeFromTypeInfo, _In_ const OrtTypeInfo* type_info, _Out_ enum ONNXType* out); + + /// @} + /// \name OrtTensorTypeAndShapeInfo + /// @{ + + /** \brief Create an ::OrtTensorTypeAndShapeInfo object + * + * \param[out] out Returns newly created ::OrtTensorTypeAndShapeInfo. Must be freed with OrtApi::ReleaseTensorTypeAndShapeInfo + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CreateTensorTypeAndShapeInfo, _Outptr_ OrtTensorTypeAndShapeInfo** out); + + /** \brief Set element type in ::OrtTensorTypeAndShapeInfo + * + * \param[in] info + * \param[in] type + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SetTensorElementType, _Inout_ OrtTensorTypeAndShapeInfo* info, enum ONNXTensorElementDataType type); + + /** \brief Set shape information in ::OrtTensorTypeAndShapeInfo + * + * \param[in] info + * \param[in] dim_values Array with `dim_count` elements. Can contain negative values. + * \param[in] dim_count Number of elements in `dim_values` + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SetDimensions, OrtTensorTypeAndShapeInfo* info, _In_ const int64_t* dim_values, size_t dim_count); + + /** \brief Get element type in ::OrtTensorTypeAndShapeInfo + * + * \see OrtApi::SetTensorElementType + * + * \param[in] info + * \param[out] out + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetTensorElementType, _In_ const OrtTensorTypeAndShapeInfo* info, + _Out_ enum ONNXTensorElementDataType* out); + + /** \brief Get dimension count in ::OrtTensorTypeAndShapeInfo + * + * \see OrtApi::GetDimensions + * + * \param[in] info + * \param[out] out + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetDimensionsCount, _In_ const OrtTensorTypeAndShapeInfo* info, _Out_ size_t* out); + + /** \brief Get dimensions in ::OrtTensorTypeAndShapeInfo + * + * \param[in] info + * \param[out] dim_values Array with `dim_values_length` elements. On return, filled with the dimensions stored in the ::OrtTensorTypeAndShapeInfo + * \param[in] dim_values_length Number of elements in `dim_values`. Use OrtApi::GetDimensionsCount to get this value + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetDimensions, _In_ const OrtTensorTypeAndShapeInfo* info, _Out_ int64_t* dim_values, + size_t dim_values_length); + + /** \brief Get symbolic dimension names in ::OrtTensorTypeAndShapeInfo + * + * \param[in] info + * \param[in] dim_params Array with `dim_params_length` elements. On return filled with pointers to null terminated strings of the dimension names + * \param[in] dim_params_length Number of elements in `dim_params`. Use OrtApi::GetDimensionsCount to get this value + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetSymbolicDimensions, _In_ const OrtTensorTypeAndShapeInfo* info, + _Out_writes_all_(dim_params_length) const char* dim_params[], size_t dim_params_length); + + /** \brief Get total number of elements in a tensor shape from an ::OrtTensorTypeAndShapeInfo + * + * Return the number of elements specified by the tensor shape (all dimensions multiplied by each other). + * For 0 dimensions, 1 is returned. If any dimension is less than 0, the result is always -1. + * + * Examples:
+ * [] = 1
+ * [1,3,4] = 12
+ * [2,0,4] = 0
+ * [-1,3,4] = -1
+ * + * \param[in] info + * \param[out] out Number of elements + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetTensorShapeElementCount, _In_ const OrtTensorTypeAndShapeInfo* info, _Out_ size_t* out); + + /// @} + /// \name OrtValue + /// @{ + + /** \brief Get type and shape information from a tensor ::OrtValue + * + * \param[in] value Must be a tensor (not a map/sequence/etc) or will return failure + * \param[out] out Newly created ::OrtTensorTypeAndShapeInfo. Must be freed with OrtApi::ReleaseTensorTypeAndShapeInfo + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetTensorTypeAndShape, _In_ const OrtValue* value, _Outptr_ OrtTensorTypeAndShapeInfo** out); + + /** \brief Get type information of an OrtValue + * + * \param[in] value + * \param[out] out Newly created ::OrtTypeInfo. Must be freed with OrtApi::ReleaseTypeInfo + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetTypeInfo, _In_ const OrtValue* value, _Outptr_result_maybenull_ OrtTypeInfo** out); + + /** \brief Get ONNXType of an ::OrtValue + * + * \param[in] value + * \param[out] out + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetValueType, _In_ const OrtValue* value, _Out_ enum ONNXType* out); + + /// @} + /// \name OrtMemoryInfo + /// @{ + + /** \brief Create an ::OrtMemoryInfo + * + * \param[in] name + * \param[in] type + * \param[in] id + * \param[in] mem_type + * \param[out] out Newly created ::OrtMemoryInfo. Must be freed with OrtAPi::ReleaseMemoryInfo + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CreateMemoryInfo, _In_ const char* name, enum OrtAllocatorType type, int id, + enum OrtMemType mem_type, _Outptr_ OrtMemoryInfo** out); + + /** \brief Create an ::OrtMemoryInfo for CPU memory + * + * Special case version of OrtApi::CreateMemoryInfo for CPU based memory. Same as using OrtApi::CreateMemoryInfo with name = "Cpu" and id = 0. + * + * \param[in] type + * \param[in] mem_type + * \param[out] out + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CreateCpuMemoryInfo, enum OrtAllocatorType type, enum OrtMemType mem_type, + _Outptr_ OrtMemoryInfo** out); + + /** \brief Compare ::OrtMemoryInfo objects for equality + * + * Compares all settings of each ::OrtMemoryInfo for equality + * + * \param[in] info1 + * \param[in] info2 + * \param[out] out Set to 0 if equal, -1 if not equal + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CompareMemoryInfo, _In_ const OrtMemoryInfo* info1, _In_ const OrtMemoryInfo* info2, _Out_ int* out); + + /** \brief Get name from ::OrtMemoryInfo + * + * \param[in] ptr + * \param[out] out Writes null terminated string to this pointer. Do NOT free the returned pointer. It is valid for the lifetime of the ::OrtMemoryInfo + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(MemoryInfoGetName, _In_ const OrtMemoryInfo* ptr, _Out_ const char** out); + + /** \brief Get the device id from ::OrtMemoryInfo + */ + ORT_API2_STATUS(MemoryInfoGetId, _In_ const OrtMemoryInfo* ptr, _Out_ int* out); + + /** \brief Get the ::OrtMemType from ::OrtMemoryInfo + */ + ORT_API2_STATUS(MemoryInfoGetMemType, _In_ const OrtMemoryInfo* ptr, _Out_ OrtMemType* out); + + /** \brief Get the ::OrtAllocatorType from ::OrtMemoryInfo + */ + ORT_API2_STATUS(MemoryInfoGetType, _In_ const OrtMemoryInfo* ptr, _Out_ OrtAllocatorType* out); + + /// @} + /// \name OrtAllocator + /// @{ + + /// \brief Calls OrtAllocator::Alloc function + ORT_API2_STATUS(AllocatorAlloc, _Inout_ OrtAllocator* ort_allocator, size_t size, _Outptr_ void** out); + /// \brief Calls OrtAllocator::Free function + ORT_API2_STATUS(AllocatorFree, _Inout_ OrtAllocator* ort_allocator, void* p); + /// \brief Calls OrtAllocator::Info function + ORT_API2_STATUS(AllocatorGetInfo, _In_ const OrtAllocator* ort_allocator, _Outptr_ const struct OrtMemoryInfo** out); + + /** \brief Get the default allocator + * + * The default allocator is a CPU based, non-arena. Always returns the same pointer to the same default allocator. + * + * \param[out] out Returned value should NOT be freed + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetAllocatorWithDefaultOptions, _Outptr_ OrtAllocator** out); + + /// @} + /// \name OrtSessionOptions + /// @{ + + /** \brief Override session symbolic dimensions + * + * Override symbolic dimensions (by specific denotation strings) with actual values if known at session initialization time to enable + * optimizations that can take advantage of fixed values (such as memory planning, etc) + * + * \param[in] options + * \param[in] dim_denotation + * \param[in] dim_value + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(AddFreeDimensionOverride, _Inout_ OrtSessionOptions* options, _In_ const char* dim_denotation, + _In_ int64_t dim_value); + + /// @} + /// \name OrtValue + /// @{ + + /* Internal information (not seen in Doxygen) + * + * APIs to support non-tensor types - map and sequence. + * Currently only the following types are supported + * Note: the following types should be kept in sync with data_types.h + * Map types + * ========= + * std::map + * std::map + * std::map + * std::map + * std::map + * std::map + * std::map + * std::map + * + * Sequence types + * ============== + * std::vector + * std::vector + * std::vector + * std::vector + * std::vector> + * std::vector + */ + + /** \brief Get non tensor data from an ::OrtValue + * + * If `value` is of type ONNX_TYPE_MAP, you need to retrieve the keys and values + * separately. Use index=0 to retrieve keys and index=1 to retrieve values. + * If `value` is of type ONNX_TYPE_SEQUENCE, use index to retrieve the index'th element + * of the sequence. + * + * \param[in] value + * \param[in] index See above for usage based on `value` type + * \param[in] allocator Allocator used to allocate ::OrtValue + * \param[out] out Created ::OrtValue that holds the element requested. Must be freed with OrtApi::ReleaseValue + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetValue, _In_ const OrtValue* value, int index, _Inout_ OrtAllocator* allocator, + _Outptr_ OrtValue** out); + + /** \brief Get non tensor value count from an ::OrtValue + * + * If `value` is of type ONNX_TYPE_MAP 2 will always be returned. For ONNX_TYPE_SEQUENCE + * the number of elements in the sequence will be returned + * + * \param[in] value + * \param[out] out + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetValueCount, _In_ const OrtValue* value, _Out_ size_t* out); + + /** \brief Create a map or sequence ::OrtValue + * + * To construct a map (ONNX_TYPE_MAP), use num_values = 2 and `in` should be an array of 2 ::OrtValue%s + * representing keys and values.
+ * + * To construct a sequence (ONNX_TYPE_SEQUENCE), use num_values = N where N is the number of the elements in the + * sequence. 'in' should be an array of N ::OrtValue%s. + * + * \param[in] in See above for details + * \param[in] num_values + * \param[in] value_type Must be either ONNX_TYPE_MAP or ONNX_TYPE_SEQUENCE + * \param[out] out Newly created ::OrtValue. Must be freed with OrtApi::ReleaseValue + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CreateValue, _In_reads_(num_values) const OrtValue* const* in, size_t num_values, + enum ONNXType value_type, _Outptr_ OrtValue** out); + + /** \brief Create an opaque (custom user defined type) ::OrtValue + * + * Constructs an ::OrtValue that contains a value of non-standard type created for + * experiments or while awaiting standardization. ::OrtValue in this case would contain + * an internal representation of the Opaque type. Opaque types are distinguished from + * each other by two strings 1) domain and 2) type name. The combination of the two + * must be unique, so the type representation is properly identified internally. The combination + * must be properly registered from within ORT at both compile/run time or by another API. + * + * To construct the ::OrtValue pass domain and type names, also a pointer to a data container + * the type of which must be known to both ORT and the client program. That data container may or may + * not match the internal representation of the Opaque type. The sizeof(data_container) is passed for + * verification purposes. + * + * \param[in] domain_name Null terminated string of the domain name + * \param[in] type_name Null terminated string of the type name + * \param[in] data_container User pointer Data to populate ::OrtValue + * \param[in] data_container_size Size in bytes of what `data_container` points to + * \param[out] out Newly created ::OrtValue. Must be freed with OrtApi::ReleaseValue + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CreateOpaqueValue, _In_z_ const char* domain_name, _In_z_ const char* type_name, + _In_ const void* data_container, size_t data_container_size, _Outptr_ OrtValue** out); + + /** \brief Get internal data from an opaque (custom user defined type) ::OrtValue + * + * Copies internal data from an opaque value into a user provided buffer + * + * \see OrtApi::CreateOpaqueValue + * + * \param[in] domain_name Null terminated string of the domain name + * \param[in] type_name Null terminated string of the type name + * \param[in] in The opaque ::OrtValue + * \param[out] data_container Buffer to copy data into + * \param[out] data_container_size Size in bytes of the buffer pointed to by data_container. Must match the size of the internal buffer. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetOpaqueValue, _In_ const char* domain_name, _In_ const char* type_name, _In_ const OrtValue* in, + _Out_ void* data_container, size_t data_container_size); + + /// @} + /// \name OrtKernelInfo + /// Custom operator APIs. + /// @{ + + /** \brief Get a float stored as an attribute in the graph node + * + * \param[in] info ::OrtKernelInfo instance + * \param[in] name Null terminated string of the name of the attribute + * \param[out] out Pointer to memory where the attribute will be stored + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(KernelInfoGetAttribute_float, _In_ const OrtKernelInfo* info, _In_ const char* name, + _Out_ float* out); + + /** \brief Fetch a 64-bit int stored as an attribute in the graph node + * + * \param[in] info ::OrtKernelInfo instance + * \param[in] name Null terminated string of the name of the attribute + * \param[out] out Pointer to memory where the attribute will be stored + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(KernelInfoGetAttribute_int64, _In_ const OrtKernelInfo* info, _In_ const char* name, + _Out_ int64_t* out); + + /** \brief Fetch a string stored as an attribute in the graph node + * + * If `out` is nullptr, the value of `size` is set to the true size of the string + * attribute, and a success status is returned. + * + * If the `size` parameter is greater than or equal to the actual string attribute's size, + * the value of `size` is set to the true size of the string attribute, the provided memory + * is filled with the attribute's contents, and a success status is returned. + * + * If the `size` parameter is less than the actual string attribute's size and `out` + * is not nullptr, the value of `size` is set to the true size of the string attribute + * and a failure status is returned.) + * + * \param[in] info ::OrtKernelInfo instance + * \param[in] name Null terminated string of the name of the attribute + * \param[out] out Pointer to memory where the attribute will be stored + * \param[in,out] size See above comments for details + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(KernelInfoGetAttribute_string, _In_ const OrtKernelInfo* info, _In_ const char* name, _Out_ char* out, + _Inout_ size_t* size); + + /// @} + /// \name OrtKernelContext + /// Custom operator APIs. + /// @{ + + /** \brief Used for custom operators, get the input count of a kernel + * + * \see ::OrtCustomOp + */ + ORT_API2_STATUS(KernelContext_GetInputCount, _In_ const OrtKernelContext* context, _Out_ size_t* out); + + /** \brief Used for custom operators, get the output count of a kernel + * + * \see ::OrtCustomOp + */ + ORT_API2_STATUS(KernelContext_GetOutputCount, _In_ const OrtKernelContext* context, _Out_ size_t* out); + + /** \brief Used for custom operators, get an input of a kernel + * + * The function attempts fetches the input of the kernel. If the input is optional + * and not present, the function returns success and out is set to nullptr. + * + * \param[in] context ::OrtKernelContext instance + * \param[in] index See KernelContext_GetInputCount for boundaries check. + * \param[out] out OrtValue if the input is present otherwise is set nullptr + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(KernelContext_GetInput, _In_ const OrtKernelContext* context, _In_ size_t index, + _Out_ const OrtValue** out); + + /** \brief Used for custom operators, get an output of a kernel + * + * The function attempts fetches the output of the kernel. If the output is optional + * and not present, the function returns success and out is set to nullptr. + * + * \param[in] context ::OrtKernelContext instance + * \param[in] index See KernelContext_GetOutputCount for boundaries check. + * \param[in] dim_values output dimensions + * \param[in] dim_count number of dimensions + * \param[out] out a ptr to OrtValue to output otherwise set to nullptr + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(KernelContext_GetOutput, _Inout_ OrtKernelContext* context, _In_ size_t index, + _In_ const int64_t* dim_values, size_t dim_count, _Outptr_ OrtValue** out); + + /// @} + /// \name OrtEnv + /// @{ + ORT_CLASS_RELEASE(Env); + /// @} + /// \name OrtStatus + /// @{ + ORT_CLASS_RELEASE(Status); + /// @} + /// \name OrtMemoryInfo + /// @{ + ORT_CLASS_RELEASE(MemoryInfo); + /// @} + /// \name OrtSession + /// @{ + ORT_CLASS_RELEASE(Session); // Don't call ReleaseSession from Dllmain (because session owns a thread pool) + /// @} + /// \name OrtValue + /// @{ + ORT_CLASS_RELEASE(Value); + /// @} + /// \name OrtRunOptions + /// @{ + ORT_CLASS_RELEASE(RunOptions); + /// @} + /// \name OrtTypeInfo + /// @{ + ORT_CLASS_RELEASE(TypeInfo); + /// @} + /// \name OrtTensorTypeAndShapeInfo + /// @{ + ORT_CLASS_RELEASE(TensorTypeAndShapeInfo); + /// @} + /// \name OrtSessionOptions + /// @{ + ORT_CLASS_RELEASE(SessionOptions); + /// @} + /// \name OrtCustomOpDomain + /// @{ + ORT_CLASS_RELEASE(CustomOpDomain); + + /// @} + /// \name OrtTypeInfo + /// @{ + + /** \brief Get denotation from type information + * + * Augments ::OrtTypeInfo to return denotations on the type. + * + * This is used by WinML to determine if an input/output is intended to be an Image or a Tensor. + * + * \param[in] type_info + * \param[out] denotation Pointer to the null terminated denotation string is written to this pointer. This pointer is valid until the object is destroyed or the name is changed, do not free. + * \param[out] len Length in bytes of the string returned in `denotation` + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetDenotationFromTypeInfo, _In_ const OrtTypeInfo* type_info, _Out_ const char** const denotation, + _Out_ size_t* len); + + /** \brief Get detailed map information from an ::OrtTypeInfo + * + * This augments ::OrtTypeInfo to return an ::OrtMapTypeInfo when the type is a map. + * The OrtMapTypeInfo has additional information about the map's key type and value type. + * + * This is used by WinML to support model reflection APIs. + * + * \param[out] type_info + * \param[out] out A pointer to the ::OrtMapTypeInfo. Do not free this value. If type_info + * does not contain a map, this value will be set to nullptr. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CastTypeInfoToMapTypeInfo, _In_ const OrtTypeInfo* type_info, + _Outptr_result_maybenull_ const OrtMapTypeInfo** out); + + /** \brief Cast ::OrtTypeInfo to an ::OrtSequenceTypeInfo + * + * This api augments ::OrtTypeInfo to return an ::OrtSequenceTypeInfo when the type is a sequence. + * The ::OrtSequenceTypeInfo has additional information about the sequence's element type. + * + * This is used by WinML to support model reflection APIs. + * + * \param[in] type_info + * \param[out] out A pointer to the OrtSequenceTypeInfo. Do not free this value. If type_info + * doesn not contain a sequence, this value will be set to nullptr. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CastTypeInfoToSequenceTypeInfo, _In_ const OrtTypeInfo* type_info, + _Outptr_result_maybenull_ const OrtSequenceTypeInfo** out); + + /// @} + /// \name OrtMapTypeInfo + /// @{ + + /** \brief Get key type from an ::OrtMapTypeInfo + * + * Key types are restricted to being scalar types. + * + * This is used by WinML to support model reflection APIs. + * + * \param[in] map_type_info + * \param[out] out + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetMapKeyType, _In_ const OrtMapTypeInfo* map_type_info, _Out_ enum ONNXTensorElementDataType* out); + + /** \brief Get the value type from an ::OrtMapTypeInfo + * + * \param[in] map_type_info + * \param[out] type_info A copy of the OrtTypeInfo for the map value type. + * The user must free this value with ReleaseTypeInfo. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetMapValueType, _In_ const OrtMapTypeInfo* map_type_info, _Outptr_ OrtTypeInfo** type_info); + + /// @} + /// \name OrtSequenceTypeInfo + /// @{ + + /** \brief Get element type from an ::OrtSequenceTypeInfo + * + * This is used by WinML to support model reflection APIs. + * + * \param[in] sequence_type_info + * \param[out] type_info A copy of the OrtTypeInfo for the sequence element type. + * The user must free this value with ReleaseTypeInfo. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetSequenceElementType, _In_ const OrtSequenceTypeInfo* sequence_type_info, + _Outptr_ OrtTypeInfo** type_info); + + /// @} + /// \name OrtMapTypeInfo + /// @{ + ORT_CLASS_RELEASE(MapTypeInfo); + /// @} + /// \name OrtSequenceTypeInfo + /// @{ + ORT_CLASS_RELEASE(SequenceTypeInfo); + + /// @} + /// \name OrtSession + /// @{ + + /** \brief End profiling and return filename of the profile data + * + * Profiling is turned on through OrtApi::EnableProfiling + * + * \param[in] session + * \param[in] allocator + * \param[out] out Null terminated string of the filename, allocated using `allocator`. Must be freed using `allocator` + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SessionEndProfiling, _In_ OrtSession* session, _Inout_ OrtAllocator* allocator, _Outptr_ char** out); + + /** \brief Get ::OrtModelMetadata from an ::OrtSession + * + * \param[in] session + * \param[out] out Newly created ::OrtModelMetadata. Must be freed using OrtApi::ReleaseModelMetadata + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SessionGetModelMetadata, _In_ const OrtSession* session, _Outptr_ OrtModelMetadata** out); + + /// @} + /// \name OrtModelMetadata + /// @{ + + /** \brief Get `producer name` from an ::OrtModelMetadata + * + * \param[in] model_metadata + * \param[in] allocator + * \param[out] value Set to a null terminated string allocated using `allocator`. Must be freed using `allocator` + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(ModelMetadataGetProducerName, _In_ const OrtModelMetadata* model_metadata, + _Inout_ OrtAllocator* allocator, _Outptr_ char** value); + + /** \brief Get `graph name` from an ::OrtModelMetadata + * + * \param[in] model_metadata + * \param[in] allocator + * \param[out] value Set to a null terminated string allocated using `allocator`. Must be freed using `allocator` + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(ModelMetadataGetGraphName, _In_ const OrtModelMetadata* model_metadata, + _Inout_ OrtAllocator* allocator, _Outptr_ char** value); + + /** \brief Get `domain` from an ::OrtModelMetadata + * + * \param[in] model_metadata + * \param[in] allocator + * \param[out] value Set to a null terminated string allocated using `allocator`. Must be freed using `allocator` + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(ModelMetadataGetDomain, _In_ const OrtModelMetadata* model_metadata, _Inout_ OrtAllocator* allocator, + _Outptr_ char** value); + + /** \brief Get `description` from an ::OrtModelMetadata + * + * \param[in] model_metadata + * \param[in] allocator + * \param[out] value Set to a null terminated string allocated using `allocator`. Must be freed using `allocator` + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(ModelMetadataGetDescription, _In_ const OrtModelMetadata* model_metadata, + _Inout_ OrtAllocator* allocator, _Outptr_ char** value); + + /** \brief Return data for a key in the custom metadata map in an ::OrtModelMetadata + * + * \param[in] model_metadata + * \param[in] allocator + * \param[in] key Null terminated string + * \param[out] value Set to a null terminated string allocated using `allocator`. Must be freed using `allocator` + * `value` will be set to nullptr if the given key is not found in the custom metadata map. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(ModelMetadataLookupCustomMetadataMap, _In_ const OrtModelMetadata* model_metadata, + _Inout_ OrtAllocator* allocator, _In_ const char* key, _Outptr_result_maybenull_ char** value); + + /** \brief Get version number from an ::OrtModelMetadata + * + * \param[in] model_metadata + * \param[out] value Set to the version number + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(ModelMetadataGetVersion, _In_ const OrtModelMetadata* model_metadata, _Out_ int64_t* value); + + ORT_CLASS_RELEASE(ModelMetadata); + + /// @} + /// \name OrtEnv + /// @{ + + /** \brief Create an OrtEnv + * + * Create an environment with global threadpools that will be shared across sessions. + * Use this in conjunction with OrtApi::DisablePerSessionThreads or else the session will use + * its own thread pools. + * + * \param[in] log_severity_level The log severity level. + * \param[in] logid The log identifier. + * \param[in] tp_options + * \param[out] out Returned newly created OrtEnv. Must be freed with OrtApi::ReleaseEnv + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CreateEnvWithGlobalThreadPools, OrtLoggingLevel log_severity_level, _In_ const char* logid, + _In_ const OrtThreadingOptions* tp_options, _Outptr_ OrtEnv** out); + + /// @} + /// \name OrtSessionOptions + /// @{ + + /** \brief Use global thread pool on a session + * + * Disable using per session thread pool and use the shared global threadpool. + * This should be used in conjunction with OrtApi::CreateEnvWithGlobalThreadPools. + * + * \param[in] options + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(DisablePerSessionThreads, _Inout_ OrtSessionOptions* options); + + /// @} + /// \name OrtThreadingOptions + /// @{ + + /** \brief Create an ::OrtThreadingOptions + * + * \param[out] out Newly created ::OrtThreadingOptions. Must be freed with OrtApi::ReleaseThreadingOptions + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CreateThreadingOptions, _Outptr_ OrtThreadingOptions** out); + + ORT_CLASS_RELEASE(ThreadingOptions); + + /// @} + /// \name OrtModelMetadata + /// @{ + + /** + * + * \param[in] model_metadata + * \param[in] allocator + * \param[out] keys Array of null terminated strings (array count = num_keys) allocated using `allocator`. + * The strings and the pointer array must be freed using `allocator` + * `keys` will be set to nullptr if the custom metadata map is empty. + * \param[out] num_keys Set to the number of elements in the `keys` array + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(ModelMetadataGetCustomMetadataMapKeys, _In_ const OrtModelMetadata* model_metadata, + _Inout_ OrtAllocator* allocator, _Outptr_result_buffer_maybenull_(*num_keys) char*** keys, _Out_ int64_t* num_keys); + + /// @} + /// \name OrtSessionOptions + /// @{ + + /** + * + * Override symbolic dimensions (by specific name strings) with actual values + * if known at session initialization time to enable optimizations that can + * take advantage of fixed values (such as memory planning, etc) + * + */ + ORT_API2_STATUS(AddFreeDimensionOverrideByName, + _Inout_ OrtSessionOptions* options, _In_ const char* dim_name, + _In_ int64_t dim_value); + + /// @} + /// \name Misc + /// @{ + + /** \brief Get the names of all available providers + * + * \note The providers in the list are not guaranteed to be usable. They may fail to load due to missing system dependencies. + * For example, if the CUDA/cuDNN libraries are not installed, the CUDA provider will report an error when it is added to the session options. + * + * \param[out] out_ptr Set to a pointer to an array of null terminated strings of the available providers. The entries and the + * array itself must be freed using OrtApi::ReleaseAvailableProviders + * \param[out] provider_length Set to the number of entries in the `out_ptr` array + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetAvailableProviders, _Outptr_ char*** out_ptr, _Out_ int* provider_length); + + /** \brief Release data from OrtApi::GetAvailableProviders. This API will never fail + * so you can rely on it in a noexcept code. + * + * \param[in] ptr The `out_ptr` result from OrtApi::GetAvailableProviders. + * \param[in] providers_length The `provider_length` result from OrtApi::GetAvailableProviders + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(ReleaseAvailableProviders, _In_ char** ptr, + _In_ int providers_length); + + /// @} + /// \name OrtValue + /// @{ + + /** \brief Get the length of a single string in a string tensor + * + * \param[in] value A string tensor + * \param[in] index Index of the string in the tensor + * \param[out] out Set to number of bytes of the string element + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetStringTensorElementLength, _In_ const OrtValue* value, size_t index, _Out_ size_t* out); + + /** \brief Get a single string from a string tensor + * + * \param[in] value A string tensor + * \param[in] s_len Number of bytes in the `s` buffer. Must match the value returned by OrtApi::GetStringTensorElementLength. + * \param[in] index Index of the string in the tensor + * \param[out] s The string element contents in UTF-8 encoding. The string is NOT null-terminated. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetStringTensorElement, _In_ const OrtValue* value, size_t s_len, size_t index, _Out_writes_bytes_all_(s_len) void* s); + + /** \brief Set a single string in a string tensor + * + * \param[in] value A string tensor + * \param[in] s A null terminated UTF-8 encoded string + * \param[in] index Index of the string in the tensor to set + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(FillStringTensorElement, _Inout_ OrtValue* value, _In_ const char* s, size_t index); + + /// @} + /// \name OrtSessionOptions + /// @{ + + /** \brief Set a session configuration entry as a pair of strings + * + * If a configuration with same key exists, this will overwrite the configuration with the given config_value. + * + * The config_key and the format of config_value are defined in onnxruntime_session_options_config_keys.h + * + * \param[in] options + * \param[in] config_key A null terminated string representation of the config key + * \param[in] config_value A null terminated string representation of the config value + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(AddSessionConfigEntry, _Inout_ OrtSessionOptions* options, + _In_z_ const char* config_key, _In_z_ const char* config_value); + + /// @} + /// \name OrtAllocator + /// @{ + + /** \brief Create an allocator for an ::OrtSession following an ::OrtMemoryInfo + * + * The allocator wraps the internal allocator from the OrtSession and becomes invalid when the session does. + * + * \param[in] session + * \param[in] mem_info valid ::OrtMemoryInfo instance + * \param[out] out Newly created ::OrtAllocator. Must be freed with OrtApi::ReleaseAllocator + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CreateAllocator, _In_ const OrtSession* session, _In_ const OrtMemoryInfo* mem_info, + _Outptr_ OrtAllocator** out); + + /** \brief Release an ::OrtAllocator obtained from OrtApi::CreateAllocator + */ + ORT_CLASS_RELEASE(Allocator); + + /// @} + /// \name OrtSession + /// @{ + + /** \brief Run a model using Io Bindings for the inputs & outputs + * + * \see OrtApi::Run + * + * \param[in] session + * \param[in] run_options + * \param[in] binding_ptr + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(RunWithBinding, _Inout_ OrtSession* session, _In_ const OrtRunOptions* run_options, _In_ const OrtIoBinding* binding_ptr); + + /** \brief Create an ::OrtIoBinding instance + * + * An IoBinding object allows one to bind pre-allocated ::OrtValue%s to input names. + * Thus if you want to use a raw on device buffer as input or output you can avoid + * extra copy during runtime. + * + * \param[in] session + * \param[out] out Newly created ::OrtIoBinding. Must be freed with OrtApi::ReleaseIoBinding + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CreateIoBinding, _Inout_ OrtSession* session, _Outptr_ OrtIoBinding** out); + + /// @} + /// \name OrtIoBinding + /// @{ + + /** \brief Release an ::OrtIoBinding obtained from OrtApi::CreateIoBinding + */ + ORT_CLASS_RELEASE(IoBinding); + + /** \brief Bind an ::OrtValue to an ::OrtIoBinding input + * + * When using OrtApi::RunWithBinding this value is used for the named input + * + * \param[in] binding_ptr + * \param[in] name Name for the model input + * \param[in] val_ptr ::OrtValue of Tensor type. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(BindInput, _Inout_ OrtIoBinding* binding_ptr, _In_ const char* name, _In_ const OrtValue* val_ptr); + + /** \brief Bind an ::OrtValue to an ::OrtIoBinding output + * + * When using OrtApi::RunWithBinding this value is used for the named output + * + * \param[in] binding_ptr + * \param[in] name Null terminated string of the model output name + * \param[in] val_ptr ::OrtValue of Tensor type. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(BindOutput, _Inout_ OrtIoBinding* binding_ptr, _In_ const char* name, _In_ const OrtValue* val_ptr); + + /** \brief Bind an ::OrtIoBinding output to a device + * + * Binds the ::OrtValue to a device which is specified by ::OrtMemoryInfo. + * You can either create an instance of ::OrtMemoryInfo with a device id or obtain one from the allocator that you have created/are using + * This is useful when one or more outputs have dynamic shapes and, it is hard to pre-allocate and bind a chunk of + * memory within ::OrtValue ahead of time. + * + * \see OrtApi::RunWithBinding + * + * \param[in] binding_ptr + * \param[in] name Null terminated string of the device name + * \param[in] mem_info_ptr + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(BindOutputToDevice, _Inout_ OrtIoBinding* binding_ptr, _In_ const char* name, _In_ const OrtMemoryInfo* mem_info_ptr); + + /** \brief Get the names of an ::OrtIoBinding's outputs + * + * Returns the names of the outputs in the order they were bound. This is useful after running the model + * with bound outputs because the returned names are in order in which output ::OrtValue are returned. This is useful if + * the order of outputs and their names is not known. + * + * \param[in] binding_ptr + * \param[in] allocator Allocator used to allocate continuous buffers for output strings and lengths. + * \param[out] buffer Returns an array of non-null terminated UTF-8 strings. The number of strings stored is returned in the count parameter. + * This buffer is allocated using `allocator` and must be freed using it. + * \param[out] lengths Returns an array of `count` lengths of the strings returned in `buffer` + * This buffer is allocated using `allocator` and must be freed using it. + * \param[out] count Number of strings returned. If `binding_ptr` has no bound outputs, zero is returned, + * no memory allocation is performed and buffer and lengths are set to nullptr. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetBoundOutputNames, _In_ const OrtIoBinding* binding_ptr, _In_ OrtAllocator* allocator, + _Out_ char** buffer, _Out_writes_all_(count) size_t** lengths, _Out_ size_t* count); + + /** \brief Get the output ::OrtValue objects from an ::OrtIoBinding + * + * Returns an array of pointers to individually allocated ::OrtValue%s that contain results of a model execution with OrtApi::RunWithBinding + * The array contains the same number of ::OrtValue%s and they are in the same order as they were bound with OrtApi::BindOutput + * or OrtApi::BindOutputToDevice. + * + * The returned ::OrtValue%s must be released using OrtApi::ReleaseValue after they are no longer needed. + * The array is allocated using the specified instance of the allocator and must be freed using the same allocator after + * all the ::OrtValue%s contained therein are individually released. + * + * \param[in] binding_ptr + * \param[in] allocator Allocator used to allocate output array + * \param[out] output Set to the allocated array of allocated ::OrtValue outputs. Set to nullptr if there are 0 outputs. + * \param[out] output_count Set to number of ::OrtValue%s returned + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetBoundOutputValues, _In_ const OrtIoBinding* binding_ptr, _In_ OrtAllocator* allocator, + _Out_writes_all_(output_count) OrtValue*** output, _Out_ size_t* output_count); + + /** \brief Clears any previously set Inputs for an ::OrtIoBinding + */ + void(ORT_API_CALL* ClearBoundInputs)(_Inout_ OrtIoBinding* binding_ptr) NO_EXCEPTION ORT_ALL_ARGS_NONNULL; + + /** \brief Clears any previously set Outputs for an ::OrtIoBinding + */ + void(ORT_API_CALL* ClearBoundOutputs)(_Inout_ OrtIoBinding* binding_ptr) NO_EXCEPTION ORT_ALL_ARGS_NONNULL; + + /// @} + /// \name OrtValue + /// @{ + + /** \brief Direct memory access to a specified tensor element + * + * For example, given a tensor with shape of [3,224,224], a pointer to the element at location [2,150,128] can be retrieved + * + * This function only works for numeric type tensors (No strings, etc). + * This is a no-copy method whose returned pointer is valid until the passed in ::OrtValue is free'd. + * + * \param[in] value + * \param[in] location_values Pointer to an array of index values that specify an element's location relative to its shape + * \param[in] location_values_count Number of elements in location_values. Must match the number of elements in the tensor's shape. + * \param[out] out Set to a pointer to the element specified + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(TensorAt, _Inout_ OrtValue* value, const int64_t* location_values, size_t location_values_count, _Outptr_ void** out); + + /// @} + /// \name OrtEnv + /// @{ + + /** \brief Create an allocator and register it with the ::OrtEnv + * + * Enables sharing the allocator between multiple sessions that use the same env instance. + * Lifetime of the created allocator will be valid for the duration of the environment. + * Returns an error if an allocator with the same ::OrtMemoryInfo is already registered. + * + * See https://onnxruntime.ai/docs/get-started/with-c.html for details. + * + * \param[in] env ::OrtEnv instance + * \param[in] mem_info + * \param[in] arena_cfg Pass nullptr for defaults + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CreateAndRegisterAllocator, _Inout_ OrtEnv* env, _In_ const OrtMemoryInfo* mem_info, + _In_ const OrtArenaCfg* arena_cfg); + + /** \brief Set language projection + * + * Set the language projection for collecting telemetry data when Env is created. + * + * The default is ORT_PROJECTION_C, which means it will classify the language not in the list to C also. + * + * \param[in] ort_env + * \param[in] projection + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SetLanguageProjection, _In_ const OrtEnv* ort_env, _In_ OrtLanguageProjection projection); + + /// @} + /// \name OrtSession + /// @{ + + /** \brief Return the time that profiling was started + * + * \note The timer precision varies per platform. On Windows and MacOS, the precision will be ~100ns + * + * \param[in] session + * \param[out] out nanoseconds of profiling's start time + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SessionGetProfilingStartTimeNs, _In_ const OrtSession* session, _Outptr_ uint64_t* out); + + /// @} + /// \name OrtThreadingOptions + /// @{ + + /** \brief Set global intra-op thread count + * + * This configures the global thread pool options to be used in the call to OrtApi::CreateEnvWithGlobalThreadPools + * + * \param[in] tp_options + * \param[in] intra_op_num_threads Number of threads, special values:
+ * 0 = Use default thread count
+ * 1 = The invoking thread will be used; no threads will be created in the thread pool. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SetGlobalIntraOpNumThreads, _Inout_ OrtThreadingOptions* tp_options, int intra_op_num_threads); + + /** \brief Set global inter-op thread count + * + * This configures the global thread pool options to be used in the call to OrtApi::CreateEnvWithGlobalThreadPools + * + * \param[in] tp_options + * \param[in] inter_op_num_threads Number of threads, special values:
+ * 0 = Use default thread count
+ * 1 = The invoking thread will be used; no threads will be created in the thread pool. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SetGlobalInterOpNumThreads, _Inout_ OrtThreadingOptions* tp_options, int inter_op_num_threads); + + /** \brief Set global spin control options + * + * This will configure the global thread pool options to be used in the call to OrtApi::CreateEnvWithGlobalThreadPools. + * Allow spinning of thread pools when their queues are empty. This will set the value for both + * inter_op and intra_op threadpools. + * + * \param[in] tp_options + * \param[in] allow_spinning Valid values are 0 or 1.
+ * 0 = It won't spin (recommended if CPU usage is high)
+ * 1 = Threadpool will spin to wait for queue to become non-empty + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SetGlobalSpinControl, _Inout_ OrtThreadingOptions* tp_options, int allow_spinning); + + /// @} + /// \name OrtSessionOptions + /// @{ + + /** \brief Add a pre-allocated initializer to a session + * + * If a model contains an initializer with a name that is same as the name passed to this call, + * ORT will use this initializer instance instead of deserializing one from the model file. This + * is useful when you want to share the same initializer across sessions. + * + * \param[in] options + * \param[in] name Null terminated string of the initializer name + * \param[in] val ::OrtValue containing the initializer. Its lifetime and the underlying initializer buffer must be + * managed by the user (created using the OrtApi::CreateTensorWithDataAsOrtValue) and it must outlive the session object + * to which it is added. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(AddInitializer, _Inout_ OrtSessionOptions* options, _In_z_ const char* name, + _In_ const OrtValue* val); + + /// @} + /// \name OrtEnv + /// @{ + + /** + * Create a custom environment with global threadpools and logger that will be shared across sessions. + * Use this in conjunction with OrtApi::DisablePerSessionThreads or else the session will use + * its own thread pools. + * + * \param[in] logging_function A pointer to a logging function. + * \param[in] logger_param A pointer to arbitrary data passed as the ::OrtLoggingFunction `param` parameter to + * `logging_function`. + * \param[in] log_severity_level The log severity level. + * \param[in] logid The log identifier. + * \param[in] tp_options + * \param[out] out Newly created OrtEnv. Must be freed with OrtApi::ReleaseEnv + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CreateEnvWithCustomLoggerAndGlobalThreadPools, OrtLoggingFunction logging_function, _In_opt_ void* logger_param, OrtLoggingLevel log_severity_level, + _In_ const char* logid, _In_ const struct OrtThreadingOptions* tp_options, _Outptr_ OrtEnv** out); + + /// @} + /// \name OrtSessionOptions + /// @{ + + /** \brief Append CUDA provider to session options + * + * If CUDA is not available (due to a non CUDA enabled build, or if CUDA is not installed on the system), this function will return failure. + * + * \param[in] options + * \param[in] cuda_options + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_CUDA, + _In_ OrtSessionOptions* options, _In_ const OrtCUDAProviderOptions* cuda_options); + + /** \brief Append ROCM execution provider to the session options + * + * If ROCM is not available (due to a non ROCM enabled build, or if ROCM is not installed on the system), this function will return failure. + * + * \param[in] options + * \param[in] rocm_options + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_ROCM, + _In_ OrtSessionOptions* options, _In_ const OrtROCMProviderOptions* rocm_options); + + /** \brief Append OpenVINO execution provider to the session options + * + * If OpenVINO is not available (due to a non OpenVINO enabled build, or if OpenVINO is not installed on the system), this function will fail. + * + * \param[in] options + * \param[in] provider_options + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_OpenVINO, + _In_ OrtSessionOptions* options, _In_ const OrtOpenVINOProviderOptions* provider_options); + + /// @} + /// \name OrtThreadingOptions + /// @{ + + /** \brief Set threading flush-to-zero and denormal-as-zero + * + * Sets global thread pool options to be used in the call to OrtApi::CreateEnvWithGlobalThreadPools. + * Flush-to-zero and denormal-as-zero are applied to threads in both intra and inter global thread pool. + * \note This option is not needed if the models used have no denormals. Having no denormals is recommended as this option may hurt model accuracy. + * + * \param[in] tp_options + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SetGlobalDenormalAsZero, _Inout_ OrtThreadingOptions* tp_options); + + /// @} + /// \name OrtArenaCfg + /// @{ + + /** \deprecated Use OrtApi::CreateArenaCfgV2 + * + * This will create the configuration of an arena that can eventually be used to define an arena based allocator's behavior + * + * \param[in] max_mem Use 0 to allow ORT to choose the default + * \param[in] arena_extend_strategy Use -1 to allow ORT to choose the default, 0 = kNextPowerOfTwo, 1 = kSameAsRequested + * \param[in] initial_chunk_size_bytes Use -1 to allow ORT to choose the default + * \param[in] max_dead_bytes_per_chunk Use -1 to allow ORT to choose the default + * \param[in] out A pointer to an OrtArenaCfg instance + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CreateArenaCfg, _In_ size_t max_mem, int arena_extend_strategy, int initial_chunk_size_bytes, + int max_dead_bytes_per_chunk, _Outptr_ OrtArenaCfg** out); + + ORT_CLASS_RELEASE(ArenaCfg); + + /// @} + /// \name OrtModelMetadata + /// @{ + + /** + * Use this to obtain the description of the graph present in the model + * (doc_string field of the GraphProto message within the ModelProto message). + * If it doesn't exist, an empty string will be returned. + * + * \param[in] model_metadata An instance of ::OrtModelMetadata + * \param[in] allocator Allocator used to allocate the string that will be returned back + * \param[out] value Set to a null terminated string allocated using `allocator`. The caller is responsible for freeing it using `allocator` + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(ModelMetadataGetGraphDescription, _In_ const OrtModelMetadata* model_metadata, + _Inout_ OrtAllocator* allocator, _Outptr_ char** value); + + /// @} + /// \name OrtSessionOptions + /// @{ + + /** \brief Append TensorRT provider to session options + * + * If TensorRT is not available (due to a non TensorRT enabled build, or if TensorRT is not installed on the system), this function will return failure. + * + * \param[in] options + * \param[in] tensorrt_options + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_TensorRT, + _In_ OrtSessionOptions* options, _In_ const OrtTensorRTProviderOptions* tensorrt_options); + + /// @} + /// \name Misc + /// @{ + + /** \brief Set current GPU device ID + * + * Set the current device id of the GPU execution provider (CUDA/tensorrt/rocm). The device id should be less + * than the total number of devices available. This is only useful when multiple-GPUs are installed and it is + * required to restrict execution to a single GPU. + * + * \param[in] device_id + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SetCurrentGpuDeviceId, _In_ int device_id); + + /** \brief Get current GPU device ID + * + * Get the current device id of the GPU execution provider (CUDA/tensorrt/rocm). + * + * \see OrtApi::SetCurrentGpuDeviceId + * + * \param[out] device_id + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetCurrentGpuDeviceId, _In_ int* device_id); + + /// @} + /// \name OrtKernelInfo + /// Custom operator APIs. + /// @{ + + /** \brief Fetch an array of int64_t values stored as an attribute in the graph node + * + * + * If `out` is nullptr, the value of `size` is set to the true size of the attribute + * array's size, and a success status is returned. + * + * If the `size` parameter is greater than or equal to the actual attribute array's size, + * the value of `size` is set to the true size of the attribute array's size, + * the provided memory is filled with the attribute's contents, + * and a success status is returned. + * + * If the `size` parameter is less than the actual attribute array's size and `out` + * is not nullptr, the value of `size` is set to the true size of the attribute array's size + * and a failure status is returned.) + * + * \param[in] info instance + * \param[in] name name of the attribute to be parsed + * \param[out] out pointer to memory where the attribute's contents are to be stored + * \param[in, out] size actual size of attribute array + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(KernelInfoGetAttributeArray_float, _In_ const OrtKernelInfo* info, _In_ const char* name, + _Out_ float* out, _Inout_ size_t* size); + + /** \brief Fetch an array of int64_t values stored as an attribute in the graph node + * + * If `out` is nullptr, the value of `size` is set to the true size of the attribute + * array's size, and a success status is returned. + * + * If the `size` parameter is greater than or equal to the actual attribute array's size, + * the value of `size` is set to the true size of the attribute array's size, + * the provided memory is filled with the attribute's contents, + * and a success status is returned. + * + * If the `size` parameter is less than the actual attribute array's size and `out` + * is not nullptr, the value of `size` is set to the true size of the attribute array's size + * and a failure status is returned.) + * + * \param[in] info instance + * \param[in] name name of the attribute to be parsed + * \param[out] out pointer to memory where the attribute's contents are to be stored + * \param[in, out] size actual size of attribute array + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(KernelInfoGetAttributeArray_int64, _In_ const OrtKernelInfo* info, _In_ const char* name, + _Out_ int64_t* out, _Inout_ size_t* size); + + /// @} + /// \name OrtArenaCfg + /// @{ + + /** \brief Create an ::OrtArenaCfg + * + * Create the configuration of an arena that can eventually be used to define an arena based allocator's behavior. + * + * Supported keys are (See https://onnxruntime.ai/docs/get-started/with-c.html for details on what the + * following parameters mean and how to choose these values.): + * "max_mem": Maximum memory that can be allocated by the arena based allocator. + * Use 0 for ORT to pick the best value. Default is 0. + * "arena_extend_strategy": 0 = kNextPowerOfTwo, 1 = kSameAsRequested. + * Use -1 to allow ORT to choose the default. + * "initial_chunk_size_bytes": (Possible) Size of the first allocation in the arena. + * Only relevant if arena strategy is `kNextPowerOfTwo`. Use -1 to allow ORT to choose the default. + * Ultimately, the first allocation size is determined by the allocation memory request. + * "max_dead_bytes_per_chunk": Threshold of unused memory in an allocated chunk of arena memory after + * crossing which the current chunk is chunked into 2. + * "initial_growth_chunk_size_bytes": (Possible) Size of the second allocation in the arena. + * Only relevant if arena strategy is `kNextPowerOfTwo`. Use -1 to allow ORT to choose the default. + * "max_power_of_two_extend_bytes": The maximum extend size if arena strategy is `kNextPowerOfTwo`. + * It is not an allocation limit, it is only a limit for extension when requested byte is less than the limit. + * When requested bytes is more than the limit, allocator will still return as requested. + * Use -1 to allow ORT to choose the default 1GB for max_power_of_two_extend_bytes. + * Ultimately, the allocation size is determined by the allocation memory request. + * Further allocation sizes are governed by the arena extend strategy. + * + * \param[in] arena_config_keys Keys to configure the arena + * \param[in] arena_config_values Values to configure the arena + * \param[in] num_keys Number of keys in `arena_config_keys` and `arena_config_values` + * \param[out] out Newly created ::OrtArenaCfg. Must be freed with OrtApi::ReleaseArenaCfg + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CreateArenaCfgV2, _In_reads_(num_keys) const char* const* arena_config_keys, + _In_reads_(num_keys) const size_t* arena_config_values, _In_ size_t num_keys, + _Outptr_ OrtArenaCfg** out); + + /// @} + /// \name OrtRunOptions + /// @{ + + /** \brief Set a single run configuration entry as a pair of strings + * + * If a configuration with same key exists, this will overwrite the configuration with the given config_value + * + * The config_key and the format of config_value are defined in onnxruntime_run_options_config_keys.h + * + * \param[in] options + * \param[in] config_key A null terminated string representation of the config key + * \param[in] config_value A null terminated string representation of the config value + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(AddRunConfigEntry, _Inout_ OrtRunOptions* options, + _In_z_ const char* config_key, _In_z_ const char* config_value); + + /// @} + /// \name OrtPrepackedWeightsContainer + /// @{ + + /** \brief Create an ::OrtPrepackedWeightsContainer + * + * This container will hold pre-packed buffers of shared initializers for sharing between sessions + * (i.e.) if there are shared initializers that can be shared between sessions, the pre-packed buffers + * of these (if any) may possibly be shared to provide memory footprint savings. Pass this container + * to sessions that you would like to share pre-packed buffers of shared initializers at session + * creation time. + * + * \param[out] out Newly created ::OrtPrepackedWeightsContainer. Must be freed with OrtApi::ReleasePrepackedWeightsContainer + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CreatePrepackedWeightsContainer, _Outptr_ OrtPrepackedWeightsContainer** out); + + /** \brief Release OrtPrepackedWeightsContainer instance + * + * \note instance must not be released until the sessions using it are released + */ + ORT_CLASS_RELEASE(PrepackedWeightsContainer); + + /// @} + /// \name OrtSession + /// @{ + + /** \brief Create session with prepacked weights container + * + * Same functionality offered by OrtApi::CreateSession except that a container that contains + * pre-packed weights' buffers is written into/read from by the created session. + * This is useful when used in conjunction with OrtApi::AddInitializer which injects + * shared initializer info into sessions. Wherever possible, the pre-packed versions of these + * shared initializers are cached in this container so that multiple sessions can just re-use + * these instead of duplicating these in memory. + * + * \param[in] env OrtEnv instance instance + * \param[in] model_path Null terminated string of the path (wchar on Windows, char otherwise) + * \param[in] options + * \param[in] prepacked_weights_container + * \param[out] out Newly created ::OrtSession. Must be freed with OrtApi::ReleaseSession + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CreateSessionWithPrepackedWeightsContainer, _In_ const OrtEnv* env, _In_ const ORTCHAR_T* model_path, + _In_ const OrtSessionOptions* options, + _Inout_ OrtPrepackedWeightsContainer* prepacked_weights_container, + _Outptr_ OrtSession** out); + + /** \brief Create session from memory with prepacked weights container + * + * Same functionality offered by OrtApi::CreateSessionFromArray except that a container that contains + * pre-packed weights' buffers is written into/read from by the created session. + * This is useful when used in conjunction with OrtApi::AddInitializer which injects + * shared initializer info into sessions. Wherever possible, the pre-packed versions of these + * shared initializers are cached in this container so that multiple sessions can just re-use + * these instead of duplicating these in memory. + * + * \param[in] env + * \param[in] model_data Array of bytes holding the model + * \param[in] model_data_length Number of bytes in `model_data_model` + * \param[in] options + * \param[in] prepacked_weights_container + * \param[out] out Newly created ::OrtSession. Must be freed with OrtApi::ReleaseSession + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CreateSessionFromArrayWithPrepackedWeightsContainer, _In_ const OrtEnv* env, + _In_ const void* model_data, size_t model_data_length, + _In_ const OrtSessionOptions* options, + _Inout_ OrtPrepackedWeightsContainer* prepacked_weights_container, + _Outptr_ OrtSession** out); + + /// @} + /// \name OrtSessionOptions + /// @{ + + /** \brief Append TensorRT execution provider to the session options + * + * If TensorRT is not available (due to a non TensorRT enabled build), this function will return failure. + * + * This is slightly different from OrtApi::SessionOptionsAppendExecutionProvider_TensorRT, it takes an + * ::OrtTensorRTProviderOptions which is publicly defined. This takes an opaque ::OrtTensorRTProviderOptionsV2 + * which must be created with OrtApi::CreateTensorRTProviderOptions. + * + * For OrtApi::SessionOptionsAppendExecutionProvider_TensorRT, the user needs to instantiate ::OrtTensorRTProviderOptions + * as well as allocate/release buffers for some members of ::OrtTensorRTProviderOptions. + * Here, OrtApi::CreateTensorRTProviderOptions and Ortapi::ReleaseTensorRTProviderOptions will do the memory management for you. + * + * \param[in] options + * \param[in] tensorrt_options + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_TensorRT_V2, + _In_ OrtSessionOptions* options, _In_ const OrtTensorRTProviderOptionsV2* tensorrt_options); + + /// @} + /// \name OrtTensorRTProviderOptionsV2 + /// @{ + + /** \brief Create an OrtTensorRTProviderOptionsV2 + * + * \param[out] out Newly created ::OrtTensorRTProviderOptionsV2. Must be released with OrtApi::ReleaseTensorRTProviderOptions + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CreateTensorRTProviderOptions, _Outptr_ OrtTensorRTProviderOptionsV2** out); + + /** \brief Set options in a TensorRT Execution Provider. + * + * Please refer to https://onnxruntime.ai/docs/execution-providers/TensorRT-ExecutionProvider.html#cc + * to know the available keys and values. Key should be in null terminated string format of the member of ::OrtTensorRTProviderOptionsV2 + * and value should be its related range. Recreates the options and only sets the supplied values. + * + * For example, key="trt_max_workspace_size" and value="2147483648" + * + * \param[in] tensorrt_options + * \param[in] provider_options_keys Array of UTF-8 null-terminated string for provider options keys + * \param[in] provider_options_values Array of UTF-8 null-terminated string for provider options values + * \param[in] num_keys Number of elements in the `provider_option_keys` and `provider_options_values` arrays + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(UpdateTensorRTProviderOptions, _Inout_ OrtTensorRTProviderOptionsV2* tensorrt_options, + _In_reads_(num_keys) const char* const* provider_options_keys, + _In_reads_(num_keys) const char* const* provider_options_values, + _In_ size_t num_keys); + + /** \brief Get serialized TensorRT provider options string. + * + * For example, "trt_max_workspace_size=2147483648;trt_max_partition_iterations=10;trt_int8_enable=1;......" + * + * \param tensorrt_options - OrtTensorRTProviderOptionsV2 instance + * \param allocator - a ptr to an instance of OrtAllocator obtained with OrtApi::CreateAllocator or OrtApi::GetAllocatorWithDefaultOptions + * the specified allocator will be used to allocate continuous buffers for output strings and lengths. + * \param ptr - is a UTF-8 null terminated string allocated using 'allocator'. The caller is responsible for using the same allocator to free it. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetTensorRTProviderOptionsAsString, _In_ const OrtTensorRTProviderOptionsV2* tensorrt_options, _Inout_ OrtAllocator* allocator, _Outptr_ char** ptr); + + /** \brief Release an ::OrtTensorRTProviderOptionsV2 + * + * \note This is an exception in the naming convention of other Release* functions, as the name of the method does not have the V2 suffix, but the type does + */ + void(ORT_API_CALL* ReleaseTensorRTProviderOptions)(_Frees_ptr_opt_ OrtTensorRTProviderOptionsV2* input); + + /// @} + /// \name OrtSessionOptions + /// @{ + + /** \brief Enable custom operators + * + * See onnxruntime-extensions: https://github.com/microsoft/onnxruntime-extensions.git + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(EnableOrtCustomOps, _Inout_ OrtSessionOptions* options); + + /// @} + /// \name OrtAllocator + /// @{ + + /** \brief Register a custom allocator + * + * Enables sharing between multiple sessions that use the same env instance. + * Returns an error if an allocator with the same ::OrtMemoryInfo is already registered. + * + * The behavior of this is exactly the same as OrtApi::CreateAndRegisterAllocator except + * instead of ORT creating an allocator based on provided info, in this case + * ORT uses the user-provided custom allocator. + * See https://onnxruntime.ai/docs/get-started/with-c.html for details. + * + * \param[in] env + * \param[in] allocator User provided allocator + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(RegisterAllocator, _Inout_ OrtEnv* env, _In_ OrtAllocator* allocator); + + /** \brief Unregister a custom allocator + * + * It is an error if you provide an ::OrtMemoryInfo not corresponding to any + * registered allocators for sharing. + * + * \param[in] env + * \param[in] mem_info + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(UnregisterAllocator, _Inout_ OrtEnv* env, + _In_ const OrtMemoryInfo* mem_info); + + /// @} + /// \name OrtValue + /// @{ + + /** \brief Sets *out to 1 iff an ::OrtValue is a SparseTensor, and 0 otherwise + * + * \param[in] value existing ::OrtValue + * \param[out] out unless an error occurs, contains 1 iff the value contains an instance + * of sparse tensor or 0 otherwise. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(IsSparseTensor, _In_ const OrtValue* value, _Out_ int* out); + + /** \brief Create an ::OrtValue with a sparse tensor that is empty. + * + * Use FillSparseTensor() functions to populate sparse tensor with non-zero values and + * format specific indices data. + * Use ReleaseValue to destroy the sparse tensor, this will also release the buffer inside the output value + * if any was allocated. + * \param[in,out] allocator allocator to use when performing an allocation. Allocation will be performed + * by FillSparseTensor() APIs. The lifespan of the allocator instance must eclipse the lifespan + * this sparse tensor instance as the same allocator will be used to free memory. + * \param[in] dense_shape shape of the original dense tensor + * \param[in] dense_shape_len number of shape dimensions being passed + * \param[in] type must be one of TENSOR_ELEMENT_DATA_TYPE_xxxx + * \param[out] out Should be freed by calling ReleaseValue + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CreateSparseTensorAsOrtValue, _Inout_ OrtAllocator* allocator, _In_ const int64_t* dense_shape, + size_t dense_shape_len, ONNXTensorElementDataType type, _Outptr_ OrtValue** out); + + /** + * This fills populates an empty tensor that was created using OrtApi::CreateSparseTensorAsOrtValue. + * This will allocate required memory and copy the supplied NNZ values and COO indices into that memory allocation. + * Memory allocation is performed using the allocator that was specified with OrtApi::CreateSparseTensorAsOrtValue. + * + * \param[in,out] ort_value ::OrtValue to populate with data + * \param[in] data_mem_info serves to identify the location of the data to be copied. If the allocator specified + * at the creation time has memory info that is not the same as mem_info argument to this function a X-device copy will be performed. + * String data is assumed to be on CPU and will only be copied into a CPU allocated buffer. + * \param[in] values_shape pointer to values shape array + * \param[in] values_shape_len length of the values_shape + * \param[in] values pointer to an array of values. For strings, pass const char**. + * \param[in] indices_data pointer to a location of COO indices + * \param[in] indices_num number of COO indices + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(FillSparseTensorCoo, _Inout_ OrtValue* ort_value, _In_ const OrtMemoryInfo* data_mem_info, + _In_ const int64_t* values_shape, size_t values_shape_len, _In_ const void* values, + _In_ const int64_t* indices_data, size_t indices_num); + + /** + * This fills populates an empty tensor that was created using OrtApi::CreateSparseTensorAsOrtValue. + * This will allocate required memory and copy the supplied NNZ values and CSR indices into that memory allocation. + * Memory allocation is performed using the allocator that was specified with OrtApi::CreateSparseTensorAsOrtValue. + * + * \param[in,out] ort_value ::OrtValue to populate with data + * \param[in] data_mem_info serves to identify the location of the data to be copied. If the allocator specified + * at the creation time has memory info that is not the same as mem_info argument to this function a X-device copy will be performed. + * String data is assumed to be on CPU and will only be copied into a CPU allocated buffer. + * \param[in] values_shape pointer to values shape array + * \param[in] values_shape_len length of the values_shape + * \param[in] values - pointer to an array of values. For strings, pass const char**. + * \param[in] inner_indices_data pointer to a location of CSR inner indices + * \param[in] inner_indices_num number of CSR inner indices + * \param[in] outer_indices_data pointer to a location of CSR outer indices + * \param[in] outer_indices_num number of CSR outer indices + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(FillSparseTensorCsr, _Inout_ OrtValue* ort_value, _In_ const OrtMemoryInfo* data_mem_info, + _In_ const int64_t* values_shape, size_t values_shape_len, _In_ const void* values, + _In_ const int64_t* inner_indices_data, size_t inner_indices_num, + _In_ const int64_t* outer_indices_data, size_t outer_indices_num); + + /** + * This fills populates an empty tensor that was created using OrtApi::CreateSparseTensorAsOrtValue. + * This will allocate required memory and copy the supplied NNZ values and BlockSparse indices into that memory allocation. + * Memory allocation is performed using the allocator that was specified with OrtApi::CreateSparseTensorAsOrtValue. + * + * \param[in,out] ort_value ::OrtValue to populate with data + * \param[in] data_mem_info serves to identify the location of the data to be copied. If the allocator specified + * at the creation time has memory info that is not the same as mem_info argument to this function a X-device copy will be performed. + * String data is assumed to be on CPU and will only be copied into a CPU allocated buffer. + * \param[in] values_shape + * \param[in] values_shape_len + * \param[in] values structure with values information + * \param[in] indices_shape_data pointer to a location of indices shape + * \param[in] indices_shape_len length of the block sparse indices shape + * \param[in] indices_data pointer to a location of indices data. Shape will determine the length of the indices data. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(FillSparseTensorBlockSparse, _Inout_ OrtValue* ort_value, _In_ const OrtMemoryInfo* data_mem_info, + _In_ const int64_t* values_shape, size_t values_shape_len, _In_ const void* values, + _In_ const int64_t* indices_shape_data, size_t indices_shape_len, + _In_ const int32_t* indices_data); + + /** + * Create an ::OrtValue with a sparse tensor. This is the first step. + * Next, use UseIndices() functions to supply sparse tensor with + * format specific indices data and set its sparse format to a specific enum value. + * This will not perform memory allocations. It will + * use supplied user buffer which should outlive the created sparse tensor. + * Use OrtApi::ReleaseValue to destroy the sparse tensor. It would not release the supplied values buffer. + * This function can not be used to map strings from the user allocated memory. Strings must always be copied + * and have UTF-8 encoding. Therefore, use OrtApi::CreateSparseTensorAsOrtValue above and then fill it with data + * using appropriate Make*() function. + * + * \param[in] info memory info where sparse values reside. + * \param[in,out] p_data pointer to a user allocated buffer with values. To create a full sparse tensor with no non-zero + * values, pass nullptr + * \param[in] dense_shape shape of the original dense tensor + * \param[in] dense_shape_len number of shape dimensions being passed + * \param[in] values_shape shape of the values data. To create a fully sparse tensor with no non-zero values, + * pass {0} shape. + * \param[in] values_shape_len number of values shape dimensions + * \param[in] type must be one of TENSOR_ELEMENT_DATA_TYPE_xxxx + * \param[out] out Should be freed by calling ReleaseValue + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(CreateSparseTensorWithValuesAsOrtValue, _In_ const OrtMemoryInfo* info, _Inout_ void* p_data, + _In_ const int64_t* dense_shape, size_t dense_shape_len, + _In_ const int64_t* values_shape, size_t values_shape_len, + ONNXTensorElementDataType type, _Outptr_ OrtValue** out); + + /** + * This assigns Coo format indices to the SparseTensor that was created by + * OrtApi::CreateSparseTensorWithValuesAsOrtValue above. It also sets OrtSparseFormat to + * ORT_SPARSE_COO. This will not allocate any additional memory for data. The life span of + * indices_data buffer should eclipse the life span of this ::OrtValue. + * + * \param[in,out] ort_value ::OrtValue instance constructed with OrtApi::CreateSparseTensorWithValuesAsOrtValue + * \param[in,out] indices_data pointer to a user pre-allocated buffer or nullptr for fully sparse tensors. + * \param[in] indices_num number of COO indices. Should either be 0 for fully sparse tensors, be equal + * to the number of nnz values specified to OrtApi::CreateSparseTensorWithValuesAsOrtValue for 1-D {nnz} indices or + * be twice as number of nnz values for a 2-D indices {nnz, 2} + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(UseCooIndices, _Inout_ OrtValue* ort_value, _Inout_ int64_t* indices_data, size_t indices_num); + + /** + * The assigns CSR format indices to the SparseTensor that was created by + * OrtApi::CreateSparseTensorWithValuesAsOrtValue above. It also sets OrtSparseFormat to + * ORT_SPARSE_CSRC. This will not allocate any additional memory for data. The life spans of + * inner_data and outer_data buffers should eclipse the life span of this ::OrtValue. + * + * \param[in,out] ort_value ::OrtValue instance constructed with OrtApi::CreateSparseTensorWithValuesAsOrtValue + * \param[in,out] inner_data pointer to a user pre-allocated buffer or nullptr for fully sparse tensors. + * \param[in] inner_num number of inner CSR indices. Should either be 0 for fully sparse tensors or be equal + * to the number of nnz values specified to OrtApi::CreateSparseTensorWithValuesAsOrtValue. + * \param[in,out] outer_data pointer to user pre-allocated buffer or nullptr for fully sparse tensors. + * \param[in] outer_num number of CSR outer indices. Should either be 0 for fully sparse tensors or + * equal to rows + 1 of the dense shape. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(UseCsrIndices, _Inout_ OrtValue* ort_value, _Inout_ int64_t* inner_data, size_t inner_num, + _Inout_ int64_t* outer_data, size_t outer_num); + + /** + * The assigns BlockSparse format indices to the SparseTensor that was created by + * OrtApi::CreateSparseTensorWithValuesAsOrtValue above. It also sets OrtSparseFormat to + * ORT_SPARSE_BLOCK_SPARSE. This will not allocate any additional memory for data. The life span of + * indices_data buffer must eclipse the lifespan of this ::OrtValue. + * + * \param[in,out] ort_value OrtValue instance constructed with OrtApi::CreateSparseTensorWithValuesAsOrtValue + * \param[in] indices_shape pointer to indices shape. Use {0} for fully sparse tensors + * \param[in] indices_shape_len length of the indices shape + * \param[in,out] indices_data pointer to user pre-allocated buffer or nullptr for fully sparse tensors. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(UseBlockSparseIndices, _Inout_ OrtValue* ort_value, const int64_t* indices_shape, size_t indices_shape_len, _Inout_ int32_t* indices_data); + + /** \brief Returns sparse tensor format enum iff a given ort value contains an instance of sparse tensor. + * + * \param[in] ort_value ::OrtValue that contains an instance of sparse tensor + * \param[out] out pointer to out parameter + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetSparseTensorFormat, _In_ const OrtValue* ort_value, _Out_ enum OrtSparseFormat* out); + + /** \brief Returns data type and shape of sparse tensor values (nnz) iff ::OrtValue contains a SparseTensor. + * + * \param[in] ort_value An ::OrtValue that contains a fully constructed sparse tensor + * \param[out] out Must be freed by OrtApi::ReleaseTensorTypeAndShapeInfo + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetSparseTensorValuesTypeAndShape, _In_ const OrtValue* ort_value, _Outptr_ OrtTensorTypeAndShapeInfo** out); + + /** \brief Returns numeric data for sparse tensor values (nnz). For string values use GetStringTensor*(). + * + * \param[in] ort_value an instance of ::OrtValue containing sparse tensor + * \param[out] out returns a pointer to values data. Do not attempt to free this ptr. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetSparseTensorValues, _In_ const OrtValue* ort_value, _Outptr_ const void** out); + + /** \brief Returns data type, shape for the type of indices specified by indices_format. + * + * \param[in] ort_value ::OrtValue containing sparse tensor. + * \param[in] indices_format One of the indices formats. It is an error to request a format that the sparse + * tensor does not contain. + * \param[out] out an instance of ::OrtTensorTypeAndShapeInfo. Must be freed by OrtApi::ReleaseTensorTypeAndShapeInfo + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetSparseTensorIndicesTypeShape, _In_ const OrtValue* ort_value, enum OrtSparseIndicesFormat indices_format, _Outptr_ OrtTensorTypeAndShapeInfo** out); + + /** \brief Returns indices data for the type of the indices specified by indices_format + * + * \param[in] ort_value ::OrtValue containing sparse tensor. + * \param[in] indices_format One of the indices formats. It is an error to request a format that the sparse tensor does not contain. + * \param[out] num_indices Pointer to where the number of indices entries is returned + * \param[out] indices Returned pointer to the indices data. Do not free the returned pointer as it refers to internal data owned by the ::OrtValue + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetSparseTensorIndices, _In_ const OrtValue* ort_value, enum OrtSparseIndicesFormat indices_format, _Out_ size_t* num_indices, _Outptr_ const void** indices); + /// @} + /// \name OrtSessionOptions + /// @{ + + /** + * \brief Sets out to 1 iff an optional type OrtValue has an element, 0 otherwise (OrtValue is None) + * Use this API to find if the optional type OrtValue is None or not. + * If the optional type OrtValue is not None, use the OrtValue just like any other OrtValue. + * For example, if you get an OrtValue that corresponds to Optional(tensor) and + * if HasValue() returns true, use it as tensor and so on. + + * \param[in] value Input OrtValue. + * \param[out] out indicating if the input OrtValue contains data (1) or if it is a None (0) + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(HasValue, _In_ const OrtValue* value, _Out_ int* out); + + /// @} + /// \name OrtKernelContext + /// Custom operator APIs. + /// @{ + + /** \brief Used for custom operators, gets the GPU compute stream to use to launch the custom a GPU kernel + * \see ::OrtCustomOp + * \param[in] context OrtKernelContext instance + * \param[out] out Returns pointer to a GPU compute stream that can be used to launch the custom GPU kernel. + * If retrieving the GPU compute stream is not relevant (GPU not enabled in the build, kernel partitioned to + * some other EP), then a nullptr is returned as the output param. + * Do not free or mutate the returned pointer as it refers to internal data owned by the underlying session. + * Only use it for custom kernel launching. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(KernelContext_GetGPUComputeStream, _In_ const OrtKernelContext* context, _Outptr_ void** out); + + /// @} + /// \name GetTensorMemoryInfo + /// @{ + /** \brief Returns a pointer to the ::OrtMemoryInfo of a Tensor + * \param[in] value ::OrtValue containing tensor. + * \param[out] mem_info ::OrtMemoryInfo of the tensor. Do NOT free the returned pointer. It is valid for the lifetime of the ::OrtValue + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetTensorMemoryInfo, _In_ const OrtValue* value, _Out_ const OrtMemoryInfo** mem_info); + + /// @} + /// \name GetExecutionProviderApi + /// @{ + /** \brief Get a pointer to the requested version of the Execution Provider specific + * API extensions to the OrtApi + * \param[in] provider_name The name of the execution provider name. Currently only the following + * values are supported: "DML". + * \param[in] version Must be ::ORT_API_VERSION. + * \param[out] provider_api A void pointer containing a reference to the execution provider versioned api structure. + * For example, the provider_api pointer can be cast to the OrtDmlApi* when the provider_name is "DML". + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetExecutionProviderApi, _In_ const char* provider_name, _In_ uint32_t version, _Outptr_ const void** provider_api); + + /// @} + + /// \name SessionOptions + /// @{ + /** \brief Set custom thread creation function + * + * \param[in] options Session options + * \param[in] ort_custom_create_thread_fn Custom thread creation function + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SessionOptionsSetCustomCreateThreadFn, _Inout_ OrtSessionOptions* options, _In_ OrtCustomCreateThreadFn ort_custom_create_thread_fn); + + /** \brief Set creation options for custom thread + * + * \param[in] options Session options + * \param[in] ort_custom_thread_creation_options Custom thread creation options (can be nullptr) + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SessionOptionsSetCustomThreadCreationOptions, _Inout_ OrtSessionOptions* options, _In_ void* ort_custom_thread_creation_options); + + /** \brief Set custom thread join function + * + * \param[in] options Session options + * \param[in] ort_custom_join_thread_fn Custom join thread function, must not be nullptr when ort_custom_create_thread_fn is set + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SessionOptionsSetCustomJoinThreadFn, _Inout_ OrtSessionOptions* options, _In_ OrtCustomJoinThreadFn ort_custom_join_thread_fn); + /// @} + + /// \name OrtThreadingOptions + /// @{ + /** \brief Set custom thread creation function for global thread pools + * + * \param[inout] tp_options + * \param[in] ort_custom_create_thread_fn Custom thread creation function + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SetGlobalCustomCreateThreadFn, _Inout_ OrtThreadingOptions* tp_options, _In_ OrtCustomCreateThreadFn ort_custom_create_thread_fn); + + /** \brief Set custom thread creation options for global thread pools + * + * \param[inout] tp_options + * \param[in] ort_custom_thread_creation_options Custom thread creation options (can be nullptr) + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SetGlobalCustomThreadCreationOptions, _Inout_ OrtThreadingOptions* tp_options, _In_ void* ort_custom_thread_creation_options); + + /** \brief Set custom thread join function for global thread pools + * + * \param[inout] tp_options + * \param[in] ort_custom_join_thread_fn Custom thread join function, must not be nullptr when global ort_custom_create_thread_fn is set + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SetGlobalCustomJoinThreadFn, _Inout_ OrtThreadingOptions* tp_options, _In_ OrtCustomJoinThreadFn ort_custom_join_thread_fn); + /// @} + + /** \brief Synchronize bound inputs. The call may be necessary for some providers, such as cuda, + * in case the system that allocated bound memory operated on a different stream. However, the + * operation is provider specific and could be a no-op. + * + * \param[inout] binding_ptr + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SynchronizeBoundInputs, _Inout_ OrtIoBinding* binding_ptr); + + /** \brief Synchronize bound outputs. The call may be necessary for some providers, such as cuda, + * in case the system that allocated bound memory operated on a different stream. However, the + * operation is provider specific and could be a no-op. + * + * \param[inout] binding_ptr + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(SynchronizeBoundOutputs, _Inout_ OrtIoBinding* binding_ptr); + + /// \name OrtSessionOptions + /// @{ + + /** \brief Append CUDA execution provider to the session options + * + * If CUDA is not available (due to a non CUDA enabled build), this function will return failure. + * + * This is slightly different from OrtApi::SessionOptionsAppendExecutionProvider_CUDA, it takes an + * ::OrtCUDAProviderOptions which is publicly defined. This takes an opaque ::OrtCUDAProviderOptionsV2 + * which must be created with OrtApi::CreateCUDAProviderOptions. + * + * For OrtApi::SessionOptionsAppendExecutionProvider_CUDA, the user needs to instantiate ::OrtCUDAProviderOptions + * as well as allocate/release buffers for some members of ::OrtCUDAProviderOptions. + * Here, OrtApi::CreateCUDAProviderOptions and Ortapi::ReleaseCUDAProviderOptions will do the memory management for you. + * + * \param[in] options + * \param[in] cuda_options + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.11. + */ + ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_CUDA_V2, + _In_ OrtSessionOptions* options, _In_ const OrtCUDAProviderOptionsV2* cuda_options); + + /// @} + /// \name OrtCUDAProviderOptionsV2 + /// @{ + + /** \brief Create an OrtCUDAProviderOptionsV2 + * + * \param[out] out Newly created ::OrtCUDAProviderOptionsV2. Must be released with OrtApi::ReleaseCudaProviderOptions + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.11. + */ + ORT_API2_STATUS(CreateCUDAProviderOptions, _Outptr_ OrtCUDAProviderOptionsV2** out); + + /** \brief Set options in a CUDA Execution Provider. + * + * Please refer to https://onnxruntime.ai/docs/execution-providers/CUDA-ExecutionProvider.html#configuration-options + * to know the available keys and values. Key should be in null terminated string format of the member of ::OrtCUDAProviderOptionsV2 + * and value should be its related range. Recreates the options and only sets the supplied values. + * + * For example, key="device_id" and value="0" + * + * \param[in] cuda_options + * \param[in] provider_options_keys Array of UTF-8 null-terminated string for provider options keys + * \param[in] provider_options_values Array of UTF-8 null-terminated string for provider options values + * \param[in] num_keys Number of elements in the `provider_option_keys` and `provider_options_values` arrays + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.11. + */ + ORT_API2_STATUS(UpdateCUDAProviderOptions, _Inout_ OrtCUDAProviderOptionsV2* cuda_options, + _In_reads_(num_keys) const char* const* provider_options_keys, + _In_reads_(num_keys) const char* const* provider_options_values, + _In_ size_t num_keys); + + /** + * Get serialized CUDA provider options string. + * + * For example, "device_id=0;arena_extend_strategy=0;......" + * + * \param cuda_options - OrtCUDAProviderOptionsV2 instance + * \param allocator - a ptr to an instance of OrtAllocator obtained with CreateAllocator() or GetAllocatorWithDefaultOptions() + * the specified allocator will be used to allocate continuous buffers for output strings and lengths. + * \param ptr - is a UTF-8 null terminated string allocated using 'allocator'. The caller is responsible for using the same allocator to free it. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.11. + */ + ORT_API2_STATUS(GetCUDAProviderOptionsAsString, _In_ const OrtCUDAProviderOptionsV2* cuda_options, _Inout_ OrtAllocator* allocator, _Outptr_ char** ptr); + + /** \brief Release an ::OrtCUDAProviderOptionsV2 + * + * \note This is an exception in the naming convention of other Release* functions, as the name of the method does not have the V2 suffix, but the type does + * + * \since Version 1.11. + */ + void(ORT_API_CALL* ReleaseCUDAProviderOptions)(_Frees_ptr_opt_ OrtCUDAProviderOptionsV2* input); + + /// @} + + /** \brief Append MIGraphX provider to session options + * + * If MIGraphX is not available (due to a non MIGraphX enabled build, or if MIGraphX is not installed on the system), this function will return failure. + * + * \param[in] options + * \param[in] migraphx_options + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.11. + */ + ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_MIGraphX, + _In_ OrtSessionOptions* options, _In_ const OrtMIGraphXProviderOptions* migraphx_options); + + /** \brief Replace initialized Tensors with external data with the data provided in initializers. + * + * The function will find the initialized TensorProtos with external data in the graph with the provided names and + * replace them with the provided tensors. The API verifies that the TensorProto being replaced + * has an external data reference and has the same name, dimensions and data type as its replacement. The replacement + * will occur before any of the optimizations take place. The data will be copied into the graph + * since TensorProto can't refer to the user provided buffers. + * + * Once the model has been loaded, the OrtValue(s) added to SessionOptions instance will be removed + * from the internal SessionOptions copy to save memory, the user provided buffers can then be deallocated + * and the SessionOptions instance that refers to them can be destroyed. + * + * \param[in] options + * \param[in] initializer_names Array of null terminated UTF-8 encoded strings of the initializers names. + * \param[in] initializers Array of ::OrtValue type + * \param[in] num_initializers Number of elements in the initializer_names and initializers + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.12. + */ + ORT_API2_STATUS(AddExternalInitializers, _In_ OrtSessionOptions* options, + _In_reads_(num_initializers) const char* const* initializer_names, + _In_reads_(num_initializers) const OrtValue* const* initializers, size_t num_initializers); + + /** \brief: Create attribute of onnxruntime operator + * + * \param[in] name Name of the attribute + * \param[in] data Data content of the attribute + * \param[in] len Number of bytes stored in data for ORT_OP_ATTR_STRING. + Number of elements if data represents an array (e.g., ORT_OP_ATTR_INTS). Otherwise, set to 1. + * \param[in] type Data type + * \param[out] op_attr Attribute that has been created, which must be released by OrtApi::ReleaseOpAttr + * + * \since Version 1.12. + */ + ORT_API2_STATUS(CreateOpAttr, + _In_ const char* name, + _In_ const void* data, + _In_ int len, + _In_ OrtOpAttrType type, + _Outptr_ OrtOpAttr** op_attr); + + /* \brief: Release op attribute + * + * \param[in] opAttr Attribute created by OrtApi::CreateOpAttr + * + * \since Version 1.12. + */ + ORT_CLASS_RELEASE(OpAttr); + + /** \brief: Create onnxruntime native operator + * + * \param[in] info Kernel info + * \param[in] op_name Operator name + * \param[in] domain Operator domain + * \param[in] version Operator opset version + * \param[in] type_constraint_names Name of the type constraints, such as "T" or "T1" + * \param[in] type_constraint_values Type of each constraints + * \param[in] type_constraint_count Number of constraints + * \param[in] attr_values Attributes used to initialize the operator + * \param[in] attr_count Number of the attributes + * \param[in] input_count Number of inputs + * \param[in] output_count Number of outputs + * \param[out] ort_op Operator that has been created + * + * \since Version 1.12. + */ + ORT_API2_STATUS(CreateOp, + _In_ const OrtKernelInfo* info, + _In_z_ const char* op_name, + _In_z_ const char* domain, + int version, + _In_reads_(type_constraint_count) const char** type_constraint_names, + _In_reads_(type_constraint_count) const ONNXTensorElementDataType* type_constraint_values, + int type_constraint_count, + _In_reads_(attr_count) const OrtOpAttr* const* attr_values, + int attr_count, + int input_count, + int output_count, + _Outptr_ OrtOp** ort_op); + + /** \brief: Invoke the operator created by OrtApi::CreateOp + * The inputs must follow the order as specified in onnx specification + * + * \param[in] context Kernel context + * \param[in] ort_op Operator that has been created + * \param[in] input_values Array of inputs + * \param[in] input_count Number of inputs + * \param[in] output_values Array of outputs + * \param[in] output_count Number of outputs + * + * \since Version 1.12. + */ + ORT_API2_STATUS(InvokeOp, + _In_ const OrtKernelContext* context, + _In_ const OrtOp* ort_op, + _In_ const OrtValue* const* input_values, + _In_ int input_count, + _Inout_ OrtValue* const* output_values, + _In_ int output_count); + + /* \brief: Release an onnxruntime operator + * + * \param[in] Op Operator created by OrtApi::CreateOp + * + * \since Version 1.12. + */ + ORT_CLASS_RELEASE(Op); + + /** \brief: Append execution provider to the session options. + * \param[in] options + * \param[in] provider_name - provider to add. + * \param[in] provider_options_keys - keys to configure the provider options + * \param[in] provider_options_values - values to configure the provider options + * \param[in] num_keys - number of keys passed in + * + * Currently supported provider names: + * QNNExecutionProvider (or QNN) + * OpenVINOExecutionProvider (or OpenVINO) + * XnnpackExecutionProvider (or XNNPACK) + * WebNNExecutionProvider (or WEBNN) + * WebGpuExecutionProvider (or WebGPU) + * AzureExecutionProvider (or AZURE) + * JsExecutionProvider (or JS) + * VitisAIExecutionProvider (or VitisAI) + * CoreMLExecutionProvider (or CoreML) + * + * Note: If an execution provider has a dedicated SessionOptionsAppendExecutionProvider_ function + * that should be used to add it. + * + * QNN supported keys: + * "backend_type": Type of QNN backend. Specifies a backend path that is the associated QNN backend library file + * name. E.g., given backend type "htp", on Windows, the backend path would be "QnnHtp.dll", and on other + * platforms, it would be "libQnnHtp.so". Mutually exclusive with "backend_path". + * Available options: + * -# "cpu" + * -# "gpu" + * -# "htp": Default. + * -# "saver" + * -# "ir" + * "backend_path": File path to QNN backend library. Mutually exclusive with "backend_type". + * "profiling_level": QNN profiling level. + * Available options: + * -# "off": Default. + * -# "basic" + * -# "detailed" + * "profiling_file_path": QNN profiling file path if ETW not enabled. + * "rpc_control_latency": QNN RPC control latency. + * "vtcm_mb": QNN VTCM size in MB. default to 0(not set). + * "htp_performance_mode": QNN performance mode. + * Available options: + * -# "burst" + * -# "balanced" + * -# "default": Default. + * -# "high_performance" + * -# "high_power_saver" + * -# "low_balanced" + * -# "extreme_power_saver" + * -# "low_power_saver" + * -# "power_saver" + * -# "sustained_high_performance" + * "dump_qnn_ir_dlc": Use the QnnIr backend library to write .dlc files for each subgraph dispatched to QNN. When + * enabled, inference results will be incorrect. Use only for debugging. + * -# "0": Default: disabled + * -# "1": enabled + * "dump_qnn_ir_dlc_dir": Set the directory into which QnnIr will be configured to write QNN graphs as .dlc files. + * Default is current working directory. + * "qnn_ir_backend_path": File path to the QnnIr backend library. If "dump_qnn_ir_dlc" is enabled, use this path + * instead of looking for the Ir backend in the standard location. + * "qnn_saver_path": File path to the QNN Saver backend library. If specified, QNN Saver will be enabled and will + * dump QNN API calls to disk for replay/debugging. QNN Saver produces incorrect model inference results and + * may alter model/EP partitioning. Use only for debugging. + * "qnn_context_priority": QNN context priority. + * Available options: + * -# "low" + * -# "normal": Default. + * -# "normal_high" + * -# "high" + * "htp_graph_finalization_optimization_mode": Set the optimization mode for graph finalization on the HTP backend. + * Available options: + * -# "0": Default. + * -# "1": Faster preparation time, less optimal graph. + * -# "2": Longer preparation time, more optimal graph. + * -# "3": Longest preparation time, most likely even more optimal graph. See QNN SDK documentation for specific + * details. + * "soc_model": The SoC model number. Refer to the QNN SDK documentation for valid values. + * Defaults to "0" (unknown). + * "htp_arch": The minimum HTP architecture the driver will use to select compatible QNN operators. + * Available options: + * -# "0": Default (none). + * -# "68" + * -# "69" + * -# "73" + * -# "75" + * -# "81" + * "device_id": The ID of the device to use when setting 'htp_arch'. Defaults to "0" (for single device). + * "enable_htp_fp16_precision": Used for float32 model for HTP backend. + * Enable the float32 model to be inferenced with fp16 precision. Otherwise, it will be fp32 precision. + * -# "0": With fp32 precision. + * -# "1": Default. With fp16 precision. + * "offload_graph_io_quantization": Offload graph input quantization and graph output dequantization to another + * execution provider (typically CPU EP). + * -# "0": Disabled. QNN EP will handle quantization and dequantization of graph I/O. + * -# "1": Enabled. This is the default value. + * "enable_htp_spill_fill_buffer": Enable HTP spill fill buffer setting. The flag is used while generating context + * binary. + * -# "0": Default. Disabled. + * -# "1": Enabled. + * "enable_htp_shared_memory_allocator": Enable the QNN HTP shared memory allocator. Requires libcdsprpc.so/dll to + * be available. + * -# "0": Default. Disabled. + * -# "1": Enabled. + * "dump_json_qnn_graph": Set to "1" to dump QNN graphs generated by QNN EP as JSON files. Each graph partition + * assigned to QNN EP is dumped to a separate file. + * "json_qnn_graph_dir": Directory in which to dump QNN JSON graphs. If not specified, QNN graphs are dumped in the + * program's current working directory. Ignored if "dump_json_qnn_graph" is not set. + * "op_packages": QNN UDO op_package for QNN EP, allowed format: + *   "::[:],::[:]", + *   where op_type is the name of the operation, op_package_path is the path to the op package shared library, + * interface is the symbol name to register the op life cycle functions, and target is the backend type. For more + * details, refer to: https://docs.qualcomm.com/bundle/publicresource/topics/80-63442-50/op_packages.html + * [Advanced] "skip_qnn_version_check": Set to "1" to allow a different version of QNN to be used than what was compiled + * into ONNX Runtime. Differences in operator support, accuracy, performance, and QNN's ABI may lead to crashes, inaccurate + * results, and poor performance. Use with caution and test thoroughly. + * + * XNNPACK supported keys: + * "intra_op_num_threads": number of thread-pool size to use for XNNPACK execution provider. + * default value is 0, which means to use the session thread-pool size. + * + * \since Version 1.12. + */ + ORT_API2_STATUS(SessionOptionsAppendExecutionProvider, _In_ OrtSessionOptions* options, + _In_ const char* provider_name, + _In_reads_(num_keys) const char* const* provider_options_keys, + _In_reads_(num_keys) const char* const* provider_options_values, + _In_ size_t num_keys); + + /* \brief: Get a copy of kernel info + * + * \param[in] info Kernel info + * \param[out] info_copy Copy of kernel info + * + * \since Version 1.12. + */ + ORT_API2_STATUS(CopyKernelInfo, + _In_ const OrtKernelInfo* info, + _Outptr_ OrtKernelInfo** info_copy); + + /* \brief: Release kernel info + * + * \param[in] KernelInfo A copy of kernel info returned by CopyKernelInfo + * + * \since Version 1.12. + */ + ORT_CLASS_RELEASE(KernelInfo); + + /// \name Ort Training + /// @{ + /** \brief Gets the Training C Api struct + * + * Call this function to access the ::OrtTrainingApi structure that holds pointers to functions that enable + * training with onnxruntime. + * \note A NULL pointer will be returned and no error message will be printed if the training api + * is not supported with this build. A NULL pointer will be returned and an error message will be + * printed if the provided version is unsupported, for example when using a runtime older than the + * version created with this header file. + * + * \param[in] version Must be ::ORT_API_VERSION + * \return The ::OrtTrainingApi struct for the version requested. + * + * \since Version 1.13 + */ + const OrtTrainingApi*(ORT_API_CALL* GetTrainingApi)(uint32_t version)NO_EXCEPTION; + + /// @} + + /** \brief Append CANN provider to session options + * + * If CANN is not available (due to a non CANN enabled build, or if CANN is not installed on the system), this function will return failure. + * + * \param[in] options + * \param[in] cann_options + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.13. + */ + ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_CANN, + _In_ OrtSessionOptions* options, _In_ const OrtCANNProviderOptions* cann_options); + + /** \brief Create an OrtCANNProviderOptions + * + * \param[out] out created ::OrtCANNProviderOptions. Must be released with OrtApi::ReleaseCANNProviderOptions + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.13. + */ + ORT_API2_STATUS(CreateCANNProviderOptions, _Outptr_ OrtCANNProviderOptions** out); + + /** \brief Set options in a CANN Execution Provider. + * + * \param[in] cann_options + * \param[in] provider_options_keys Array of UTF-8 null-terminated string for provider options keys + * \param[in] provider_options_values Array of UTF-8 null-terminated string for provider options values + * \param[in] num_keys Number of elements in the `provider_option_keys` and `provider_options_values` arrays + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.13. + */ + ORT_API2_STATUS(UpdateCANNProviderOptions, _Inout_ OrtCANNProviderOptions* cann_options, + _In_reads_(num_keys) const char* const* provider_options_keys, + _In_reads_(num_keys) const char* const* provider_options_values, + _In_ size_t num_keys); + + /** \brief Get serialized CANN provider options string. + * + * \param[in] cann_options OrtCANNProviderOptions instance + * \param[in] allocator a ptr to an instance of OrtAllocator obtained with CreateAllocator() + * or GetAllocatorWithDefaultOptions(), the specified allocator will be used to allocate + * continuous buffers for output strings and lengths. + * \param[out] ptr is a UTF-8 null terminated string allocated using 'allocator'. + * The caller is responsible for using the same allocator to free it. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.13. + */ + ORT_API2_STATUS(GetCANNProviderOptionsAsString, _In_ const OrtCANNProviderOptions* cann_options, + _Inout_ OrtAllocator* allocator, _Outptr_ char** ptr); + + /** \brief Release an OrtCANNProviderOptions + * + * \param[in] input The pointer of OrtCANNProviderOptions which will been deleted + * + * \since Version 1.13. + */ + void(ORT_API_CALL* ReleaseCANNProviderOptions)(_Frees_ptr_opt_ OrtCANNProviderOptions* input); + + /* \brief Get OrtDevice type from MemoryInfo + * + * \since Version 1.14 + */ + void(ORT_API_CALL* MemoryInfoGetDeviceType)(_In_ const OrtMemoryInfo* ptr, _Out_ OrtMemoryInfoDeviceType* out); + + /* \brief Update the OrtEnv instance with custom log severity level + * + * \param[in] ort_env The OrtEnv instance being used + * \param[in] log_severity_level The log severity level. + * + * \since Version 1.14. + */ + ORT_API2_STATUS(UpdateEnvWithCustomLogLevel, _In_ OrtEnv* ort_env, OrtLoggingLevel log_severity_level); + + /* \brief Set affinities for intra op threads + * + * Affinity string follows format: + * logical_processor_id,logical_processor_id;logical_processor_id,logical_processor_id + * Semicolon isolates configurations among threads, while comma split processors where ith thread expected to attach to. + * e.g. 1,2,3;4,5 + * specifies affinities for two threads, with the 1st thread attach to the 1st, 2nd, and 3rd processor, and 2nd thread to the 4th and 5th. + * To ease the configuration, an "interval" is also allowed: + * e.g. 1-8;8-16;17-24 + * orders that the 1st thread runs on first eight processors, 2nd thread runs on next eight processors, and so forth. + * Note: + * 1. Once set, the number of thread affinities must equal to intra_op_num_threads - 1, + * ort does not set affinity on the main thread which is started and managed by the calling app; + * 2. For windows, ort will infer the group id from a logical processor id, for example, assuming there are two groups with each has 64 logical processors, + * an id of 64 will be inferred as the last processor of the 1st group, while 65 will be interpreted as the 1st processor of the second group. + * Hence 64-65 is an invalid configuration, because a windows thread cannot be attached to processors across group boundary. + * + * \since Version 1.14 + */ + ORT_API2_STATUS(SetGlobalIntraOpThreadAffinity, _Inout_ OrtThreadingOptions* tp_options, const char* affinity_string); + + /** \brief Register custom ops from a shared library. + * + * Loads a shared library (.dll on windows, .so on linux, etc) named 'library_name' and looks for this entry point: + * OrtStatus* RegisterCustomOps(OrtSessionOptions * options, const OrtApiBase* api); + * It then passes in the provided session options to this function along with the api base. + * + * The handle to the loaded library is automatically released by ORT when the last OrtSession that references the + * library handle is released. If no OrtSession is created, then the library handle is released when the provided + * OrtSessionOptions is released. + * + * \param[in] options The session options. + * \param[in] library_name The name of the shared library to load and register. Refer to OS-specific dynamic library + * loading utilities (e.g., LoadLibraryEx on Windows or dlopen on Linux/MacOS) for information + * on the format of library names and search paths. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.14 + */ + ORT_API2_STATUS(RegisterCustomOpsLibrary_V2, _Inout_ OrtSessionOptions* options, _In_ const ORTCHAR_T* library_name); + + /** \brief Register custom ops by calling a RegisterCustomOpsFn function. + * + * Searches for registration_func_name and if found calls it. + * + * The library containing the function must either be linked against or previously loaded by the executable. + * + * If you want ONNX Runtime to load the library and manage its lifetime, use RegisterCustomOpsLibrary_V2. + * + * RegisterCustomOpsUsingFunction can be used in scenarios where it may not be possible for ONNX Runtime to load + * the library from a path. e.g. mobile platforms where the library must be linked into the app. + * + * The registration function must have the signature of RegisterCustomOpsFn: + * OrtStatus* (*fn)(OrtSessionOptions* options, const OrtApiBase* api); + * + * See https://onnxruntime.ai/docs/reference/operators/add-custom-op.html for details on how the registration + * function should be implemented. + * + * \param[in] options OrtSessionOptions that is passed through as the first argument in the call to the + * registration function. + * \param[in] registration_func_name Name of registration function to use. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.14 + */ + ORT_API2_STATUS(RegisterCustomOpsUsingFunction, _Inout_ OrtSessionOptions* options, + _In_ const char* registration_func_name); + + /// \name OrtKernelInfo + /// Custom operator APIs. + /// @{ + + /** \brief Get the number of inputs from ::OrtKernelInfo. + * + * Used in the CreateKernel callback of an OrtCustomOp to query the number of inputs + * during kernel/session creation. + * + * \param[in] info Instance of ::OrtKernelInfo. + * \param[out] out Pointer to variable assigned with the result on success. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.14 + */ + ORT_API2_STATUS(KernelInfo_GetInputCount, _In_ const OrtKernelInfo* info, _Out_ size_t* out); + + /** \brief Get the number of outputs from ::OrtKernelInfo. + * + * Used in the CreateKernel callback of an OrtCustomOp to query the number of outputs + * during kernel/session creation. + * + * \param[in] info Instance of ::OrtKernelInfo. + * \param[out] out Pointer to variable assigned with the result on success. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.14 + */ + ORT_API2_STATUS(KernelInfo_GetOutputCount, _In_ const OrtKernelInfo* info, _Out_ size_t* out); + + /** \brief Get the name of a ::OrtKernelInfo's input. + * + * Used in the CreateKernel callback of an OrtCustomOp to query an input's name + * during kernel/session creation. + * + * If `out` is nullptr, the value of `size` is set to the size of the name + * string (including null-terminator), and a success status is returned. + * + * If the `size` parameter is greater than or equal to the name string's size and `out` is not nullptr, + * the value of `size` is set to the true size of the string (including null-terminator), + * the provided memory is filled with the string's contents, and a success status is returned. + * + * If the `size` parameter is less than the actual string's size and `out` + * is not nullptr, the value of `size` is set to the true size of the string + * and a failure status is returned. + * + * \param[in] info An instance of ::OrtKernelInfo. + * \param[in] index The index of the input name to get. Returns a failure status if out-of-bounds. + * \param[out] out Memory location into which to write the UTF-8 null-terminated string representing the input's name. + * \param[in,out] size Pointer to the size of the `out` buffer. See above comments for details. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.14 + */ + ORT_API2_STATUS(KernelInfo_GetInputName, _In_ const OrtKernelInfo* info, size_t index, _Out_opt_ char* out, + _Inout_ size_t* size); + + /** \brief Get the name of a ::OrtKernelInfo's output. + * + * Used in the CreateKernel callback of an OrtCustomOp to query an output's name + * during kernel/session creation. + * + * If `out` is nullptr, the value of `size` is set to the size of the name + * string (including null-terminator), and a success status is returned. + * + * If the `size` parameter is greater than or equal to the name string's size and `out` is not nullptr, + * the value of `size` is set to the true size of the string (including null-terminator), + * the provided memory is filled with the string's contents, and a success status is returned. + * + * If the `size` parameter is less than the actual string's size and `out` + * is not nullptr, the value of `size` is set to the true size of the string + * and a failure status is returned. + * + * \param[in] info An instance of ::OrtKernelInfo. + * \param[in] index The index of the output name to get. Returns a failure status if out-of-bounds. + * \param[out] out Memory location into which to write the UTF-8 null-terminated string representing the output's + * name. + * \param[in,out] size Pointer to the size of the `out` buffer. See above comments for details. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.14 + */ + ORT_API2_STATUS(KernelInfo_GetOutputName, _In_ const OrtKernelInfo* info, size_t index, _Out_opt_ char* out, + _Inout_ size_t* size); + + /** \brief Get the type information for a ::OrtKernelInfo's input. + * + * Used in the CreateKernel callback of an OrtCustomOp to query the shape and type information + * of an input during kernel/session creation. + * + * \param[in] info An instance of ::OrtKernelInfo. + * \param[in] index Which input to get the type information for + * \param[out] type_info Pointer set to the resulting ::OrtTypeInfo. Must be freed with OrtApi::ReleaseTypeInfo. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.14 + */ + ORT_API2_STATUS(KernelInfo_GetInputTypeInfo, _In_ const OrtKernelInfo* info, size_t index, + _Outptr_ OrtTypeInfo** type_info); + + /** \brief Get the type information for a ::OrtKernelInfo's output. + * + * Used in the CreateKernel callback of an OrtCustomOp to query the shape and type information + * of an output during kernel/session creation. + * + * \param[in] info An instance of ::OrtKernelInfo. + * \param[in] index Which input to get the type information for + * \param[out] type_info Pointer set to the resulting ::OrtTypeInfo. Must be freed with OrtApi::ReleaseTypeInfo. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.14 + */ + ORT_API2_STATUS(KernelInfo_GetOutputTypeInfo, _In_ const OrtKernelInfo* info, size_t index, + _Outptr_ OrtTypeInfo** type_info); + + /** \brief Get a ::OrtValue tensor stored as an attribute in the graph node. + * + * Used in the CreateKernel callback of an OrtCustomOp to get a tensor attribute. + * + * \param[in] info ::OrtKernelInfo instance. + * \param[in] name UTF-8 null-terminated string representing the attribute's name. + * \param[in] allocator Allocator used to allocate the internal tensor state. + * \param[out] out Returns newly created ::OrtValue. Must be freed with OrtApi::ReleaseValue, + * which will also free internal tensor state allocated with the provided allocator. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(KernelInfoGetAttribute_tensor, _In_ const OrtKernelInfo* info, _In_z_ const char* name, + _Inout_ OrtAllocator* allocator, _Outptr_ OrtValue** out); + + /// @} + /// \name OrtSessionOptions + /// Custom operator APIs + /// @{ + + /** \brief Checks if the given session configuration entry exists. + * + * The config_key formats are defined in onnxruntime_session_options_config_keys.h + * + * Can be used in a custom operator library to check for session configuration entries + * that target one or more custom operators in the library. Example: The config entry + * custom_op.myop.some_key targets a custom op named "myop". + * + * \param[in] options The ::OrtSessionOptions instance. + * \param[in] config_key A null-terminated UTF-8 string representation of the configuration key. + * \param[out] out Pointer set to 1 if the entry exists and 0 otherwise. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.14 + */ + ORT_API2_STATUS(HasSessionConfigEntry, _In_ const OrtSessionOptions* options, + _In_z_ const char* config_key, _Out_ int* out); + + /** \brief Get a session configuration value. + * + * Returns a failure status if the configuration key does not exist. + * The config_key and the format of config_value are defined in onnxruntime_session_options_config_keys.h + * + * If `config_value` is nullptr, the value of `size` is set to the true size of the string + * value (including null-terminator), and a success status is returned. + * + * If the `size` parameter is greater than or equal to the actual string value's size, + * the value of `size` is set to the true size of the string value, the provided memory + * is filled with the value's contents, and a success status is returned. + * + * If the `size` parameter is less than the actual string value's size and `config_value` + * is not nullptr, the value of `size` is set to the true size of the string value + * and a failure status is returned. + * + * Can be used in a custom operator library to get session configuration entries + * that target one or more custom operators in the library. Example: The config entry + * custom_op.myop.some_key targets a custom op named "myop". + * + * \param[in] options The session options. + * \param[in] config_key A null-terminated UTF-8 string representation of the config key. + * \param[in] config_value Pointer to memory where the null-terminated UTF-8 string value will be stored. + * \param[in,out] size Pointer to the size of the `config_value` buffer. See above comments for details. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.14 + */ + ORT_API2_STATUS(GetSessionConfigEntry, _In_ const OrtSessionOptions* options, + _In_z_ const char* config_key, _Out_ char* config_value, _Inout_ size_t* size); + + /// @} + + /** \brief Append dnnl provider to session options + * + * If oneDNN is not available, this function will return failure. + * + * \param[in] options + * \param[in] dnnl_options + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.15. + */ + ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_Dnnl, + _In_ OrtSessionOptions* options, _In_ const OrtDnnlProviderOptions* dnnl_options); + + /** \brief Create an OrtDnnlProviderOptions + * + * \param[out] out Newly created ::OrtDnnlProviderOptions. Must be released with OrtApi::ReleaseDnnlProviderOptions + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.15. + */ + ORT_API2_STATUS(CreateDnnlProviderOptions, _Outptr_ OrtDnnlProviderOptions** out); + + /** \brief Set options in a oneDNN Execution Provider. + * + * Key should be in null terminated string format of the member of ::OrtDnnlProviderOptions + * and value should be its related range. + * + * For example, key="use_arena" and value="1" + * + * \param[in] dnnl_options + * \param[in] provider_options_keys Array of UTF-8 null-terminated string for provider options keys + * \param[in] provider_options_values Array of UTF-8 null-terminated string for provider options values + * \param[in] num_keys Number of elements in the `provider_option_keys` and `provider_options_values` arrays + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.15. + */ + ORT_API2_STATUS(UpdateDnnlProviderOptions, _Inout_ OrtDnnlProviderOptions* dnnl_options, + _In_reads_(num_keys) const char* const* provider_options_keys, + _In_reads_(num_keys) const char* const* provider_options_values, + _In_ size_t num_keys); + + /** + * Get serialized oneDNN provider options string. + * + * For example, "use_arena=1;......" + * + * \param dnnl_options - OrtDnnlProviderOptions instance + * \param allocator - a ptr to an instance of OrtAllocator obtained with CreateAllocator() or GetAllocatorWithDefaultOptions() + * the specified allocator will be used to allocate continuous buffers for output strings and lengths. + * \param ptr - is a UTF-8 null terminated string allocated using 'allocator'. The caller is responsible for using the same allocator to free it. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.15. + */ + ORT_API2_STATUS(GetDnnlProviderOptionsAsString, _In_ const OrtDnnlProviderOptions* dnnl_options, _Inout_ OrtAllocator* allocator, _Outptr_ char** ptr); + + /** \brief Release an ::OrtDnnlProviderOptions + * + * \since Version 1.15. + */ + void(ORT_API_CALL* ReleaseDnnlProviderOptions)(_Frees_ptr_opt_ OrtDnnlProviderOptions* input); + + /// \name OrtKernelInfo + /// Custom operator APIs. + /// @{ + + /** \brief Get the graph node name from ::OrtKernelInfo. + * + * If `out` is nullptr, the value of `size` is set to the size of the name + * string (including null-terminator), and a success status is returned. + * + * If the `size` parameter is greater than or equal to the name string's size and `out` is not nullptr, + * the value of `size` is set to the true size of the string (including null-terminator), + * the provided memory is filled with the string's contents, and a success status is returned. + * + * If the `size` parameter is less than the actual string's size and `out` + * is not nullptr, the value of `size` is set to the true size of the string + * and a failure status is returned. + * + * Can be used in a custom operator's CreateKernel callback to get the name of the operator's node name in the graph. + * + * \param[in] info An instance of ::OrtKernelInfo. + * \param[out] out Memory location into which to write the UTF-8 null-terminated string representing the name. + * \param[in,out] size Pointer to the size of the `out` buffer. See above comments for details. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.15 + */ + ORT_API2_STATUS(KernelInfo_GetNodeName, _In_ const OrtKernelInfo* info, _Out_opt_ char* out, _Inout_ size_t* size); + + /** \brief Get the session logger from ::OrtKernelInfo. + * + * Used in the CreateKernel callback of an OrtCustomOp to get a logger that can be used to log + * messages. + * + * \param[in] info An instance of ::OrtKernelInfo. + * \param[out] logger Pointer set to the session's ::OrtLogger. Owned by ONNX Runtime, so do not free. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.15 + */ + ORT_API2_STATUS(KernelInfo_GetLogger, _In_ const OrtKernelInfo* info, _Outptr_ const OrtLogger** logger); + + /// @} + /// \name OrtKernelContext + /// Custom operator APIs. + /// @{ + + /** \brief Get the runtime logger from ::OrtKernelContext. + * + * Used in the KernelCompute callback of an OrtCustomOp to get a logger that can be used to log + * messages during inference. + * + * \param[in] context An instance of ::OrtKernelContext. + * \param[out] logger Pointer set to the kernel context's ::OrtLogger. Owned by ONNX Runtime, so do not free. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.15 + */ + ORT_API2_STATUS(KernelContext_GetLogger, _In_ const OrtKernelContext* context, _Outptr_ const OrtLogger** logger); + + /// @} + /// \name OrtLogger + /// Custom operator APIs. + /// @{ + + /** \brief Logs a message at the given severity level using the provided ::OrtLogger. + * + * Only messages with a severity level equal or greater than the ::OrtLogger's logging severity level + * are logged. Use OrtApi::Logger_GetLoggingSeverityLevel to get the ::OrtLogger's logging severity + * level. + * + * Can be used in custom operators to log messages with the logger retrieved via OrtApi::KernelInfo_GetLogger. + * + * \param[in] logger The ::OrtLogger instance. + * \param[in] log_severity_level The message's severity level. + * \param[in] message The message to log. + * \param[in] file_path The filepath of the file in which the message is logged. Usually the value of ORT_FILE. + * \param[in] line_number The file line number in which the message is logged. Usually the value of __LINE__. + * \param[in] func_name The name of the function in which the message is logged. Usually the value of __FUNCTION__. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.15 + */ + ORT_API2_STATUS(Logger_LogMessage, _In_ const OrtLogger* logger, OrtLoggingLevel log_severity_level, + _In_z_ const char* message, _In_z_ const ORTCHAR_T* file_path, int line_number, + _In_z_ const char* func_name); + + /** \brief Get the logging severity level of the ::OrtLogger. + * + * Can be used in a custom operator to get the logging severity level of the ::OrtLogger associated with + * the ::OrtKernelInfo. + * + * \param[in] logger The ::OrtLogger instance. + * \param[out] out Pointer to variable assigned with the logging severity level on success. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.15 + */ + ORT_API2_STATUS(Logger_GetLoggingSeverityLevel, _In_ const OrtLogger* logger, _Out_ OrtLoggingLevel* out); + + /// @} + + /** \brief Get a ::OrtValue tensor stored as a constant initializer in the graph node. + * + * Used in the CreateKernel callback of an OrtCustomOp to get a tensor value. + * + * \param[in] info ::OrtKernelInfo instance. + * \param[in] index The node index. + * \param[out] is_constant Is it a constant node input or not. + * \param[out] out The OrtValue tensor value. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.15. + */ + ORT_API2_STATUS(KernelInfoGetConstantInput_tensor, _In_ const OrtKernelInfo* info, size_t index, _Out_ int* is_constant, _Outptr_ const OrtValue** out); + + /** \brief Get Optional Type information from an ::OrtTypeInfo + * + * This augments ::OrtTypeInfo to return an ::OrtOptionalTypeInfo when the type is optional. + * The OrtOptionalTypeInfo also has a nested ::OrtTypeInfo that describes the type of the optional value. + * ::OrtOptionalTypeInfo type can only appear within model metadata to describe inputs/outputs. + * The actual OrtValues that are supplied in place of optional type inputs should contain + * specific type that is described by ::OrtOptionalTypeInfo. + * + * So the picture: ::OrtTypeInfo -> ::OrtOptionalTypeInfo -> ::OrtTypeInfo (describes the type that can be supplied + * in place of the optional type when creating the actual ::OrtValue). + * + * \param[in] type_info + * \param[out] out A pointer to the ::OrtOptionalTypeInfo. Do not free this value, + * it is owned by OrtTypeInfo instance. When the type_info does not represent + * optional type, nullptr is returned in out. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.15. + */ + ORT_API2_STATUS(CastTypeInfoToOptionalTypeInfo, _In_ const OrtTypeInfo* type_info, + _Outptr_result_maybenull_ const OrtOptionalTypeInfo** out); + + /** \brief Get OrtTypeInfo for the allowed contained type from an ::OrtOptionalTypeInfo. + * + * This augments ::OrtOptionalTypeInfo to return an ::OrtTypeInfo for the contained type. + * The OrtOptionalTypeInfo has a nested ::OrtTypeInfo that describes the type of the optional value. + * ::OrtOptionalTypeInfo type can only appear within model metadata to describe inputs/outputs. + * The actual OrtValues that are supplied in place of optional type inputs should contain + * specific type that is described by the returned ::OrtTypeInfo. + * + * \param[in] optional_type_info + * \param[out] out A copy of ::OrtTypeInfo for what the optional value could be. + * The user must free this value with ReleaseTypeInfo. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.15. + */ + ORT_API2_STATUS(GetOptionalContainedTypeInfo, _In_ const OrtOptionalTypeInfo* optional_type_info, + _Outptr_ OrtTypeInfo** out); + + /** \brief Set a single string in a string tensor + * Do not zero terminate the string data. + * + * \param[in] value A string tensor + * \param[in] index - flat index of the element + * \param[in] length_in_bytes length of the buffer in utf-8 bytes (without the null terminator) + * \param[inout] buffer - address of return value + * + * \snippet{doc} snippets.dox OrtStatus Return Value + */ + ORT_API2_STATUS(GetResizedStringTensorElementBuffer, _Inout_ OrtValue* value, _In_ size_t index, _In_ size_t length_in_bytes, _Inout_ char** buffer); + + /** \brief Get Allocator from KernelContext for a specific memoryInfo. Please use C API ReleaseAllocator to release out object + * + * \param[in] context OrtKernelContext instance + * \param[in] mem_info OrtMemoryInfo instance + * \param[out] out A pointer to OrtAllocator. Must be released with OrtApi::ReleaseAllocator. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.15. + */ + ORT_API2_STATUS(KernelContext_GetAllocator, _In_ const OrtKernelContext* context, _In_ const OrtMemoryInfo* mem_info, _Outptr_ OrtAllocator** out); + + /** \brief Returns a null terminated string of the build info including git info and cxx flags + * + * \return UTF-8 encoded version string. Do not deallocate the returned buffer. + * + * \since Version 1.15. + */ + const char*(ORT_API_CALL* GetBuildInfoString)(void); + + /// \name OrtROCMProviderOptions + /// @{ + + /** \brief Create an OrtROCMProviderOptions + * + * \param[out] out Newly created ::OrtROCMProviderOptions. Must be released with OrtApi::ReleaseROCMProviderOptions + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.16. + */ + ORT_API2_STATUS(CreateROCMProviderOptions, _Outptr_ OrtROCMProviderOptions** out); + + /** \brief Set options in a ROCm Execution Provider. + * + * Please refer to https://onnxruntime.ai/docs/execution-providers/ROCm-ExecutionProvider.html + * to know the available keys and values. Key should be in null terminated string format of the member of + * ::OrtROCMProviderOptions and value should be its related range. + * + * For example, key="device_id" and value="0" + * + * \param[in] rocm_options + * \param[in] provider_options_keys Array of UTF-8 null-terminated string for provider options keys + * \param[in] provider_options_values Array of UTF-8 null-terminated string for provider options values + * \param[in] num_keys Number of elements in the `provider_option_keys` and `provider_options_values` arrays + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.16. + */ + ORT_API2_STATUS(UpdateROCMProviderOptions, _Inout_ OrtROCMProviderOptions* rocm_options, + _In_reads_(num_keys) const char* const* provider_options_keys, + _In_reads_(num_keys) const char* const* provider_options_values, + _In_ size_t num_keys); + + /** + * Get serialized ROCm provider options string. + * + * For example, "device_id=0;arena_extend_strategy=0;......" + * + * \param rocm_options - OrtROCMProviderOptions instance + * \param allocator - a ptr to an instance of OrtAllocator obtained with CreateAllocator() or GetAllocatorWithDefaultOptions() + * the specified allocator will be used to allocate continuous buffers for output strings and lengths. + * \param ptr - is a UTF-8 null terminated string allocated using 'allocator'. The caller is responsible for using the same allocator to free it. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.16. + */ + ORT_API2_STATUS(GetROCMProviderOptionsAsString, _In_ const OrtROCMProviderOptions* rocm_options, _Inout_ OrtAllocator* allocator, _Outptr_ char** ptr); + + /** \brief Release an ::OrtROCMProviderOptions + * + * \note This is an exception in the naming convention of other Release* functions, as the name of the method does not have the V2 suffix, but the type does + * + * \since Version 1.16. + */ + void(ORT_API_CALL* ReleaseROCMProviderOptions)(_Frees_ptr_opt_ OrtROCMProviderOptions* input); + + /** \brief Create an allocator with specific type and register it with the ::OrtEnv + * This API enhance CreateAndRegisterAllocator that it can create an allocator with specific type, not just CPU allocator + * Enables sharing the allocator between multiple sessions that use the same env instance. + * Lifetime of the created allocator will be valid for the duration of the environment. + * Returns an error if an allocator with the same ::OrtMemoryInfo is already registered. + * \param[in] env OrtEnv instance + * \param[in] provider_type ExecutionProvider type + * \param[in] mem_info OrtMemoryInfo instance + * \param[in] arena_cfg Arena configuration + * \param[in] provider_options_keys key of the provider options map + * \param[in] provider_options_values value of the provider options map + * \param[in] num_keys Length of the provider options map + */ + ORT_API2_STATUS(CreateAndRegisterAllocatorV2, _Inout_ OrtEnv* env, _In_ const char* provider_type, + _In_ const OrtMemoryInfo* mem_info, _In_ const OrtArenaCfg* arena_cfg, + _In_reads_(num_keys) const char* const* provider_options_keys, _In_reads_(num_keys) const char* const* provider_options_values, _In_ size_t num_keys); + + /** \brief Run the model asynchronously in a thread owned by intra op thread pool + * + * \param[in] session + * \param[in] run_options If nullptr, will use a default ::OrtRunOptions + * \param[in] input_names Array of null terminated UTF8 encoded strings of the input names + * \param[in] input Array of ::OrtValue%s of the input values + * \param[in] input_len Number of elements in the input_names and inputs arrays + * \param[in] output_names Array of null terminated UTF8 encoded strings of the output names + * \param[in] output_names_len Number of elements in the output_names and outputs array + * \param[out] output OrtValue* array of size output_names_len. + * On calling RunAsync, output[i] could either be a null or a pointer to a preallocated OrtValue. + * Later, the output array will be passed to run_async_callback with all null(s) filled with valid + * OrtValue pointer(s) allocated by onnxruntime. + * NOTE: it is customer's duty to finally release the output array and each of its member, + * regardless of whether the member (OrtValue*) is allocated by onnxruntime or preallocated by the customer. + * \param[in] run_async_callback Callback function on model run completion + * \param[in] user_data User data that pass back to run_async_callback + */ + ORT_API2_STATUS(RunAsync, _Inout_ OrtSession* session, _In_opt_ const OrtRunOptions* run_options, + _In_reads_(input_len) const char* const* input_names, + _In_reads_(input_len) const OrtValue* const* input, size_t input_len, + _In_reads_(output_names_len) const char* const* output_names, size_t output_names_len, + _Inout_updates_all_(output_names_len) OrtValue** output, + _In_ RunAsyncCallbackFn run_async_callback, _In_opt_ void* user_data); + + /** + * Update TensorRT EP provider option where its data type is pointer, for example 'user_compute_stream'. + * If the data type of the provider option can be represented by string please use UpdateTensorRTProviderOptions. + * + * Note: It's caller's responsibility to properly manage the lifetime of the instance pointed by this pointer. + * + * \param tensorrt_options - OrtTensorRTProviderOptionsV2 instance + * \param key - Name of the provider option + * \param value - A pointer to the instance that will be assigned to this provider option + * + * \since Version 1.16. + */ + ORT_API2_STATUS(UpdateTensorRTProviderOptionsWithValue, _Inout_ OrtTensorRTProviderOptionsV2* tensorrt_options, _In_ const char* key, _In_ void* value); + + /** + * Get TensorRT EP provider option where its data type is pointer. + * If the data type of the provider option can be represented by string please use GetTensorRTProviderOptionsAsString. + * + * \param tensorrt_options - OrtTensorRTProviderOptionsV2 instance + * \param key - Name of the provider option + * \param ptr - A pointer to the instance that is kept by the provider option + * + * \since Version 1.16. + */ + ORT_API2_STATUS(GetTensorRTProviderOptionsByName, _In_ const OrtTensorRTProviderOptionsV2* tensorrt_options, _In_ const char* key, _Outptr_ void** ptr); + + /** + * Update CUDA EP provider option where its data type is pointer, for example 'user_compute_stream'. + * If the data type of the provider option can be represented by string please use UpdateCUDAProviderOptions. + * + * Note: It's caller's responsibility to properly manage the lifetime of the instance pointed by this pointer. + * + * \param cuda_options - OrtCUDAProviderOptionsV2 instance + * \param key - Name of the provider option + * \param value - A pointer to the instance that will be assigned to this provider option + * + * \since Version 1.16. + */ + ORT_API2_STATUS(UpdateCUDAProviderOptionsWithValue, _Inout_ OrtCUDAProviderOptionsV2* cuda_options, _In_ const char* key, _In_ void* value); + + /** + * Get CUDA EP provider option where its data type is pointer. + * If the data type of the provider option can be represented by string please use GetCUDAProviderOptionsAsString. + * + * \param cuda_options - OrtCUDAProviderOptionsV2 instance + * \param key - Name of the provider option + * \param ptr - A pointer to the instance that is kept by the provider option + * + * \since Version 1.16. + */ + ORT_API2_STATUS(GetCUDAProviderOptionsByName, _In_ const OrtCUDAProviderOptionsV2* cuda_options, _In_ const char* key, _Outptr_ void** ptr); + + /** + * Get a EP resource. + * E.g. a cuda stream or a cublas handle + * + * \param context - Kernel context + * \param resource_version - Version of the resource + * \param resource_id - Type of resource + * \param resource - A pointer to returned resource + * + * \since Version 1.16. + */ + ORT_API2_STATUS(KernelContext_GetResource, _In_ const OrtKernelContext* context, _In_ int resource_version, + _In_ int resource_id, _Outptr_ void** resource); + + /** \brief Set user logging function + * + * By default the logger created by the CreateEnv* functions is used to create the session logger as well. + * This function allows a user to override this default session logger with a logger of their own choosing. This way + * the user doesn't have to create a separate environment with a custom logger. This addresses the problem when + * the user already created an env but now wants to use a different logger for a specific session (for debugging or + * other reasons). + * + * \param[in] options + * \param[in] user_logging_function A pointer to a logging function. + * \param[in] user_logging_param A pointer to arbitrary data passed as the ::OrtLoggingFunction `param` parameter to + * `user_logging_function`. This parameter is optional. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.17. + */ + ORT_API2_STATUS(SetUserLoggingFunction, _Inout_ OrtSessionOptions* options, + _In_ OrtLoggingFunction user_logging_function, _In_opt_ void* user_logging_param); + + /** + * Get number of input from OrtShapeInferContext + * + * \param[in] context + * \param[out] out The number of inputs + * + * \since Version 1.17. + */ + ORT_API2_STATUS(ShapeInferContext_GetInputCount, _In_ const OrtShapeInferContext* context, _Out_ size_t* out); + + /** + * Get type and shape info of an input + * + * \param[in] context + * \param[in] index The index of the input + * \param[out] info Type shape info of the input + * + * \since Version 1.17. + */ + ORT_API2_STATUS(ShapeInferContext_GetInputTypeShape, _In_ const OrtShapeInferContext* context, _In_ size_t index, _Outptr_ OrtTensorTypeAndShapeInfo** info); + + /** + * Get attribute from OrtShapeInferContext. Note that OrtShapeInferContext is a per-node context, one could only read attribute from current node. + * + * \param[in] context + * \param[in] attr_name Name of the attribute + * \param[out] attr Handle of the attribute fetched + * + * \since Version 1.17. + */ + ORT_API2_STATUS(ShapeInferContext_GetAttribute, _In_ const OrtShapeInferContext* context, _In_ const char* attr_name, _Outptr_ const OrtOpAttr** attr); + + /** + * Set type and shape info of an output + * + * \param[in] context + * \param[in] index The index of the output + * \param[out] info Type shape info of the output + * + * \since Version 1.17. + */ + ORT_API2_STATUS(ShapeInferContext_SetOutputTypeShape, _In_ const OrtShapeInferContext* context, _In_ size_t index, _In_ const OrtTensorTypeAndShapeInfo* info); + + /** + * Set symbolic shape to type shape info + * + * \param[in] info Type shape info + * \param[in] dim_params Symbolic strings + * \param[in] dim_params_length Number of strings + * + * \since Version 1.17. + */ + ORT_API2_STATUS(SetSymbolicDimensions, _In_ OrtTensorTypeAndShapeInfo* info, _In_ const char* dim_params[], _In_ size_t dim_params_length); + + /** + * Read contents of an attribute to data + * + * \param[in] op_attr + * \param[in] type Attribute type + * \param[out] data Memory address to save raw content of the attribute + * \param[in] len Number of bytes allowed to store in data + * \param[out] out Number of bytes required to save the data when the call failed, or the real number of bytes saved to data on success + * + * \note Does not support reading graph attributes. Refer to Node_GetSubgraphs. + * + * \since Version 1.17. + */ + ORT_API2_STATUS(ReadOpAttr, _In_ const OrtOpAttr* op_attr, _In_ OrtOpAttrType type, _Inout_ void* data, _In_ size_t len, _Out_ size_t* out); + + /** \brief Set whether to use deterministic compute. + * + * Default is false. If set to true, this will enable deterministic compute for GPU kernels where possible. + * Note that this most likely will have a performance cost. + * + * \param[in] options + * \param[in] value + * + * \since Version 1.17. + */ + ORT_API2_STATUS(SetDeterministicCompute, _Inout_ OrtSessionOptions* options, bool value); + + /** + * Run fn in parallel + * + * \param[in] context + * \param[in] fn Function accepting usr_data and an integer as iterator + * \param[in] total The number of times fn is to be invoked + * \param[in] num_batch Number of batches by which the "total" is to be divided in maximum. When zero, there is no limit + * \param[in] usr_data User data to be passed back to fn + * + * \since Version 1.17. + */ + ORT_API2_STATUS(KernelContext_ParallelFor, _In_ const OrtKernelContext* context, _In_ void (*fn)(void*, size_t), _In_ size_t total, _In_ size_t num_batch, _In_ void* usr_data); + + /** \brief Append OpenVINO execution provider to the session options + * + * If OpenVINO is not available (due to a non OpenVINO enabled build, or if OpenVINO is not installed on the system), this function will fail. + * + * \param[in] options + * \param[in] provider_options_keys + * \param[in] provider_options_values + * \param[in] num_keys + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.17. + */ + ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_OpenVINO_V2, + _In_ OrtSessionOptions* options, + _In_reads_(num_keys) const char* const* provider_options_keys, + _In_reads_(num_keys) const char* const* provider_options_values, + _In_ size_t num_keys); + + /** \brief Append VitisAI provider to session options + * + * If VitisAI is not available (due to a non VitisAI enabled build, or if VitisAI is not installed on the system), this function will return failure. + * + * \param[in] options + * \param[in] provider_options_keys + * \param[in] provider_options_values + * \param[in] num_keys + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.18. + */ + ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_VitisAI, + _In_ OrtSessionOptions* options, + _In_reads_(num_keys) const char* const* provider_options_keys, + _In_reads_(num_keys) const char* const* provider_options_values, + _In_ size_t num_keys); + + /** \brief Get scratch buffer from the corresponding allocator under the specific OrtMemoryInfo object. + * NOTE: callers are responsible to release this scratch buffer from the corresponding allocator + * \param[in] context OrtKernelContext instance + * \param[in] mem_info OrtMemoryInfo instance + * \param[in] count_or_bytes How many bytes is this scratch buffer + * \param[out] out A pointer to the scratch buffer + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.18. + */ + ORT_API2_STATUS(KernelContext_GetScratchBuffer, _In_ const OrtKernelContext* context, _In_ const OrtMemoryInfo* mem_info, _In_ size_t count_or_bytes, _Outptr_ void** out); + + /** \brief Get allocator from KernelInfo for a specific memory type. Please use C API ReleaseAllocator to release out object + * + * \param[in] info OrtKernelInfo instance + * \param[in] mem_type OrtMemType object + * \param[out] out A pointer to OrtAllocator + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.18. + */ + ORT_API2_STATUS(KernelInfoGetAllocator, _In_ const OrtKernelInfo* info, _In_ OrtMemType mem_type, _Outptr_ OrtAllocator** out); + + /** \brief Replace initialized Tensors with external data with the provided files in memory + * + * The function will find the initialized TensorProtos with external data in the graph with the provided + * external file names and the file content in memory. The API gets the external file name, offset, data length + * from TensorProto, and locate the tensor data from the file in memory buffer. + * It creates a Tensor to replace the existing Tensor in graph. The replacement + * will occur before any of the optimizations take place. The data will be copied into the graph + * since TensorProto can't refer to the user provided buffers. + * + * \param[in] options + * \param[in] external_initializer_file_names Array of null terminated UTF-8 encoded strings of the file names + * which holds the external initializers. + * \param[in] external_initializer_file_buffer_array Array of pointers to the buffer of the file content. + * The buffer can be freed after session creation. + * \param[in] external_initializer_file_lengths Array of size_t to indicate the length of file content + * \param[in] num_external_initializer_files Number of external files + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.18. + */ + ORT_API2_STATUS(AddExternalInitializersFromFilesInMemory, _In_ OrtSessionOptions* options, + _In_reads_(num_external_initializer_files) const ORTCHAR_T* const* external_initializer_file_names, + _In_reads_(num_external_initializer_files) char* const* external_initializer_file_buffer_array, + _In_reads_(num_external_initializer_files) const size_t* external_initializer_file_lengths, + size_t num_external_initializer_files); + + /** \brief Create an OrtLoraAdapter + * + * The function attempts to locate file specified by adapter_file_path, read it and create an OrtLoraAdapter + * instance. The adapter_file_path should be a valid path to a file that contains a valid Lora Adapter + * format. The function attempts to validate the format at load time. The file will always be memory mapped, unless + * the platform does not support memory mapping, in which case the file will be read into memory. + * + * \param[in] adapter_file_path adapter file path. + * \param[in] allocator optional pointer to a device allocator. If specified + * data is copied to the device at some point before Run() is invoked. If nullptr, data stays on CPU. + * The data would still be copied to device if required by the model at inference time. + * \param[out] out A pointer to a newly created OrtLoraAdapter instance. Must be released with + * OrtApi::ReleaseLoraAdapter. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.20. + */ + ORT_API2_STATUS(CreateLoraAdapter, const ORTCHAR_T* adapter_file_path, _In_ OrtAllocator* allocator, + _Outptr_ OrtLoraAdapter** out); + + /** \brief Create an OrtLoraAdapter + * + * The function copies the bytes from the array and creates an OrtLoraAdapter instance. + * + * + * \param[in] bytes pointer to a valid Lora Adapter format buffer. + * \param[in] num_bytes length of bytes buffer. + * \param[in] allocator optional pointer to a device allocator. If specified + * data is copied to the device at some point before Run() is invoked. If nullptr, data stays on CPU. + * The data would still be copied to device if required by the model at inference time. + * \param[out] out A pointer to a newly created OrtLoraAdapter instance. Must be released with + * OrtApi::ReleaseLoraAdapter. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.20. + */ + ORT_API2_STATUS(CreateLoraAdapterFromArray, _In_ const void* bytes, size_t num_bytes, _In_ OrtAllocator* allocator, + _Outptr_ OrtLoraAdapter** out); + + /** \brief Release an ::OrtLoraAdapter obtained from OrtApi::CreateLoraAdapter + */ + ORT_CLASS_RELEASE(LoraAdapter); + + /** \brief Add the Lora Adapter to the list of active adapters. + * + * The function adds the Lora Adapter to the list of active adapters. The Lora Adapter must be created with + * OrtApi::CreateLoraAdapter or FromArray. The Lora Adapter will be used by the session to run the model. + * The instance of the OrtRunOptions can then be used to customize the Run() calls. + * More than one OrtLoraAdapter can be active at the same time. Lora Parameters that belong to different + * Lora adapters that will be active at the same time must not overlap. + * This setting does not affect RunWithBinding. + * + * \param[in] options OrtRunOptions instance + * \param[in] adapter OrtLoraAdapter instance + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.20. + */ + ORT_API2_STATUS(RunOptionsAddActiveLoraAdapter, _Inout_ OrtRunOptions* options, _In_ const OrtLoraAdapter* adapter); + + /// @} + /// \name OrtEpDynamicOptions + /// @{ + + /** \brief Set DynamicOptions for EPs (Execution Providers) + * + * Valid options can be found in `include\onnxruntime\core\session\onnxruntime_session_options_config_keys.h` + * Look for `kOrtEpDynamicOptions` + * + * \param[in] sess OrtSession + * \param[in] keys Array of null terminated UTF8 encoded strings of EP dynamic option keys + * \param[in] values Array of null terminated UTF8 encoded string of EP dynamic option values + * \param[in] kv_len Number of elements in the keys and values arrays + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.20. + */ + ORT_API2_STATUS(SetEpDynamicOptions, _Inout_ OrtSession* sess, _In_reads_(kv_len) const char* const* keys, + _In_reads_(kv_len) const char* const* values, _In_ size_t kv_len); + + /// @} + + /** \brief Release an OrtValueInfo instance if it was not added to an OrtGraph. + * \since Version 1.22. + */ + ORT_CLASS_RELEASE(ValueInfo); + + /** \brief Release an OrtNode if it was not added to an OrtGraph. + * \since Version 1.22. + */ + ORT_CLASS_RELEASE(Node); + + /** \brief Release an OrtGraph. + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.22. + */ + ORT_CLASS_RELEASE(Graph); + + /** \brief Release an OrtModel. + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.22. + */ + ORT_CLASS_RELEASE(Model); + + /** \brief Get the value name from an OrtValueInfo instance. + * \param[in] value_info The OrtValueInfo instance. + * \param[out] name The name of the OrtValueInfo + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.22. + */ + ORT_API2_STATUS(GetValueInfoName, _In_ const OrtValueInfo* value_info, _Out_ const char** name); + + /** \brief Get the type information from an OrtValueInfo instance. + * \param[in] value_info The OrtValueInfo instance. + * \param[out] type_info The type info of the OrtValueInfo + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.22. + */ + ORT_API2_STATUS(GetValueInfoTypeInfo, _In_ const OrtValueInfo* value_info, _Outptr_ const OrtTypeInfo** type_info); + + /** \brief Get the Model Editor API instance + * + * Get the Model Editor API instance to create a new model or augment an existing model. + * + * \return Model Editor API struct + * + * \since Version 1.22. + */ + const OrtModelEditorApi*(ORT_API_CALL* GetModelEditorApi)(void); + + /** \brief Create an OrtValue for a Tensor that uses pre-existing memory. + * + * ORT will take ownership of the memory and free it using the provided deleter when no longer in use. + * + * \param[in] deleter OrtAllocator instance that will be used to free the memory. + * Only the OrtAllocator:Info and OrtAllocator::Release functions are required. + * The OrtMemoryInfo returned by OrtAllocator::Info must match the location of p_data. + * \param[in] p_data Pointer to the memory that will be used by the Tensor. ORT will take ownership of the memory. + * \param[in] p_data_len Length of the memory in bytes. + * \param[in] shape Dimensions of the Tensor. All values should be > 0. + * \param[in] shape_len Number of dimensions in the shape array. + * \param[in] type Data type of the Tensor. + * \param[out] out Newly created ::OrtValue. Must be freed with OrtApi::ReleaseValue + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(CreateTensorWithDataAndDeleterAsOrtValue, _In_ OrtAllocator* deleter, + _In_ void* p_data, size_t p_data_len, + _In_ const int64_t* shape, size_t shape_len, + ONNXTensorElementDataType type, + _Outptr_ OrtValue** out); + + /** \brief sets load cancellation flag to abort session loading process. + * + * \param[in] options instance that was passed to the session at creation time. + * \param[in] cancel setting this to true after model loading process was initiated will + * attempt to cancel the loading process. If cancellation is successful, CreateSession() + * CreateSessionFromArray() or any other session creation API that take session options as an + * argument will return an OrtStatus indicating that session loading was canceled at user request, + * error code ORT_MODEL_LOAD_CANCELED. + * The APIs above would not return any valid Session instance. This is the best case effort and the result + * is not guaranteed. The session may have already been created and initialized + * before the cancellation request was issued. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(SessionOptionsSetLoadCancellationFlag, _Inout_ OrtSessionOptions* options, + _In_ bool cancel); + + /** \brief Get the Compile API instance. + * + * Get the Compile API instance to compile ONNX models. Execution providers that support compilation fuse a subgraph + * into an EPContext node that wraps a provider-specific binary representation of the subgraph. + * For more details about the EPContext design, refer to: + * \htmlonly + * EPContext design document. + * \endhtmlonly + * + * \return Compile API struct instance. + * + * \since Version 1.22. + */ + const OrtCompileApi*(ORT_API_CALL* GetCompileApi)(void); + + // + // OrtKeyValuePairs + // + + /** \brief Create an OrtKeyValuePairs instance. + * + * \param[out] out A pointer to a newly created OrtKeyValuePairs instance. + * + * \note Must be released by calling ReleaseKeyValuePairs. + * + * \since Version 1.22. + */ + void(ORT_API_CALL* CreateKeyValuePairs)(_Outptr_ OrtKeyValuePairs** out); + + /** \brief Add a key-value pair to the OrtKeyValuePairs instance. + * + * If a pair with the same key already exists, it is overwritten. + * + * \param[in] kvps OrtKeyValuePairs instance. + * \param[in] key Key to be added. + * \param[in] value Value to be added. + * + * \note The `key` and `value` are copied internally. + * + * \since Version 1.22. + */ + + void(ORT_API_CALL* AddKeyValuePair)(_In_ OrtKeyValuePairs* kvps, _In_ const char* key, _In_ const char* value); + + /** \brief Get the value associated with a key in the OrtKeyValuePairs instance. + * + * \param[in] kvps OrtKeyValuePairs instance. + * \param[in] key Key to be searched. + * + * \return The value associated with the key, or nullptr if the key does not exist. + * + * \since Version 1.22. + */ + const char*(ORT_API_CALL* GetKeyValue)(_In_ const OrtKeyValuePairs* kvps, _In_ const char* key); + + /** \brief Get all the key-value pairs from the OrtKeyValuePairs instance. + * + * \param[in] kvps OrtKeyValuePairs instance. + * \param[out] keys Array of keys from `kvps`. + * \param[out] values Array of values from `kvps`. + * \param[out] num_entries Number of entries in `keys` and `values`. + * + * \since Version 1.22. + */ + void(ORT_API_CALL* GetKeyValuePairs)(_In_ const OrtKeyValuePairs* kvps, + _Outptr_ const char* const** keys, _Outptr_ const char* const** values, + _Out_ size_t* num_entries); + + /** \brief Remove a key-value pair from the OrtKeyValuePairs instance. + * + * \param[in] kvps OrtKeyValuePairs instance. + * \param[in] key Key to be removed. No error if not found. + * + * \since Version 1.22. + */ + void(ORT_API_CALL* RemoveKeyValuePair)(_In_ OrtKeyValuePairs* kvps, _In_ const char* key); + + /** \brief Release an OrtKeyValuePairs instance. + * + * \param[in] input OrtKeyValuePairs instance to be released. + * + * \since Version 1.22. + */ + ORT_CLASS_RELEASE(KeyValuePairs); + + /** \brief Register an execution provider library with ORT. + * + * The library must export 'CreateEpFactories' and 'ReleaseEpFactory' functions. + * See OrtEpApi for more details. + * + * \param[in] env The OrtEnv instance to register the library in. + * \param[in] registration_name The name to register the execution provider library under. + * \param[in] path The path to the execution provider library. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(RegisterExecutionProviderLibrary, _In_ OrtEnv* env, _In_ const char* registration_name, + _In_ const ORTCHAR_T* path); + + /** \brief Unregister an execution provider library with ORT. + * + * ORT will call ReleaseEpFactory for all factories created by the library, and unload the library. + * + * You MUST ensure there are no Session instances using execution providers created by the library + * before calling this function. + * + * \param[in] env The OrtEnv instance to unregister the library from. + * \param[in] registration_name The name the execution provider library was registered under. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(UnregisterExecutionProviderLibrary, _In_ OrtEnv* env, _In_ const char* registration_name); + + /** \brief Get the list of available OrtEpDevice instances. + * + * Each OrtEpDevice instance contains details of the execution provider and the device it will use. + * + * \param[in] env The OrtEnv instance to query. + * \param[out] ep_devices The OrtEpDevice instances that the execution provider will use. + * \param[out] num_ep_devices The number of OrtEpDevice instances returned. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(GetEpDevices, _In_ const OrtEnv* env, + _Outptr_ const OrtEpDevice* const** ep_devices, _Out_ size_t* num_ep_devices); + + /** \brief Append the execution provider that is responsible for the selected OrtEpDevice instances + * to the session options. + * + * \param[in] session_options Session options to add execution provider to. + * \param[in] env Environment that execution providers were registered with. + * \param[in] ep_devices One or more OrtEpDevice instances to create an execution provider for. + * Obtain from GetEpDevices. All OrtEpDevice instances must be from the same execution + * provider. It is only necessary to provide multiple OrtEpDevices if you want to use the + * same execution provider for multiple devices. + * e.g. the EP is capable of running on GPU and NPU. + * \param[in] num_ep_devices Number of OrtEpDevice instances. + * \param[in] ep_option_keys Optional keys to configure the execution provider. + * \param[in] ep_option_vals Optional values to configure the execution provider. + * \param[in] num_ep_options Number of execution provide options to add. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_V2, _In_ OrtSessionOptions* session_options, + _In_ OrtEnv* env, + _In_reads_(num_ep_devices) const OrtEpDevice* const* ep_devices, _In_ size_t num_ep_devices, + _In_reads_(num_op_options) const char* const* ep_option_keys, + _In_reads_(num_op_options) const char* const* ep_option_vals, + size_t num_ep_options); + + /** \brief Set the execution provider selection policy for the session. + * + * Allows users to specify a device selection policy for automatic execution provider (EP) selection. + * If custom selection is required please use SessionOptionsSetEpSelectionPolicyDelegate instead. + * + * \param[in] session_options The OrtSessionOptions instance. + * \param[in] policy The device selection policy to use (see OrtExecutionProviderDevicePolicy). + * + * \since Version 1.22 + */ + ORT_API2_STATUS(SessionOptionsSetEpSelectionPolicy, _In_ OrtSessionOptions* session_options, + _In_ OrtExecutionProviderDevicePolicy policy); + + /** \brief Set the execution provider selection policy delegate for the session. + * + * Allows users to provide a custom device selection policy for automatic execution provider (EP) selection. + * + * \param[in] session_options The OrtSessionOptions instance. + * \param[in] delegate Delegate callback for custom selection. + * \param[in] delegate_state Optional state that will be passed to the delegate callback. nullptr if not required. + * + * \since Version 1.22 + */ + ORT_API2_STATUS(SessionOptionsSetEpSelectionPolicyDelegate, _In_ OrtSessionOptions* session_options, + _In_ EpSelectionDelegate delegate, + _In_opt_ void* delegate_state); + + /** \brief Get the hardware device type. + * + * \param[in] device The OrtHardwareDevice instance to query. + * \return The hardware device type. + * + * \since Version 1.22. + */ + OrtHardwareDeviceType(ORT_API_CALL* HardwareDevice_Type)(_In_ const OrtHardwareDevice* device); + + /** \brief Get the hardware device's vendor identifier. + * + * \param[in] device The OrtHardwareDevice instance to query. + * \return The hardware device vendor identifier. + * + * \since Version 1.22. + */ + uint32_t(ORT_API_CALL* HardwareDevice_VendorId)(_In_ const OrtHardwareDevice* device); + + /** \brief Get the hardware device's vendor name. + * + * \param[in] device The OrtHardwareDevice instance to query. + * \return The hardware device's vendor name. + * + * \since Version 1.22. + */ + const char*(ORT_API_CALL* HardwareDevice_Vendor)(_In_ const OrtHardwareDevice* device); + + /** \brief Get the hardware device's unique identifier. + * + * \param[in] device The OrtHardwareDevice instance to query. + * \return The device id. + * + * \note This is not a unique identifier. It identifies the hardware type when combined with vendor id. + * \since Version 1.22. + */ + uint32_t(ORT_API_CALL* HardwareDevice_DeviceId)(_In_ const OrtHardwareDevice* device); + + /** \brief Get hardware device metadata. + * + * \param[in] device The OrtHardwareDevice instance to query. + * \return An OrtKeyValuePairs instance containing the metadata for the device. + * Note: ORT owns the instance so the user must not call ReleaseKeyValuePairs with it. + * + * \since Version 1.22. + */ + const OrtKeyValuePairs*(ORT_API_CALL* HardwareDevice_Metadata)(_In_ const OrtHardwareDevice* device); + + /** \brief Get the execution provider name. + * + * \param[in] ep_device The OrtEpDevice instance to query. + * \return The execution provider name. + * + * \since Version 1.22. + */ + const char*(ORT_API_CALL* EpDevice_EpName)(_In_ const OrtEpDevice* ep_device); + + /** \brief Get the execution provider's vendor name. + * + * \param[in] ep_device The OrtEpDevice instance to query. + * \return The execution provider's vendor name. + * + * \since Version 1.22. + */ + const char*(ORT_API_CALL* EpDevice_EpVendor)(_In_ const OrtEpDevice* ep_device); + + /** \brief Get the metadata for the OrtEpDevice. + * + * \param[in] ep_device The OrtEpDevice instance to query. + * \return An OrtKeyValuePairs instance containing the metadata for the device. + * + * \since Version 1.22. + */ + const OrtKeyValuePairs*(ORT_API_CALL* EpDevice_EpMetadata)(_In_ const OrtEpDevice* ep_device); + + /** \brief Get the execution provider options for the OrtEpDevice. + * + * \param[in] ep_device The OrtEpDevice instance to query. + * \return An OrtKeyValuePairs instance containing the execution provider options for the device. + * + * \since Version 1.22. + */ + const OrtKeyValuePairs*(ORT_API_CALL* EpDevice_EpOptions)(_In_ const OrtEpDevice* ep_device); + + /** \brief Get the OrtHardwareDevice instance for the OrtEpDevice. + * + * \param[in] ep_device The OrtEpDevice instance to query. + * \return The OrtHardwareDevice instance for the device. + * + * \since Version 1.22. + */ + const OrtHardwareDevice*(ORT_API_CALL* EpDevice_Device)(_In_ const OrtEpDevice* ep_device); + + /** \brief Get the OrtEpApi instance for implementing an execution provider. + * + * \since Version 1.22. + */ + const OrtEpApi*(ORT_API_CALL* GetEpApi)(void); + + /** \brief Compute total size in bytes of the tensor data contained in an OrtValue. + * + * Returns the total number of bytes used to store the tensor data. For numeric tensors, + * this is sizeof(element_type) * total_element_count. OrtValues that are not tensors or + * that are tensors that contain strings will cause an error to be returned. + * + * \param[in] ort_value OrtValue instance containing a tensor + * \param[out] size The total size of the tensor data in bytes + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23 + */ + ORT_API2_STATUS(GetTensorSizeInBytes, _In_ const OrtValue* ort_value, _Out_ size_t* size); + + /** \brief Calls OrtAllocator::GetStats function + * + * Return a pointer to the OrtKeyValuePairs structure that contains the statistics of the allocator + * and the user should call OrtApi::ReleaseKeyValuePairs. + * + * NOTE: If the allocator does not implement this function, the OrtKeyValuePairs instance will be empty. + * + * \param[in] ort_allocator The allocator to get stats from + * \param[out] out A pointer to the OrtKeyValuePairs instance that contains the stats + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(AllocatorGetStats, _In_ const OrtAllocator* ort_allocator, _Outptr_ OrtKeyValuePairs** out); + + /** \brief Create an ::OrtMemoryInfo + * + * \param[in] name Arbitrary name. + * \param[in] device_type Device type. + * \param[in] vendor_id PCI Vendor ID. Use 0 for a generic allocator (e.g. WebGPU). + * \param[in] device_id Device ID if there are multiple devices of the same type. e.g. 2 GPU devices. + * \param[in] mem_type Memory type. Use OrtDeviceMemoryType_DEFAULT for device memory, and + * OrtDeviceMemoryType_HOST_ACCESSIBLE (if applicable) for memory used to transfer between the + * device and the CPU. Use the device_type and device_id of the GPU/NPU that the memory is also + * accessible to. + * \param[in] alignment Alignment of the memory if required. Pass 0 for default alignment. + * \param[in] allocator_type Allocator type. If OrtAllocatorType::OrtArenaAllocator, the ORT arena will be used. + * Caveat: Support for OrtArenaAllocator is currently limited to usage of internal ORT + * allocators via CreateAllocator/CreateAndRegisterAllocator/CreateAndRegisterAllocatorV2. + * \param[out] out Newly created ::OrtMemoryInfo. Must be freed with OrtApi::ReleaseMemoryInfo + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23 + */ + ORT_API2_STATUS(CreateMemoryInfo_V2, _In_ const char* name, _In_ enum OrtMemoryInfoDeviceType device_type, + _In_ uint32_t vendor_id, _In_ int32_t device_id, _In_ enum OrtDeviceMemoryType mem_type, + _In_ size_t alignment, enum OrtAllocatorType allocator_type, + _Outptr_ OrtMemoryInfo** out); + + /** \brief Get the device memory type from ::OrtMemoryInfo + * + * \param[in] ptr The OrtMemoryInfo instance to query. + * \return The device memory type. + * + * \since Version 1.23 + */ + ORT_API_T(OrtDeviceMemoryType, MemoryInfoGetDeviceMemType, _In_ const OrtMemoryInfo* ptr); + + /** \brief Get the vendor id from ::OrtMemoryInfo + * + * \param[in] ptr The OrtMemoryInfo instance to query. + * \return The vendor id. + * + * \since Version 1.23 + */ + ORT_API_T(uint32_t, MemoryInfoGetVendorId, _In_ const OrtMemoryInfo* ptr); + + /// \name OrtValueInfo + /// @{ + + /** \brief Get the OrtNode that produces the value represented by the given OrtValueInfo. + * Optionally returns the associated output index. + * + * \param[in] value_info The OrtValueInfo instance. + * \param[out] producer_node Output parameter set to the OrtNode that produces the OrtValueInfo. + * \param[out] producer_output_index Optional output parameter set to the OrtNode instance's output index + * that produces the value. Ignored if set to NULL. + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.23. + */ + ORT_API2_STATUS(ValueInfo_GetValueProducer, _In_ const OrtValueInfo* value_info, + _Outptr_ const OrtNode** producer_node, _Out_opt_ size_t* producer_output_index); + + /** \brief Get the number of consumers of a value as a node input. + * + * Only nodes are considered "consumers" by this function. To check if an OrtValueInfo is a graph output, + * call ValueInfo_IsGraphOutput(). + * + * A single OrtNode may use a single value for more than one input (e.g., Mul(x, x)), so the returned + * `num_consumers` may be larger than the number of unique OrtNode instances that consume the value. + * + * \param[in] value_info The OrtValueInfo instance. + * \param[out] num_consumers Output parameter set to the number of consumers of the value. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(ValueInfo_GetValueNumConsumers, _In_ const OrtValueInfo* value_info, _Out_ size_t* num_consumers); + + /** \brief Returns information (OrtNode and input index) for all consumer nodes that use the value as an input. + * + * Only nodes are considered "consumers" by this function. + * + * Caller provides 2 pre-allocated arrays that will be filled with the OrtNode and input index values. + * Use ValueInfo_GetValueNumConsumers() to get the number of consumers of the value. + * + * An OrtNode instance may appear multiple times if it uses the given value more than once. + * Example: For a node MulNode(x, x) that consumes the value 'x' twice, the following is returned: + * - nodes: [MulNode, MulNode] + * - input_indices: [0, 1] + * + * \param[in] value_info The OrtValueInfo instance. + * \param[out] nodes Pre-allocated array of size `num_consumers` that is filled with OrtNode instances. + * \param[out] input_indices Pre-allocated array of `num_consumers` elements that is filled + * with input indices. Index is set to -1 for an "implicit" input to a consumer node + * that contains a subgraph (e.g., If, Loop) with nodes that use the value internally. + * \param[in] num_consumers The size of the `consumer_nodes` and `consumer_input_indices` arrays. + * Typical usage sets this to the value of ValueInfo_GetValueNumConsumers(). + * An error status is returned if `num_consumers` is less than the number of actual + * consumers. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(ValueInfo_GetValueConsumers, _In_ const OrtValueInfo* value_info, + _Out_writes_all_(num_consumers) const OrtNode** nodes, + _Out_writes_all_(num_consumers) int64_t* input_indices, + _In_ size_t num_consumers); + + /** \brief Get the underlying initializer value, as an OrtValue, from the given OrtValueInfo. + * + * Sets the output parameter to NULL if the given OrtValueInfo does not represent an initializer. + * Does not return an error status in this case. + * + * Supports initializers defined in an outer scope (i.e., a parent graph). + * + * Supports initializers stored in an external file. For external initializers, ORT memory maps + * the initializer data on the first call to this function. If caller needs custom memory mapping, + * use ValueInfo_GetExternalInitializerInfo to get the location of the initializer data. + * + * \param[in] value_info The OrtValueInfo instance. + * \param[out] initializer_value Output parameter set to the initializer value or NULL. Do not cache the OrtValue + * as it is released when the owning OrtGraph is released. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(ValueInfo_GetInitializerValue, _In_ const OrtValueInfo* value_info, + _Outptr_ const OrtValue** initializer_value); + + /** \brief Get information about an external initializer (e.g., filepath, file offset, byte size). + * + * Sets the output parameter `info` to NULL if the given OrtValueInfo does not represent an initializer + * with external data. In this case, a NULL status (non-error) is returned. + * + * \param[in] value_info The OrtValueInfo instance. + * \param[out] info Output parameter set to an OrtExternalInitializerInfo instance that can be used to query + * file path, file offset, etc. ORT sets this to NULL if the OrtValueInfo does not represent + * an external initializer. + * Must release with ReleaseExternalInitializerInfo. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(ValueInfo_GetExternalInitializerInfo, _In_ const OrtValueInfo* value_info, + _Outptr_result_maybenull_ OrtExternalInitializerInfo** info); + + /** \brief Returns a boolean indicating if the given value is a required graph input. + * + * For ONNX IR version < 4, all graph inputs without a matching initializer are required. + * + * For ONNX IR version >=4, a graph input with a matching initializer is an optional graph input + * with the initializer serving as the default value. + * + * \param[in] value_info The OrtValueInfo instance representing the graph value. + * \param[out] is_required_graph_input Output parameter set to true if the graph value is a required graph input. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(ValueInfo_IsRequiredGraphInput, _In_ const OrtValueInfo* value_info, + _Out_ bool* is_required_graph_input); + + /** \brief Returns a boolean indicating if the given value is an optional graph input. + * + * Optional graph inputs were introduced in ONNX IR version 4. For ONNX IR version >=4, a graph input with a + * matching initializer is an optional graph input with the initializer serving as the default value. + * The matching initializer is also known as a non-constant initializer. + * + * \param[in] value_info The OrtValueInfo instance representing the graph value. + * \param[out] is_optional_graph_input Output parameter set to true if the graph value is an optional graph input. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(ValueInfo_IsOptionalGraphInput, _In_ const OrtValueInfo* value_info, + _Out_ bool* is_optional_graph_input); + + /** \brief Returns a boolean indicating if the given value is a graph output. + * + * \param[in] value_info The OrtValueInfo instance representing the graph value. + * \param[out] is_graph_output Output parameter set to true if the graph value is a graph output. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(ValueInfo_IsGraphOutput, _In_ const OrtValueInfo* value_info, _Out_ bool* is_graph_output); + + /** \brief Returns a boolean indicating if the given value is a constant initializer. + * + * For ONNX IR version < 4, all initializers are constant. + * + * For ONNX IR version >=4, an initializer that serves as the default value for a matching graph input is not a + * constant initializer. + * + * \param[in] value_info The OrtValueInfo instance representing the graph value. + * \param[out] is_constant_initializer Output parameter set to true if the graph value is a constant initializer. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(ValueInfo_IsConstantInitializer, _In_ const OrtValueInfo* value_info, + _Out_ bool* is_constant_initializer); + + /** \brief Returns a boolean indicating if the given value is defined in an outer scope. + * + * Certain operator types (e.g., If and Loop) contain nested subgraphs. This function enables + * determining whether a value is defined in a parent node's graph. + * + * \param[in] value_info The OrtValueInfo instance representing the graph value. + * \param[out] is_from_outer_scope Output parameter set to true if the value is defined in an outer + * scope (i.e., a parent graph). + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(ValueInfo_IsFromOuterScope, _In_ const OrtValueInfo* value_info, + _Out_ bool* is_from_outer_scope); + + /// @} + + /// \name OrtGraph + /// @{ + + /** \brief Returns a graph's name. + * + * \param[in] graph The OrtGraph instance. + * \param[out] graph_name Output parameter set to the graph's name. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Graph_GetName, _In_ const OrtGraph* graph, _Outptr_ const char** graph_name); + + /** \brief Get the filepath to the model from which an OrtGraph is constructed. + * + * \note The model's filepath is empty if the filepath is unknown, such as when the model is loaded from bytes + * via CreateSessionFromArray. + * + * \param[in] graph The OrtGraph instance. + * \param[out] model_path Output parameter set to the model's null-terminated filepath. + * Set to an empty path string if unknown. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Graph_GetModelPath, _In_ const OrtGraph* graph, _Outptr_ const ORTCHAR_T** model_path); + + /** \brief Returns the ONNX IR version. + * + * \param[in] graph The OrtGraph instance. + * \param[out] onnx_ir_version Output parameter set to the ONNX IR version. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Graph_GetOnnxIRVersion, _In_ const OrtGraph* graph, _Out_ int64_t* onnx_ir_version); + + /** \brief Returns the number of operator sets that the graph's model uses. + * + * \note An operator set is uniquely identified by the (domain, opset_version) pair. All models must have at + * least one entry that specifies which entry of the ONNX operator set is used. The ONNX domain is represented by + * an empty string. + * + * \param[in] graph The OrtGraph instance. + * \param[out] num_operator_sets Output parameter set to the number of operator sets that the graph's model uses. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Graph_GetNumOperatorSets, _In_ const OrtGraph* graph, _Out_ size_t* num_operator_sets); + + /** \brief Returns the operator sets that the graph's model uses. + * + * \note An operator set is uniquely identified by the (domain, opset_version) pair. All models must have at + * least one entry that specifies which entry of the ONNX operator set is used. The ONNX domain is represented by + * an empty string. + * + * \param[in] graph The OrtGraph instance. + * \param[out] domains Pre-allocated array of `num_operator_sets` elements that is filled with + * null-terminated domain names. + * \param[out] opset_versions Pre-allocated array of `num_operator_sets` elements that is filled with + * the opset version of the corresponding domain in the `domains` array. + * \param[in] num_operator_sets The size of the `domains` and `opset_versions` arrays. + * Typical usage sets this to the result of Graph_GetNumOperatorSets(). + * An error status is returned if `num_operator_sets` is less than the actual number + * of operator sets. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Graph_GetOperatorSets, _In_ const OrtGraph* graph, + _Out_writes_(num_operator_sets) const char** domains, + _Out_writes_(num_operator_sets) int64_t* opset_versions, _In_ size_t num_operator_sets); + + /** \brief Returns the number of graph inputs. + * + * \note The count includes initializers that are included in the list of graph inputs. + * + * \param[in] graph The OrtGraph instance. + * \param[out] num_inputs Output parameter set to the number of graph inputs. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Graph_GetNumInputs, _In_ const OrtGraph* graph, _Out_ size_t* num_inputs); + + /** \brief Returns the graph's inputs as OrtValueInfo instances. + * + * \note The result includes initializers that are included in the list of graph inputs. + * + * \param[in] graph The OrtGraph instance. + * \param[out] inputs Pre-allocated array of `num_inputs` elements that is filled with the graph's inputs. + * \param[in] num_inputs The size of the `inputs` array. + * Typical usage sets this to the result of Graph_GetNumInputs(). An error status is + * returned if `num_inputs` is less than the number of graph inputs. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Graph_GetInputs, _In_ const OrtGraph* graph, + _Out_writes_(num_inputs) const OrtValueInfo** inputs, _In_ size_t num_inputs); + + /** \brief Returns the number of graph outputs. + * + * \param[in] graph The OrtGraph instance. + * \param[out] num_outputs Output parameter set to the number of graph outputs. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Graph_GetNumOutputs, _In_ const OrtGraph* graph, _Out_ size_t* num_outputs); + + /** \brief Returns the graph's outputs as OrtValueInfo instances. + * + * \param[in] graph The OrtGraph instance. + * \param[out] outputs Pre-allocated array of `num_outputs` elements that is filled with the graph's outputs. + * \param[in] num_outputs The size of the `outputs` array. + * Typical usage sets this to the result of Graph_GetNumOutputs(). An error status is + * returned if `num_outputs` is less than the number of graph outputs. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Graph_GetOutputs, _In_ const OrtGraph* graph, + _Out_writes_(num_outputs) const OrtValueInfo** outputs, _In_ size_t num_outputs); + + /** \brief Returns the number of graph initializers. + * + * Counts constant and non-constant initializers. + * + * \param[in] graph The OrtGraph instance. + * \param[out] num_initializers Output parameter set to the number of graph initializers. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Graph_GetNumInitializers, _In_ const OrtGraph* graph, _Out_ size_t* num_initializers); + + /** \brief Returns the graph's initializers as OrtValueInfo instances. + * + * Includes constant and non-constant initializers. + * + * For ONNX IR version < 4, all initializers are constant. + * + * For ONNX IR version >= 4, an initializer with a name that matches a graph input is considered a + * non-constant initializer. + * + * Call ValueInfo_GetInitializerValue to get the initializer's data. + * + * \param[in] graph The OrtGraph instance. + * \param[out] initializers Pre-allocated array of `num_outputs` elements that is filled with the initializers. + * \param[in] num_initializers The size of the `initializers` array. Typical usage sets this to the + * result of Graph_GetNumInitializers(). An error status is returned if + * `num_initializers` is less than the number of graph initializers. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Graph_GetInitializers, _In_ const OrtGraph* graph, + _Out_writes_(num_initializers) const OrtValueInfo** initializers, + _In_ size_t num_initializers); + + /** \brief Returns the number of graph nodes. + * + * \param[in] graph The OrtGraph instance. + * \param[out] num_nodes Output parameter set to the number of graph nodes. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Graph_GetNumNodes, _In_ const OrtGraph* graph, _Out_ size_t* num_nodes); + + /** \brief Returns the graph's nodes as OrtNode instances. + * + * The nodes are sorted using a stable topological ordering. Callers are responsible for maintaining their + * own node ordering if a different order is required. + * + * \param[in] graph The OrtGraph instance. + * \param[out] nodes Pre-allocated array of `num_nodes` elements that is filled with the graph's nodes. + * \param[in] num_nodes The size of the `nodes` array. Typical usage sets this to the + * result of Graph_GetNumNodes(). An error status is returned if + * `num_nodes` is less than the number of graph nodes. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Graph_GetNodes, _In_ const OrtGraph* graph, + _Out_writes_(num_nodes) const OrtNode** nodes, _In_ size_t num_nodes); + + /** \brief Get the parent node for the given graph, if any exists. + * + * Certain operator types (e.g., If and Loop) contain nested subgraphs. This function enables + * access to the parent node (e.g., the If and Loop node) from a nested subgraph. + * + * \param[in] graph The OrtGraph instance. + * \param[out] node Output parameter that is set to the graph's parent node. + * Set to NULL if a parent node does not exist (e.g., for a top-level graph). + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Graph_GetParentNode, _In_ const OrtGraph* graph, _Outptr_result_maybenull_ const OrtNode** node); + + /** \brief Returns an OrtGraph that contains a subset of nodes in the source OrtGraph. + * + * \note The lifetime of "dst_graph" is tied to that of "src_graph", as they both internally reference + * the same underlying graph. "dst_graph" preserves the input order of "src_graph", and + * its output order corresponds to the outputs produced by the nodes in "nodes" with the given order. + * + * \param[in] src_graph The source OrtGraph instance. + * \param[in] nodes A subset of the nodes/OrtNodes in 'graph'. + * \param[in] num_nodes Number of nodes. + * \param[out] dst_graph An OrtGraph created from a given set of nodes. Must be released by calling ReleaseGraph. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Graph_GetGraphView, _In_ const OrtGraph* src_graph, _In_ const OrtNode** nodes, + _In_ size_t num_nodes, _Outptr_ OrtGraph** dst_graph); + + /// @} + + /// \name OrtNode + /// @{ + + /** \brief Returns a node's identifier. + * + * The node's identifier is only unique in the node's parent graph. Different nested subgraphs + * (e.g., subgraphs contained by If and Loop nodes) may reuse identifiers. + * + * \param[in] node The OrtNode instance. + * \param[out] node_id Output parameter set to the node's identifier. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Node_GetId, _In_ const OrtNode* node, _Out_ size_t* node_id); + + /** \brief Returns a node's name. Can be an empty string. + * + * \param[in] node The OrtNode instance. + * \param[out] node_name Output parameter set to the node's name. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Node_GetName, _In_ const OrtNode* node, _Outptr_ const char** node_name); + + /** \brief Returns a node's operator type (e.g., "Conv"). + * + * \param[in] node The OrtNode instance. + * \param[out] operator_type Output parameter set to the name of the node's operator type. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Node_GetOperatorType, _In_ const OrtNode* node, _Outptr_ const char** operator_type); + + /** \brief Returns a node's domain name. + * + * \param[in] node The OrtNode instance. + * \param[out] domain_name Output parameter set to the node's domain name. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Node_GetDomain, _In_ const OrtNode* node, _Outptr_ const char** domain_name); + + /** \brief Get the opset version in which the given node's operator type was first defined. + * + * \param[in] node The OrtNode instance. + * \param[out] since_version The opset version in which the node's operator type was first defined. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Node_GetSinceVersion, _In_ const OrtNode* node, _Out_ int* since_version); + + /** \brief Returns the number of node inputs. + * + * \param[in] node The OrtNode instance. + * \param[out] num_inputs Output parameter set to the number of node inputs. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Node_GetNumInputs, _In_ const OrtNode* node, _Out_ size_t* num_inputs); + + /** \brief Returns the node's inputs as OrtValueInfo instances. + * + * \param[in] node The OrtNode instance. + * \param[out] inputs Pre-allocated array of `num_inputs` elements that is filled with the node's inputs. + * \param[in] num_inputs The size of the `inputs` array. + * Typical usage sets this to the result of Node_GetNumInputs(). An error status is + * returned if `num_inputs` is less than the number of node inputs. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Node_GetInputs, _In_ const OrtNode* node, + _Out_writes_(num_inputs) const OrtValueInfo** inputs, _In_ size_t num_inputs); + + /** \brief Returns the number of node outputs. + * + * \param[in] node The OrtNode instance. + * \param[out] num_outputs Output parameter set to the number of node outputs. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Node_GetNumOutputs, _In_ const OrtNode* node, _Out_ size_t* num_outputs); + + /** \brief Returns the node's outputs as OrtValueInfo instances. + * + * \param[in] node The OrtNode instance. + * \param[out] outputs Pre-allocated array of `num_outputs` elements that is filled with the node's outputs. + * \param[in] num_outputs The size of the `outputs` array. + * Typical usage sets this to the result of Node_GetNumOutputs(). An error status is + * returned if `num_outputs` is less than the number of node outputs. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Node_GetOutputs, _In_ const OrtNode* node, + _Out_writes_(num_outputs) const OrtValueInfo** outputs, _In_ size_t num_outputs); + + /** \brief Returns the number of node implicit inputs. + * + * Certain operator types (e.g., If and Loop) contain nested subgraphs. The internal nodes within the nested subgraphs + * may use values from the outer scope. Those "outer scope" values are considered implicit inputs to the node that + * contains the subgraphs (e.g., the If or Loop node). + * + * \param[in] node The OrtNode instance. + * \param[out] num_implicit_inputs Output parameter set to the number of node implicit inputs. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Node_GetNumImplicitInputs, _In_ const OrtNode* node, _Out_ size_t* num_implicit_inputs); + + /** \brief Get the implicit inputs, as OrtValueInfo instances, that are used within the given node's subgraphs. + * + * \note Only certain operator types (e.g., If and Loop) contain nested subgraphs. + * The internal nodes within the nested subgraphs may use values from the outer scope. Those "outer scope" values + * are considered implicit inputs to the node that contains the subgraphs (e.g., the If or Loop node). + * + * \param[in] node The OrtNode instance. + * \param[out] implicit_inputs Pre-allocated array of `num_implicit_inputs` elements that is filled the node's + * implicit inputs. + * \param[in] num_implicit_inputs The size of the `implicit_inputs` array. Typical usage sets this to the result + * of Node_GetNumImplicitInputs(). An error status is returned if + * `num_implicit_inputs` is less than the number of node implicit inputs. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Node_GetImplicitInputs, _In_ const OrtNode* node, + _Out_writes_(num_implicit_inputs) const OrtValueInfo** implicit_inputs, + _In_ size_t num_implicit_inputs); + + /** \brief Returns the number of node attributes. + * + * \param[in] node The OrtNode instance. + * \param[out] num_attributes Output parameter set to the number of node attributes. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Node_GetNumAttributes, _In_ const OrtNode* node, _Out_ size_t* num_attributes); + + /** \brief Returns a node's attributes as OrtOpAttr instances. + * + * \param[in] node The OrtNode instance. + * \param[out] attributes Pre-allocated array of `num_attributes` elements that is filled with the node's attributes. + * \param[in] num_attributes The size of the `num_attributes` array. + * Typical usage sets this to the result of Node_GetNumAttributes(). An error status is + * returned if `num_attributes` is less than the number of node attributes. + * + * \note ONNX Runtime automatically sets optional (unset) attributes to their default values if the default value + * is a constant expression that does not depend on other tensor/model characteristics. Conv's 'kernel_shape' + * attribute is an example of an optional attribute that does not have a constant default value. This function + * does not provide any unset optional attributes without a constant default value. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Node_GetAttributes, _In_ const OrtNode* node, + _Out_writes_(num_attributes) const OrtOpAttr** attributes, _In_ size_t num_attributes); + + /** \brief Gets the OrtNode's attribute as OrtOpAttr by name. + * + * \param[in] node The OrtNode instance. + * \param[in] attribute_name The name of the attribute + * \param[out] attribute Output parameter set to the OrtOpAttr instance if an attribute by the given name exists. + * For an unset optional attribute, `attribute` is set to NULL and a non-error status is + * returned. For an invalid attribute name, `attribute` is set to NULL and an error status with + * code ORT_NOT_FOUND is returned. + * + * \note ONNX Runtime automatically sets optional (unset) attributes to their default values if the default value + * is a constant expression that does not depend on other tensor/model characteristics. Conv's 'kernel_shape' + * attribute is an example of an optional attribute that does not have a constant default value. This function + * does not provide any unset optional attributes without a constant default value. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Node_GetAttributeByName, _In_ const OrtNode* node, _In_ const char* attribute_name, + _Outptr_result_maybenull_ const OrtOpAttr** attribute); + + /** \brief Get the OrtNode's 'TENSOR' attribute as an OrtValue. + * + * \param[in] attribute The OrtOpAttr instance. + * \param[out] attr_tensor If successful, contains the 'TENSOR' attribute as a newly created OrtValue. + Must be freed with OrtApi::ReleaseValue. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(OpAttr_GetTensorAttributeAsOrtValue, _In_ const OrtOpAttr* attribute, + _Outptr_result_maybenull_ OrtValue** attr_tensor); + + /** \brief Get the attribute type as OrtOpAttrType from an OrtOpAttr. + * + * \param[in] attribute The OrtOpAttr instance. + * \param[out] type Output the attribute type as OrtOpAttrType. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(OpAttr_GetType, _In_ const OrtOpAttr* attribute, _Out_ OrtOpAttrType* type); + + /** \brief Get the attribute name from an OrtOpAttr. + * + * \param[in] attribute The OrtOpAttr instance. + * \param[out] name Output parameter set to the attribute's name. The name is a null-terminated string. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(OpAttr_GetName, _In_ const OrtOpAttr* attribute, _Outptr_ const char** name); + + /** \brief Returns the number of subgraphs contained by the given node. + * + * \note Only certain operator types (e.g., If and Loop) contain nested subgraphs. + * + * \param[in] node The OrtNode instance. + * \param[out] num_subgraphs Output parameter set to the number of node subgraphs. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Node_GetNumSubgraphs, _In_ const OrtNode* node, _Out_ size_t* num_subgraphs); + + /** \brief Get the subgraphs, as OrtGraph instances, contained by the given node. + * + * \note Only certain operator types (e.g., If and Loop) contain nested subgraphs. ONNX nodes store subgraphs in + * their attributes, however, this function must be used to obtain subgraphs from an OrtNode. + * + * \param[in] node The OrtNode instance. + * \param[out] subgraphs Pre-allocated array of `num_subgraphs` elements that is filled with the node's subgraphs. + * \param[in] num_subgraphs The size of the `num_subgraphs` array. + * Typical usage sets this to the result of Node_GetNumSubgraphs(). An error status is + * returned if `num_subgraphs` is less than the number of node subgraphs. + * \param[out] attribute_names Optional pre-allocated array of `num_subgraphs` elements that is filled with the + * attribute names that correspond to the subgraphs. Ignored if set to NULL. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Node_GetSubgraphs, _In_ const OrtNode* node, + _Out_writes_(num_subgraphs) const OrtGraph** subgraphs, _In_ size_t num_subgraphs, + _Out_writes_opt_(num_subgraphs) const char** attribute_names); + + /** \brief Get the node's parent OrtGraph instance. + * + * Can return NULL if the OrtNode was created without an owning graph. + * In another case, this API may also return NULL if `node` is obtained by calling Graph_GetParentNode() + * on an OrtGraph that is a subgraph of a control-flow op, and the parent graph has not been created yet, + * for example during ORT's GetCapability() when processing the innermost subgraph. + * + * \param[in] node The OrtNode instance. + * \param[out] graph Output parameter set to the node's OrtGraph. Can be set to NULL + * if the node is not currently contained by a graph. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Node_GetGraph, _In_ const OrtNode* node, _Outptr_result_maybenull_ const OrtGraph** graph); + + /** \brief Returns the execution provider name that this node is assigned to run on. + * Returns NULL if the node has not been assigned to any execution provider yet. + * For plugin execution providers, the name is the one returned by OrtEp::GetName. + * + * \param[in] node The OrtNode instance. + * \param[out] out Output execution provider type and can be NULL if node has not been assigned. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Node_GetEpName, _In_ const OrtNode* node, _Outptr_result_maybenull_ const char** out); + + /// @} + + /// \name OrtExternalInitializerInfo + /// @{ + + /** \brief Release an OrtExternalInitializerInfo instance. + * + * \param[in] input OrtExternalInitializerInfo instance to be released. + * + * \since Version 1.23. + */ + ORT_CLASS_RELEASE(ExternalInitializerInfo); + + /** \brief Get the relative path to the file that stores the initializer's data. + * + * \note The path is relative to the filesystem directory where the ONNX model was stored. + * Caller can use Graph_GetModelPath to get the model's full path and construct the absolute path to the + * external initializer file if necessary. + * + * \param[in] info The OrtExternalInitializerInfo instance. + * \return The relative path to the file that stores the initializer's data. Do NOT free this pointer. + * + * \since Version 1.23. + */ + ORT_API_T(const ORTCHAR_T*, ExternalInitializerInfo_GetFilePath, _In_ const OrtExternalInitializerInfo* info); + + /** \brief Get the byte offset within the file where the initializer's data is stored. + * + * \param[in] info The OrtExternalInitializerInfo instance. + * \return The byte offset where the initializer's data is stored within the file. + * + * \since Version 1.23. + */ + ORT_API_T(int64_t, ExternalInitializerInfo_GetFileOffset, _In_ const OrtExternalInitializerInfo* info); + + /** \brief Get the size in bytes of the initializer's data within the file. + * + * \param[in] info The OrtExternalInitializerInfo instance. + * \return The size in bytes of the initializer's data within the file. + * + * \since Version 1.23. + */ + ORT_API_T(size_t, ExternalInitializerInfo_GetByteSize, _In_ const OrtExternalInitializerInfo* info); + + /// @} + + /// \name OrtRunOptions + /// @{ + + /** \brief Get a run configuration entry. + * + * If a run configuration entry with key `config_key` doesn't exist, `config_value` will be set to NULL. + * + * `config_key`s are defined in onnxruntime_run_options_config_keys.h. + * + * \param[in] options The OrtRunOptions instance. + * \param[in] config_key The configuration entry key. A null-terminated string. + * \return The configuration entry value. Either a null-terminated string if the entry was found. nullptr otherwise. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23 + */ + ORT_API_T(const char*, GetRunConfigEntry, _In_ const OrtRunOptions* options, + _In_z_ const char* config_key); + + /// @} + + /** \brief Get the OrtMemoryInfo for the device. + * + * \param[in] ep_device The OrtEpDevice instance to query. + * \param[in] memory_type The memory type to return. + * \return A pointer to the OrtMemoryInfo for the device. This may be nullptr if not set. + * If memory_type is OrtDeviceMemoryType_DEFAULT and nullptr is returned the EP uses CPU memory. + * + * \since Version 1.23 + */ + ORT_API_T(const OrtMemoryInfo*, EpDevice_MemoryInfo, _In_ const OrtEpDevice* ep_device, + _In_ OrtDeviceMemoryType memory_type); + + /** \brief Create/replace a shared allocator for the OrtEpDevice in the OrtEnv. + * + * OrtEpDevice maps to the EP factory, and the factory provides the allocator implementation. + * + * Both OrtDeviceMemoryType_DEFAULT and OrtDeviceMemoryType_HOST_ACCESSIBLE are optional for an EP to provide. + * It is EP implementation dependent as to what is available. + * + * If a shared allocator already exists for the OrtEpDevice and OrtDeviceMemoryType, it is replaced. This allows + * changing the shared allocator configuration from the default. e.g. adding an arena. + * + * \param[in] env The OrtEnv instance to create the shared allocator in. + * \param[in] ep_device The OrtEpDevice instance to create the shared allocator for. + * \param[in] mem_type The memory type to use for the shared allocator. + * \param[in] allocator_type The type of allocator to create. Only OrtDeviceAllocator is valid currently. + * \param[in] allocator_options Optional key-value pairs to configure the allocator. If arena based, see + * include/onnxruntime/core/framework/allocator.h for the keys and values that can be + * used. + * \param[out] allocator A pointer to the created shared allocator. Owned by the OrtEnv instance. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23 + */ + ORT_API2_STATUS(CreateSharedAllocator, _In_ OrtEnv* env, _In_ const OrtEpDevice* ep_device, + _In_ OrtDeviceMemoryType mem_type, _In_ OrtAllocatorType allocator_type, + _In_opt_ const OrtKeyValuePairs* allocator_options, + _Outptr_opt_ OrtAllocator** allocator); + + /** \brief Get a shared allocator from the OrtEnv. + * + * By default there is a shared allocator created for all OrtEpDevice instances, so if you get the OrtMemoryInfo + * from the OrtEpDevice using EpDevice_MemoryInfo a shared allocator is guaranteed to exist. + * + * This will also match and return custom allocators added with RegisterAllocator. + * + * It is not an error to not find a matching allocator. + * + * \param[in] env The OrtEnv instance to get the shared allocator from. + * \param[in] mem_info The OrtMemoryInfo instance to get the shared allocator for. + * \param[out] allocator A pointer to the shared allocator, or nullptr if no shared allocator exists for + * the given memory info. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23 + */ + ORT_API2_STATUS(GetSharedAllocator, _In_ OrtEnv* env, _In_ const OrtMemoryInfo* mem_info, + _Outptr_result_maybenull_ OrtAllocator** allocator); + + /** \brief Release a shared allocator from the OrtEnv for the OrtEpDevice and memory type. + * + * This will release the shared allocator for the given OrtEpDevice and memory type. + * If no shared allocator exists, this is a no-op. + * + * \param[in] env The OrtEnv instance to release the shared allocator from. + * \param[in] ep_device The OrtEpDevice instance to release the shared allocator for. + * \param[in] mem_type The memory type of the shared allocator to release. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23 + */ + ORT_API2_STATUS(ReleaseSharedAllocator, _In_ OrtEnv* env, _In_ const OrtEpDevice* ep_device, + _In_ OrtDeviceMemoryType mem_type); + + /** \brief Get a const pointer to the raw data inside a tensor + * + * Used to read the internal tensor data directly. + * \note The returned pointer is valid until the OrtValue is destroyed. + * + * \param[in] value A tensor type (string tensors are not supported) + * \param[out] out Filled in with a pointer to the internal storage + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23 + */ + ORT_API2_STATUS(GetTensorData, _In_ const OrtValue* value, _Outptr_ const void** out); + + /** \brief Get Session configuration entries. + * + * \param[in] options The session options. + * \param[out] out A pointer to a newly created OrtKeyValuePairs instance. + * + * An OrtKeyValuePairs instance containing all session configuration entries. + * Note: the user should call OrtApi::ReleaseKeyValuePairs. + * + * \since Version 1.23. + */ + ORT_API2_STATUS(GetSessionOptionsConfigEntries, _In_ const OrtSessionOptions* options, _Outptr_ OrtKeyValuePairs** out); + + /** \brief Get the OrtMemoryInfo for each input of the session. + * + * The memory info can be used to determine where the input tensors are required. + * + * The session must be fully initialized before calling this function as the input locations are not known until + * this has occurred. + * + * \param[in] session The OrtSession instance. + * \param[out] inputs_memory_info Pre-allocated array of size `num_inputs` that will be filled with the + * OrtMemoryInfo* value for each input. + * The order is the same as returned by SessionGetInputName. + * \param[in] num_inputs The number of inputs in the session. Must match SessionGetInputCount. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23 + */ + ORT_API2_STATUS(SessionGetMemoryInfoForInputs, _In_ const OrtSession* session, + _Out_writes_(num_inputs) const OrtMemoryInfo** inputs_memory_info, + _In_ size_t num_inputs); + + /** \brief Get the OrtMemoryInfo for each output of the session. + * + * The memory info can be used to determine the device the output tensors are produced on. + * The user can pre-allocate an OrtValue using this information or use IOBinding to keep the data on the device. + * ORT will copy the output to CPU otherwise. + * + * The session must be fully initialized before calling this function as the output locations are not known until + * this has occurred. + * + * \param[in] session The OrtSession instance. + * \param[out] outputs_memory_info Pre-allocated array of size `num_outputs` that will be filled with + * OrtMemoryInfo* values for each output. + * The order is the same as returned by SessionGetOutputName. + * \param[in] num_outputs The number of outputs in the session. Must match SessionGetOutputCount. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23 + */ + ORT_API2_STATUS(SessionGetMemoryInfoForOutputs, _In_ const OrtSession* session, + _Out_writes_(num_outputs) const OrtMemoryInfo** outputs_memory_info, + _In_ size_t num_outputs); + + /** \brief Get the OrtEpDevice (if available) for each input of the session. + * + * An OrtEpDevice will be available if auto EP selection is enabled by calling + * SessionOptionsSetEpSelectionPolicy or SessionOptionsSetEpSelectionPolicyDelegate, + * or if the OrtEpDevice was manually added to the session using SessionOptionsAppendExecutionProvider_V2. + * + * If an OrtEpDevice is not available for the input a nullptr is returned. + * + * The returned OrtEpDevice can be used to create an OrtSyncStream via CreateSyncStreamForEpDevice to asynchronously + * provide input to the inference session Run. + * + * The session must be fully initialized before calling this function as the assigned EPs are not known until + * this has occurred. + * + * \param[in] session The OrtSession instance. + * \param[out] inputs_ep_devices Pre-allocated array of size `num_inputs` that will be filled with + * OrtEpDevice* values for each input. + * The order is the same as returned by SessionGetInputName. + * \param[in] num_inputs The number of inputs in the session. Must match SessionGetInputCount. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23 + */ + ORT_API2_STATUS(SessionGetEpDeviceForInputs, _In_ const OrtSession* session, + _Out_writes_(num_inputs) const OrtEpDevice** inputs_ep_devices, + _In_ size_t num_inputs); + + /** \brief Create an OrtSyncStream for the given OrtEpDevice. + * + * The OrtSyncStream can be used to enable asynchronous operations. + * e.g. async usage of CopyTensors to provide input to an OrtSession Run call. + * + * An error code of ORT_NOT_IMPLEMENTED will be returned if the EP does not support OrtSyncStream. + * + * \param[in] ep_device The OrtEpDevice instance to create the sync stream for. + * \param[in] stream_options Options for OrtSyncStream creation. May be nullptr. + * \param[out] stream Output parameter set to the created OrtSyncStream instance. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23 + */ + ORT_API2_STATUS(CreateSyncStreamForEpDevice, _In_ const OrtEpDevice* ep_device, + _In_opt_ const OrtKeyValuePairs* stream_options, + _Outptr_ OrtSyncStream** stream); + + /** \brief Get the native handle of the sync stream. + * + * This returns the native handle for the stream. e.g. cudaStream_t for CUDA streams. + * + * \param[in] stream The OrtSyncStream instance to get the handle from. + * + * \returns The native handle of the stream. + * + * \since Version 1.23 + */ + ORT_API_T(void*, SyncStream_GetHandle, _In_ OrtSyncStream* stream); + + ORT_CLASS_RELEASE(SyncStream); + + /** \brief Copy OrtValue instances containing Tensors between devices. + * + * The overall copy must be between a single source device and a single destination device. i.e. + * - all src_tensors must have matching OrtMemoryInfo, + * - all dst_tensors must have matching OrtMemoryInfo. + * + * OrtValue instances can be created by: + * - Use GetSharedAllocator to get the shared allocator for the OrtMemoryInfo if you need to allocate memory + * on the device. + * - Use CreateTensorAsOrtValue, CreateTensorWithDataAsOrtValue or CreateTensorWithDataAndDeleterAsOrtValue + * to create an OrtValue containing a tensor depending on whether you have existing data or not, and whether + * you want ORT to free the existing data once it is done with the OrtValue. + * + * \param[in] env The OrtEnv instance to use. The data transfer implementation is provided by an execution provider + * that is registered in this OrtEnv. + * \param[in] src_tensors Array of OrtValue instances containing the source tensors to copy. + * \param[in] dst_tensors Array of OrtValue instances to copy the source tensors to. + * \param[in] stream Optional OrtSyncStream that can be used to perform the copy asynchronously. May be nullptr. + * \param[in] num_tensors The number of tensors to copy. The size of `src_tensors` and `dst_tensors` must match. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23 + */ + ORT_API2_STATUS(CopyTensors, _In_ const OrtEnv* env, + _In_reads_(num_tensors) const OrtValue* const* src_tensors, + _In_reads_(num_tensors) OrtValue* const* dst_tensors, + _In_opt_ OrtSyncStream* stream, + _In_ size_t num_tensors); + + /** \brief Get ::OrtModelMetadata from an ::OrtGraph + * + * \param[in] graph The OrtGraph instance. + * \param[out] out Newly created ::OrtModelMetadata. Must be freed using OrtApi::ReleaseModelMetadata. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Graph_GetModelMetadata, _In_ const OrtGraph* graph, _Outptr_ OrtModelMetadata** out); + + /** \brief Validate a compiled model's compatibility information for one or more EP devices. + * + * \param[in] ep_devices The EP devices to validate against (e.g., from GetEpDevices). + * All devices must belong to the same execution provider. + * \param[in] num_ep_devices The number of EP devices provided. + * \param[in] compatibility_info The compatibility info string produced when the model was compiled. + * \param[out] out_status The resulting compatibility status for the EP devices. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(GetModelCompatibilityForEpDevices, + _In_reads_(num_ep_devices) const OrtEpDevice* const* ep_devices, + _In_ size_t num_ep_devices, + _In_ const char* compatibility_info, + _Out_ OrtCompiledModelCompatibility* out_status); + + /// \name OrtExternalInitializerInfo + /// @{ + + /** \brief Creates an OrtExternalInitializerInfo instance. + * + * \param[in] filepath The relative path to the file that stores the initializer's data. ORT copies this path string. + * \param[in] file_offset The byte offset where the initializer's data is stored within the file. + * \param[in] byte_size The size in bytes of the initializer's data within the file. + * \param[out] out Output parameter set to the new OrtExternalInitializerInfo instance. + * Must be released by calling ReleaseExternalInitializerInfo(). + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(CreateExternalInitializerInfo, _In_ const ORTCHAR_T* filepath, _In_ int64_t file_offset, + _In_ size_t byte_size, _Outptr_ OrtExternalInitializerInfo** out); + + /// @} + /** \brief Fetch whether the tensor has shape information. + * \param[in] info The OrtTensorTypeAndShapeInfo instance. + * \return true if the tensor has shape information, false otherwise. + * + * \since Version 1.24 + */ + ORT_API_T(bool, TensorTypeAndShape_HasShape, _In_ const OrtTensorTypeAndShapeInfo* info); + + /** \brief Get all config entries from ::OrtKernelInfo. + * + * Gets all configuration entries from the ::OrtKernelInfo object as key-value pairs. + * Config entries are set on the ::OrtSessionOptions and are accessible in custom operator kernels. + * + * Used in the CreateKernel callback of an OrtCustomOp to access all session configuration entries + * during kernel construction. + * + * \param[in] info An instance of ::OrtKernelInfo. + * \param[out] out A pointer to a newly created OrtKeyValuePairs instance containing all config entries. + * Note: the user should call OrtApi::ReleaseKeyValuePairs. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.24 + */ + ORT_API2_STATUS(KernelInfo_GetConfigEntries, _In_ const OrtKernelInfo* info, _Outptr_ OrtKeyValuePairs** out); + + /** \brief Get the graph node's operator domain from ::OrtKernelInfo. + * + * If `out` is nullptr, the value of `size` is set to the size of the operator domain + * string (including null-terminator), and a success status is returned. + * + * If the `size` parameter is greater than or equal to the string's size and `out` is not nullptr, + * the value of `size` is set to the true size of the string (including null-terminator), + * the provided memory is filled with the string's contents, and a success status is returned. + * + * If the `size` parameter is less than the actual string's size and `out` + * is not nullptr, the value of `size` is set to the true size of the string + * and a failure status with error code ORT_INVALID_ARGUMENT is returned. + * + * \param[in] info An instance of ::OrtKernelInfo. + * \param[out] out Memory location into which to write the UTF-8 null-terminated string representing the + * operator domain. + * \param[in,out] size Pointer to the size of the `out` buffer. See above comments for details. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.24 + */ + ORT_API2_STATUS(KernelInfo_GetOperatorDomain, _In_ const OrtKernelInfo* info, _Out_opt_ char* out, + _Inout_ size_t* size); + + /** \brief Get the graph node's operator type from ::OrtKernelInfo. + * + * If `out` is nullptr, the value of `size` is set to the size of the operator type + * string (including null-terminator), and a success status is returned. + * + * If the `size` parameter is greater than or equal to the string's size and `out` is not nullptr, + * the value of `size` is set to the true size of the string (including null-terminator), + * the provided memory is filled with the string's contents, and a success status is returned. + * + * If the `size` parameter is less than the actual string's size and `out` + * is not nullptr, the value of `size` is set to the true size of the string + * and a failure status with error code ORT_INVALID_ARGUMENT is returned. + * + * \param[in] info An instance of ::OrtKernelInfo. + * \param[out] out Memory location into which to write the UTF-8 null-terminated string representing the + * operator type. + * \param[in,out] size Pointer to the size of the `out` buffer. See above comments for details. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.24 + */ + ORT_API2_STATUS(KernelInfo_GetOperatorType, _In_ const OrtKernelInfo* info, _Out_opt_ char* out, + _Inout_ size_t* size); + + /** \brief Get the opset version in which the given node's operator type was first defined from ::OrtKernelInfo. + * + * \param[in] info An instance of ::OrtKernelInfo. + * \param[out] since_version The opset version in which the node's operator type was first defined. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.24 + */ + ORT_API2_STATUS(KernelInfo_GetOperatorSinceVersion, _In_ const OrtKernelInfo* info, + _Out_ int* since_version); + + /** \brief Get the EP Interop API instance. + * + * Get the Interop API instance to work with external resources. This API provides functions + * for importing external GPU memory and semaphores for zero-copy sharing between ORT inference + * and other GPU workloads. + * + * \return Interop API struct instance. + * + * \since Version 1.24. + */ + const OrtInteropApi*(ORT_API_CALL* GetInteropApi)(void); + + /** \brief Get the EP device assigned to each session output. + * + * Returns the OrtEpDevice assigned to each output of the session after graph partitioning. + * This allows validation that outputs are placed on the expected device for external resource sharing. + * + * The EP device for each output is determined by which execution provider will produce that output + * during inferencing. This information is useful for: + * - Validating that outputs will be placed on the expected device for external resource sharing + * - Deciding whether to use external memory handles for outputs + * + * \param[in] session The OrtSession instance to query. + * \param[out] outputs_ep_devices An array to be filled with the EP device for each output. + * The array must be allocated by the caller with space for + * OrtEpDevice* values for each output. + * The order is the same as returned by SessionGetOutputName. + * \param[in] num_outputs The number of outputs in the session. Must match SessionGetOutputCount. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24 + */ + ORT_API2_STATUS(SessionGetEpDeviceForOutputs, _In_ const OrtSession* session, + _Out_writes_(num_outputs) const OrtEpDevice** outputs_ep_devices, + _In_ size_t num_outputs); + /** \brief Get the number of available hardware devices. + * + * Returns the count of hardware devices discovered on the system. + * Use this to allocate an array before calling GetHardwareDevices(). + * + * \param[in] env The OrtEnv instance where device discovery results are stored. + * \param[out] num_devices The number of OrtHardwareDevice instances available. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(GetNumHardwareDevices, _In_ const OrtEnv* env, _Out_ size_t* num_devices); + + /** \brief Get the list of available hardware devices. + * + * Enumerates hardware devices available on the system. + * Populates a user-provided array with pointers to OrtHardwareDevice instances. The caller is responsible + * for allocating the array with sufficient space (use GetNumHardwareDevices() to get the count). + * + * The returned pointers reference internal ORT data structures that are discovered once at process + * startup and remain valid for the lifetime of the OrtEnv. The caller does not need to release these + * pointers, but should not use them after calling ReleaseEnv(). + * + * \param[in] env The OrtEnv instance where device discovery results are stored. + * \param[out] devices User-allocated array to receive pointers to OrtHardwareDevice instances. + * The array must have space for at least num_devices elements. + * \param[in] num_devices The size of the user-allocated devices array. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(GetHardwareDevices, _In_ const OrtEnv* env, + _Out_writes_(num_devices) const OrtHardwareDevice** devices, + _In_ size_t num_devices); + + /** \brief Check for known incompatibility issues between hardware device and a specific execution provider. + * + * This function checks for known incompatibility issues between the specified hardware device + * and a specific execution provider. + * If returned incompatibility details have non-zero reasons, it indicates the device is not compatible. + * However, if returned detail have reason == 0, it doesn't guarantee 100% compatibility for all models, + * as models may have specific requirements. + * + * Note: This method should only be called when the OrtEnv has been initialized with execution + * providers (after RegisterExecutionProviderLibrary is called). + * + * \param[in] env The OrtEnv instance with registered execution providers. + * \param[in] ep_name The name of the execution provider to check. Required and cannot be null or empty. + * \param[in] hw The hardware device to check for incompatibility. + * \param[out] details Compatibility details including reasons for incompatibility if any. + * Must be freed with OrtApi::ReleaseDeviceEpIncompatibilityDetails. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(GetHardwareDeviceEpIncompatibilityDetails, _In_ const OrtEnv* env, + _In_ const char* ep_name, + _In_ const OrtHardwareDevice* hw, + _Outptr_ OrtDeviceEpIncompatibilityDetails** details); + + /// \name OrtDeviceEpIncompatibilityDetails + /// Accessor functions for device incompatibility details + /// @{ + + /** \brief Get the incompatibility reasons bitmask from OrtDeviceEpIncompatibilityDetails. + * + * \param[in] details The OrtDeviceEpIncompatibilityDetails instance to query. + * \param[out] reasons_bitmask Pointer to store the bitmask of incompatibility reasons. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(DeviceEpIncompatibilityDetails_GetReasonsBitmask, + _In_ const OrtDeviceEpIncompatibilityDetails* details, + _Out_ uint32_t* reasons_bitmask); + + /** \brief Get the notes from OrtDeviceEpIncompatibilityDetails. + * + * \param[in] details The OrtDeviceEpIncompatibilityDetails instance to query. + * \param[out] notes Pointer to the notes string. May be nullptr if no notes are available. + * The returned string is owned by the details object and should not be freed. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(DeviceEpIncompatibilityDetails_GetNotes, + _In_ const OrtDeviceEpIncompatibilityDetails* details, + _Outptr_result_maybenull_ const char** notes); + + /** \brief Get the execution provider error code from OrtDeviceEpIncompatibilityDetails. + * + * This allows Independent Hardware Vendors (IHVs) to define their own error codes + * to provide additional details about device incompatibility. + * + * \param[in] details The OrtDeviceEpIncompatibilityDetails instance to query. + * \param[out] error_code Pointer to store the EP-specific error code. A value of 0 indicates no error code was set. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(DeviceEpIncompatibilityDetails_GetErrorCode, + _In_ const OrtDeviceEpIncompatibilityDetails* details, + _Out_ int32_t* error_code); + + /** \brief Release an OrtDeviceEpIncompatibilityDetails instance. + * + * \since Version 1.24. + */ + ORT_CLASS_RELEASE(DeviceEpIncompatibilityDetails); + + /// @} + + /// \name Model Compatibility APIs + /// @{ + + /** \brief Extract EP compatibility info from a precompiled model file. + * + * Parses the model file to extract the compatibility info string for a specific execution provider + * from the model's metadata properties. This is only applicable to models that have been precompiled + * for an EP (e.g., via OrtCompileApi). Standard ONNX models do not contain this information. + * + * The compatibility info string must be valid UTF-8 without embedded NUL characters. + * + * \note This API performs standalone model parsing, separate from session creation. This means + * the protobuf parsing cost is incurred here and again during session creation. It is intended + * for scenarios where applications need to check compatibility before deciding whether to proceed + * with session creation, such as providing early user feedback. + * + * \note This operation parses the full ONNX ModelProto from disk. For very large models, consider + * using GetCompatibilityInfoFromModelBytes with a pre-loaded buffer if the model is already in memory. + * + * The compatibility info can then be passed to GetModelCompatibilityForEpDevices to check if a + * precompiled model is compatible with the current system. + * + * \param[in] model_path Path to the ONNX model file. + * \param[in] ep_type The execution provider type string. Must be non-empty. + * Use OrtApi::EpDevice_EpName to get this value from an OrtEpDevice. + * \param[in] allocator Allocator to use for the output string. Use OrtApi::GetAllocatorWithDefaultOptions. + * \param[out] compatibility_info Output pointer to the compatibility info string. + * Returns nullptr if no compatibility info exists for the specified EP. + * Caller must free with OrtApi::AllocatorFree when non-null. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(GetCompatibilityInfoFromModel, + _In_ const ORTCHAR_T* model_path, + _In_ const char* ep_type, + _Inout_ OrtAllocator* allocator, + _Outptr_result_maybenull_ char** compatibility_info); + + /** \brief Extract EP compatibility info from precompiled model bytes in memory. + * + * Same as GetCompatibilityInfoFromModel but reads from a memory buffer instead of a file. + * Useful when precompiled models are loaded from encrypted storage, network, or other non-file sources. + * + * \note This API performs standalone model parsing, separate from session creation. This means + * the protobuf parsing cost is incurred here and again during session creation. It is intended + * for scenarios where applications need to check compatibility before deciding whether to proceed + * with session creation, such as providing early user feedback. + * + * \param[in] model_data Pointer to the model data in memory. + * \param[in] model_data_length Size of the model data in bytes. + * \param[in] ep_type The execution provider type string. Must be non-empty. + * \param[in] allocator Allocator to use for the output string. Use OrtApi::GetAllocatorWithDefaultOptions. + * \param[out] compatibility_info Output pointer to the compatibility info string. + * Returns nullptr if no compatibility info exists for the specified EP. + * Caller must free with OrtApi::AllocatorFree when non-null. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(GetCompatibilityInfoFromModelBytes, + _In_reads_(model_data_length) const void* model_data, + _In_ size_t model_data_length, + _In_ const char* ep_type, + _Inout_ OrtAllocator* allocator, + _Outptr_result_maybenull_ char** compatibility_info); + + /// @} + + /** \brief Create an OrtEnv instance with the given options. + * + * \note Invoking this function will return the same instance of the environment as that returned by a previous call + * to another env creation function; all arguments to this function will be ignored. + * + * \param[in] options The OrtEnvCreationOptions instance that contains creation options. + * \param[out] out Output parameter set to the new OrtEnv instance. Must be freed with OrtApi::ReleaseEnv. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24 + */ + ORT_API2_STATUS(CreateEnvWithOptions, _In_ const OrtEnvCreationOptions* options, _Outptr_ OrtEnv** out); + + /** \brief Get information about the subgraphs assigned to each execution provider (EP) and the nodes within. + * + * Each returned OrtEpAssignedSubgraph instance contains details of the subgraph/nodes assigned to an execution + * provider, including the execution provider's name, and the name, domain, and operator type for every node. + * + * For compiling execution providers, a single OrtEpAssignedSubgraph instance contains information about the + * nodes that are fused and compiled within a single subgraph assigned to the execution provider. + * + * For execution providers that use kernel registration (e.g., CPU EP), each node with a registered kernel is + * contained in its own OrtEpAssignedSubgraph instance. + * + * \note The caller must enable the collection of this information by enabling the session + * configuration entry "session.record_ep_graph_assignment_info" during session creation. + * Refer to onnxruntime_session_options_config_keys.h. Otherwise, if not enabled, this function returns a + * status with error code ORT_FAIL. + * + * \note The information reported by this function is obtained immediately after running basic optimizations on the + * original graph if the session optimization level is set to ORT_ENABLE_BASIC or higher. If the session + * optimization level is set to ORT_DISABLE_ALL, only minimal/required optimizations are run before + * the information is collected. + * + * \param[in] session The OrtSession instance. + * \param[out] ep_subgraphs Output parameter set to the array of OrtEpAssignedSubgraph instances. + * \param[out] num_ep_subgraphs Output parameter set to the number of elements in the `ep_subgraphs` array. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(Session_GetEpGraphAssignmentInfo, _In_ const OrtSession* session, + _Outptr_ const OrtEpAssignedSubgraph* const** ep_subgraphs, + _Out_ size_t* num_ep_subgraphs); + + /** \brief Get the name of the execution provider to which the subgraph was assigned. + * + * \param[in] ep_subgraph The OrtEpAssignedSubgraph instance. + * \param[out] out Output parameter set to the execution provider's name as a UTF-8 null-terminated string. + * Owned by the OrtEpAssignedSubgraph instance (do not free). + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(EpAssignedSubgraph_GetEpName, _In_ const OrtEpAssignedSubgraph* ep_subgraph, + _Outptr_ const char** out); + + /** \brief Get the nodes in a subgraph assigned to a specific execution provider. + * + * \param[in] ep_subgraph The OrtEpAssignedSubgraph instance. + * \param[out] ep_nodes Output parameter set to the array of OrtEpAssignedNode instances. + * \param[out] num_ep_nodes Output parameter set to the number of OrtEpAssignedNode instance returned. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(EpAssignedSubgraph_GetNodes, _In_ const OrtEpAssignedSubgraph* ep_subgraph, + _Outptr_ const OrtEpAssignedNode* const** ep_nodes, _Out_ size_t* num_ep_nodes); + + /** \brief Get the name of the node assigned to an execution provider. + * + * \param[in] ep_node The OrtEpAssignedNode instance. + * \param[out] out Output parameter set to the node's name as a UTF-8 null-terminated string. + * Owned by the OrtEpAssignedNode instance (do not free). + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(EpAssignedNode_GetName, _In_ const OrtEpAssignedNode* ep_node, _Outptr_ const char** out); + + /** \brief Get the domain of the node assigned to an execution provider. + * + * \param[in] ep_node The OrtEpAssignedNode instance. + * \param[out] out Output parameter set to the node's domain as a UTF-8 null-terminated string. + * Owned by the OrtEpAssignedNode instance (do not free). + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(EpAssignedNode_GetDomain, _In_ const OrtEpAssignedNode* ep_node, _Outptr_ const char** out); + + /** \brief Get the operator type of the node assigned to an execution provider. + * + * \param[in] ep_node The OrtEpAssignedNode instance. + * \param[out] out Output parameter set to the node's operator type as a UTF-8 null-terminated string. + * Owned by the OrtEpAssignedNode instance (do not free). + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(EpAssignedNode_GetOperatorType, _In_ const OrtEpAssignedNode* ep_node, _Outptr_ const char** out); + + /** \brief Sets OrtSyncStream for the run options + * + * OrtSyncStream is used to synchronize the execution of the model run for the device + * of the stream. It overrides the existing stream for the duration of the Run(). + * The stream instance must be alive for the duration of the Run() call. + * + * \param[in] options + * \param[in] sync_stream The synchronization stream. Pass nullptr to clear previous setting. + * + * \since 1.24 + */ + ORT_API_T(void, RunOptionsSetSyncStream, _Inout_ OrtRunOptions* options, _In_ OrtSyncStream* sync_stream); + + /** \brief Get the element data type and shape for an OrtValue that represents a Tensor (scalar, dense, or sparse). + * + * \note This function is an alternative to ::GetTensorTypeAndShape() that does not allocate a new array for + * the shape data. The OrtValue instance's internal shape data is returned directly. + * + * \note Returns an error if the underlying OrtValue is not a Tensor. + * + * \param[in] value The OrtValue instance. + * \param[out] elem_type Output parameter set to the tensor element data type. + * \param[out] shape_data Output parameter set to the OrtValue instance's internal shape data array. + * For a scalar, `shape_data` is NULL and `shape_data_count` is 0. + * Must not be released as it is owned by the OrtValue instance. This pointer becomes invalid + * when the OrtValue is released or if the underlying shape data is updated or reallocated. + * \param[out] shape_data_count Output parameter set to the number of elements in `shape_data`. + * `shape_data_count` is 0 for a scalar. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(GetTensorElementTypeAndShapeDataReference, _In_ const OrtValue* value, + _Out_ ONNXTensorElementDataType* elem_type, + _Outptr_result_maybenull_ const int64_t** shape_data, + _Out_ size_t* shape_data_count); +}; + +/* + * Steps to use a custom op: + * 1 Create an OrtCustomOpDomain with the domain name used by the custom ops + * 2 Create an OrtCustomOp structure for each op and add them to the domain + * 3 Call OrtAddCustomOpDomain to add the custom domain of ops to the session options + */ + +// Specifies some characteristics of inputs/outputs of custom ops: +// Specify if the inputs/outputs are one of: +// 1) Non-optional (input/output must be present in the node) +// 2) Optional (input/output may be absent in the node) +// 3) Variadic: A variadic input or output specifies N (i.e., the minimum arity) or more operands. +// Only the last input or output of a custom op may be marked as variadic. +// The homogeneity of the variadic input or output determines whether all operands must be of the same +// tensor element type. +typedef enum OrtCustomOpInputOutputCharacteristic { + INPUT_OUTPUT_REQUIRED = 0, + INPUT_OUTPUT_OPTIONAL, + INPUT_OUTPUT_VARIADIC, +} OrtCustomOpInputOutputCharacteristic; + +/* + * The OrtCustomOp structure defines a custom op's schema and its kernel callbacks. The callbacks are filled in by + * the implementor of the custom op. + */ +struct OrtCustomOp { + uint32_t version; // Must be initialized to ORT_API_VERSION + + // This callback creates the kernel, which is a user defined + // parameter that is passed to the Kernel* callbacks below. It is + // recommended to use CreateKernelV2 which allows for a safe error + // propagation by returning an OrtStatusPtr. + void*(ORT_API_CALL* CreateKernel)(_In_ const struct OrtCustomOp* op, _In_ const OrtApi* api, + _In_ const OrtKernelInfo* info); + + // Returns the name of the op + const char*(ORT_API_CALL* GetName)(_In_ const struct OrtCustomOp* op); + + // Returns the type of the execution provider, return nullptr to use CPU execution provider + const char*(ORT_API_CALL* GetExecutionProviderType)(_In_ const struct OrtCustomOp* op); + + // Returns the count and types of the input & output tensors + ONNXTensorElementDataType(ORT_API_CALL* GetInputType)(_In_ const struct OrtCustomOp* op, _In_ size_t index); + size_t(ORT_API_CALL* GetInputTypeCount)(_In_ const struct OrtCustomOp* op); + ONNXTensorElementDataType(ORT_API_CALL* GetOutputType)(_In_ const struct OrtCustomOp* op, _In_ size_t index); + size_t(ORT_API_CALL* GetOutputTypeCount)(_In_ const struct OrtCustomOp* op); + + // Perform a computation step. It is recommended to use + // KernelComputeV2 which allows for a safe error propagation by + // returning an OrtStatusPtr. + void(ORT_API_CALL* KernelCompute)(_In_ void* op_kernel, _In_ OrtKernelContext* context); + void(ORT_API_CALL* KernelDestroy)(_In_ void* op_kernel); + + // Returns the characteristics of the input & output tensors + OrtCustomOpInputOutputCharacteristic(ORT_API_CALL* GetInputCharacteristic)(_In_ const struct OrtCustomOp* op, _In_ size_t index); + OrtCustomOpInputOutputCharacteristic(ORT_API_CALL* GetOutputCharacteristic)(_In_ const struct OrtCustomOp* op, _In_ size_t index); + + // Returns the memory type of the input tensors. This API allows the custom op + // to place the inputs on specific devices. By default, it returns + // OrtMemTypeDefault, which means the input is placed on the default device for + // the execution provider. If the inputs need to be with different memory types, + // this function can be overridden to return the specific memory types. + OrtMemType(ORT_API_CALL* GetInputMemoryType)(_In_ const struct OrtCustomOp* op, _In_ size_t index); + + // Returns the minimum number of input arguments expected for the variadic input. + // Applicable only for custom ops that have a variadic input. + int(ORT_API_CALL* GetVariadicInputMinArity)(_In_ const struct OrtCustomOp* op); + + // Returns true (non-zero) if all arguments of a variadic input have to be of the same type (homogeneous), + // and false (zero) otherwise. + // Applicable only for custom ops that have a variadic input. + int(ORT_API_CALL* GetVariadicInputHomogeneity)(_In_ const struct OrtCustomOp* op); + + // Returns the minimum number of output values expected for the variadic output. + // Applicable only for custom ops that have a variadic output. + int(ORT_API_CALL* GetVariadicOutputMinArity)(_In_ const struct OrtCustomOp* op); + + // Returns true (non-zero) if all outputs values of a variadic output have to be of the same type (homogeneous), + // and false (zero) otherwise. + // Applicable only for custom ops that have a variadic output. + int(ORT_API_CALL* GetVariadicOutputHomogeneity)(_In_ const struct OrtCustomOp* op); + + // Create the kernel state which is passed to each compute call. + OrtStatusPtr(ORT_API_CALL* CreateKernelV2)(_In_ const struct OrtCustomOp* op, _In_ const OrtApi* api, + _In_ const OrtKernelInfo* info, + _Out_ void** kernel); + + // Perform the computation step. + OrtStatusPtr(ORT_API_CALL* KernelComputeV2)(_In_ void* op_kernel, _In_ OrtKernelContext* context); + + OrtStatusPtr(ORT_API_CALL* InferOutputShapeFn)(_In_ const struct OrtCustomOp* op, _In_ OrtShapeInferContext*); + + // Get start range + int(ORT_API_CALL* GetStartVersion)(_In_ const struct OrtCustomOp* op); + int(ORT_API_CALL* GetEndVersion)(_In_ const struct OrtCustomOp* op); + + // Get the inplace_map that defines which output can reuse which input + // Callers will provide 2 raw int* and pass in their address, this function will fill these 2 arrays + // when return, output (*output_index)[i] may reuse the input (*input_index[i]). + // The return value is the size of these 2 arrays. + // Callers are responsible to delete these 2 arrays after use by calling OrtCustomOp::ReleaseMayInplace(). + size_t(ORT_API_CALL* GetMayInplace)(_Out_ int** input_index, _Out_ int** output_index); + + // Release the pointer input_index and output_index allocated from GetMayInplace() function. + // If GetMayInplace() is defined, this function MUST be defined as well. + void(ORT_API_CALL* ReleaseMayInplace)(_Frees_ptr_opt_ int* input_index, _Frees_ptr_opt_ int* output_index); + + // Same as GetMayInplace() and ReleaseMayInplace() + size_t(ORT_API_CALL* GetAliasMap)(_Out_ int** input_index, _Out_ int** output_index); + void(ORT_API_CALL* ReleaseAliasMap)(_Frees_ptr_opt_ int* input_index, _Frees_ptr_opt_ int* output_index); +}; + +/** + * ORT Model Editor API + */ + +/** + * \brief The OrtModelEditorApi struct provides functions to create or edit an ONNX model. + * + * See onnxruntime/test/shared_lib/test_model_editor_api.cc for example usage. + * + * \since Version 1.22. + */ +struct OrtModelEditorApi { + // Model building/editing requires a full build. We return nullptr from GetModelEditorApi if this is a minimal + // build, so it doesn't matter if there are no function pointers in this struct as a user will never get an + // OrtModelEditorApi instance. We do however need a dummy field to avoid empty struct warning. +#if defined(ORT_MINIMAL_BUILD) + const bool not_defined_in_this_build; +#else + /** \brief Create an OrtTypeInfo instance for a Tensor. + * + * Create an OrtTypeInfo instance for a Tensor to use as graph inputs/outputs with the Model Editor API. + * + * User can release `tensor_info` after creating the OrtTypeInfo. + * + * \param[in] tensor_info Tensor type and shape information. + * \param[out] type_info TypeInfo instance for the tensor. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(CreateTensorTypeInfo, _In_ const OrtTensorTypeAndShapeInfo* tensor_info, + _Outptr_ OrtTypeInfo** type_info); + + /** \brief Create an OrtTypeInfo instance for a SparseTensor. + * + * Create an OrtTypeInfo instance for a SparseTensor to use as graph inputs/outputs with the Model Editor API. + * + * User can release `tensor_info` after creating the OrtTypeInfo. + * + * \param[in] tensor_info SparseTensor type and shape information. + * \param[out] type_info TypeInfo instance for the tensor. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(CreateSparseTensorTypeInfo, _In_ const OrtTensorTypeAndShapeInfo* tensor_info, + _Outptr_ OrtTypeInfo** type_info); + + /** \brief Create an OrtTypeInfo instance for a Map. + * + * Create an OrtTypeInfo instance for a Map to use as graph inputs/outputs with the Model Editor API. + * + * User can release `map_value_type` after creating the OrtTypeInfo. + * + * \param[in] map_key_type Key type for the map. + * \param[in] map_value_type Value type for the map. + * \param[out] type_info TypeInfo instance for the map. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(CreateMapTypeInfo, ONNXTensorElementDataType map_key_type, _In_ const OrtTypeInfo* map_value_type, + _Outptr_ OrtTypeInfo** type_info); + + /** \brief Create an OrtTypeInfo instance for a Sequence. + * + * Create an OrtTypeInfo instance for a Sequence to use as graph inputs/outputs with the Model Editor API. + * + * User can release `sequence_type` after creating the OrtTypeInfo. + * + * \param[in] sequence_type Sequence type and shape information. + * \param[out] type_info TypeInfo instance for the sequence. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(CreateSequenceTypeInfo, _In_ const OrtTypeInfo* sequence_type, _Outptr_ OrtTypeInfo** type_info); + + /** \brief Create an OrtTypeInfo instance for an Optional. + * + * Create an OrtTypeInfo instance for an Optional to use as graph inputs/outputs with the Model Editor API. + * + * User can release `contained_type` after creating the OrtTypeInfo. + * + * \param[in] contained_type Tensor type and shape information. + * \param[out] type_info TypeInfo instance for the tensor. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(CreateOptionalTypeInfo, _In_ const OrtTypeInfo* contained_type, _Outptr_ OrtTypeInfo** type_info); + + /** \brief Create an OrtValueInfo for use as an OrtGraph input or output. + * + * \param[in] name The name of the input or output. + * \param[in] type_info The type information for the input or output. The provided value is copied. + * \param[out] value_info The OrtValueInfo instance. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(CreateValueInfo, _In_ const char* name, _In_ const OrtTypeInfo* type_info, + _Outptr_ OrtValueInfo** value_info); + + /** \brief Create an OrtNode to add to an OrtGraph. + * + * Create an OrtNode. + * + * Create attributes with CreateOpAttr. OrtOpAttr instances are copied. + * + * \param[in] operator_name The name of the operator. + * \param[in] domain_name The domain of the operator. Use an empty string for ONNX operators. + * \param[in] node_name The name of the node. + * \param[in] input_names The names of the inputs. + * \param[in] input_names_len The number of input names. + * \param[in] output_names The names of the outputs. + * \param[in] output_names_len The number of output names. + * \param[in] attributes The optional attributes of the node. + * \param[in] attribs_len The number of attributes. May be zero. + * \param[out] node The OrtNode instance. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(CreateNode, _In_ const char* operator_name, _In_ const char* domain_name, _In_ const char* node_name, + _In_reads_(input_names_len) const char* const* input_names, size_t input_names_len, + _In_reads_(output_names_len) const char* const* output_names, size_t output_names_len, + _In_reads_(attribs_len) _In_opt_ OrtOpAttr** attributes, _In_ size_t attribs_len, + _Outptr_ OrtNode** node); + + /** \brief Create an OrtGraph + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.22. + */ + ORT_API2_STATUS(CreateGraph, _Outptr_ OrtGraph** graph); + + /** \brief Set the inputs for the OrtGraph. + * + * Set the graph inputs. This will replace any existing inputs with the new values. + * The OrtGraph takes ownership of the OrtValueInfo instances and you should NOT call ReleaseOrtValueInfo. + * + * \param[in] graph The OrtGraph instance to update. + * \param[in] inputs The input OrtValueInfo instances. + * \param[in] inputs_len The number of input OrtValueInfo instances. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(SetGraphInputs, _Inout_ OrtGraph* graph, + _In_reads_(inputs_len) _In_ OrtValueInfo** inputs, _In_ size_t inputs_len); + + /** \brief Set the outputs for the OrtGraph. + * + * Set the graph outputs. This will replace any existing outputs with the new values. + * The OrtGraph takes ownership of the OrtValueInfo instances provided and you should NOT call ReleaseOrtValueInfo. + * + * \param[in] graph The OrtGraph instance to update. + * \param[in] outputs The output OrtValueInfo instances. + * \param[in] outputs_len The number of output OrtValueInfo instances. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(SetGraphOutputs, _Inout_ OrtGraph* graph, + _In_reads_(outputs_len) _In_ OrtValueInfo** outputs, _In_ size_t outputs_len); + + /** \brief Add an initializer to the OrtGraph + * + * ORT will take ownership of the OrtValue and you should NOT call ReleaseOrtValue. + * + * Two options: + * + * Allocated memory: + * Use CreateTensorAsOrtValue (allocates memory) and populate the tensor with the data. + * Set `data_is_external` to false. + * + * Pre-existing memory: + * Use CreateTensorWithDataAsOrtValue or CreateTensorWithDataAndDeleterAsOrtValue to create an OrtValue + * with a tensor that contains a pointer to the existing data. + * Set `data_is_external` to true. + * + * The pointer must remain valid for the duration of the inference session. + * If using CreateTensorWithDataAsOrtValue you are responsible for freeing the memory after the inference session + * is released. + * If using CreateTensorWithDataAndDeleterAsOrtValue, ORT will free the memory using the provided deleter as + * soon as the OrtValue is no longer in use. + * + * NOTE: A tensor containing pre-existing memory MUST have 128 bytes of data or more. + * For smaller tensors use CreateTensorAsOrtValue. + * + * ONNX shape inferencing does not support external data. An initializer involved in shape inferencing is + * typically small (a single value or limited by the rank of a tensor) and uses less than 128 bytes of + * memory, so this limit acts as a simple catch-all rule to avoid issues. + * e.g. Reshape's `shape`, Clip's `min` and `max`, various ops `axes`. + * + * \param[in] graph The OrtGraph instance to update. + * \param[in] name The value name for the initializer. + * \param[in] tensor The OrtValue instance containing the tensor data. + * \param[in] data_is_external Set to true if the data is external and should not be copied. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(AddInitializerToGraph, _Inout_ OrtGraph* graph, _In_ const char* name, _In_ OrtValue* tensor, + bool data_is_external); + + /** \brief Add an OrtNode to an OrtGraph + * + * Add the node to the graph. The OrtGraph will take ownership of OrtNode and you should NOT call ReleaseOrtNode. + * + * \param[in] graph The OrtGraph instance to update. + * \param[in] node The OrtNode instance to add to the graph. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(AddNodeToGraph, _Inout_ OrtGraph* graph, _In_ OrtNode* node); + + /** \brief Create an OrtModel. + * + * Create an OrtModel. + * + * This can be used to build a new model, or to augment an existing model. + * + * \param[in] domain_names The domain names for the model. + * If augmenting an existing model add additional domains if needed. + * \param[in] opset_versions The opset versions for the model. + * If augmenting an existing model add additional opset versions if needed. + * \param[in] opset_entries_len The number of domain_names and opset_versions entries. + * Domain and opset entries should be 1:1 + * \param[out] model The OrtModel instance. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(CreateModel, + _In_reads_(opset_entries_len) const char* const* domain_names, + _In_reads_(opset_entries_len) const int* opset_versions, + size_t opset_entries_len, + _Outptr_ OrtModel** model); + + /** \brief Add an OrtGraph to an OrtModel. + * + * Add the graph to a model. This should be called once when creating a new model. + * + * The OrtModel takes ownership of the OrtGraph and you should NOT call ReleaseOrtGraph. + * + * \param[in] model The OrtModel instance to update. + * \param[in] graph The OrtGraph instance to add to the model. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(AddGraphToModel, _Inout_ OrtModel* model, _In_ OrtGraph* graph); + + /** \brief Create an OrtSession using the OrtModel. + * + * Create an inference session using the OrtModel instance. + * The OrtModel should have been populated with an OrtGraph containing nodes and initializers, and SetGraphInputs + * and SetGraphOutputs must have been called. + * This will validate the model, run optimizers, and prepare the session for inferencing. + * + * ReleaseOrtModel must be called to free the OrtModel after session creation. + * + * \param[in] env The OrtEnv instance. + * \param[in] model The OrtModel instance. + * \param[in] options The OrtSessionOptions instance. + * \param[out] out The OrtSession instance. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(CreateSessionFromModel, _In_ const OrtEnv* env, _In_ const OrtModel* model, + _In_ const OrtSessionOptions* options, _Outptr_ OrtSession** out); + + /** \brief Create an OrtSession to augment an existing model. + * + * Create an OrtSession with an existing model that will be augmented with additional nodes and initializers. + * Nodes can be added before or after the existing nodes in the model. ONNX Runtime will connect the nodes when the + * model is finalized. + * + * To add nodes and initializers to the existing model, first create an OrtModel using CreateModel. + * Add nodes and initializers to the OrtModel using AddNodeToGraph and AddInitializerToGraph. + * Graph inputs/outputs should be updated with SetGraphInputs and SetGraphOutputs as needed to reflect changes made + * by the new nodes. The list of graph inputs/outputs should be for the overall model and not just the new nodes. + * + * Add the new information from the OrtModel to the original model using ApplyModelToSession, and prepare the + * session for inferencing by calling FinalizeModelEditorSession. + * + * \param{in} env The OrtEnv instance. + * \param{in} model_path The path to the existing ONNX model to augment. + * \param{in} options The OrtSessionOptions instance. + * \param{out} out The created OrtSession instance. + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(CreateModelEditorSession, _In_ const OrtEnv* env, _In_ const ORTCHAR_T* model_path, + _In_ const OrtSessionOptions* options, + _Outptr_ OrtSession** out); + + /** \brief Create an OrtSession to augment an existing model. + * + * Create an OrtSession with an existing model that will be augmented with additional nodes and initializers. + * Nodes can be added before or after the existing nodes in the model. ONNX Runtime will connect the nodes when the + * model is finalized. + * + * To add nodes and initializers to the existing model, first create an OrtModel using CreateModel. + * Add nodes and initializers to the OrtModel using AddNodeToGraph and AddInitializerToGraph. + * Graph inputs/outputs should be updated with SetGraphInputs and SetGraphOutputs as needed to reflect changes made + * by the new nodes. The list of graph inputs/outputs should be for the overall model and not just the new nodes. + * + * Add the new information from the OrtModel to the original model using ApplyModelToSession, and prepare the + * session for inferencing by calling FinalizeModelEditorSession. + * + * \param{in} env The OrtEnv instance. + * \param{in} model_data The model data for the existing model to augment. + * \param{in} model_data_length The length of the model data. + * \param{in} options The OrtSessionOptions instance. + * \param{out} out The created OrtSession instance. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(CreateModelEditorSessionFromArray, _In_ const OrtEnv* env, + _In_ const void* model_data, size_t model_data_length, + _In_ const OrtSessionOptions* options, + _Outptr_ OrtSession** out); + + /** \brief Query the session for the opset version of a domain. + * + * When using the Model Editor API to augment a model, any new nodes must conform to the opset version of the + * original model. To do that the user must be able to discover that opset version. + * Returns an error if the domain is not used in the model. + * + * \param[in] session OrtSession to query + * \param[in] domain Domain to query. The ONNX domain is an empty string. + * \param[out] opset The opset version of the domain. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(SessionGetOpsetForDomain, _In_ const OrtSession* session, _In_ const char* domain, _Out_ int* opset); + + /** \brief Apply changes to augment the ONNX model in a session created using CreateModelEditorSession[FromArray] + * + * Adds new nodes and updates graph inputs/outputs using `model` to augment the original ONNX model in the session. + * All changes will be validated. + * Call FinalizeModelEditorSession to prepare the session for inferencing. + * + * Existing input/outputs will only be updated if the OrtGraph inputs/outputs are set in the OrtModel. + * i.e. you don't need to call SetGraphInputs/SetGraphOutputs if they are unchanged. + * + * ReleaseOrtModel must be called to free the OrtModel after it is applied to the session. + * + * \param[in] session OrtSession to update. Session must have been created using CreateModelEditorSession[FromArray]. + * \param[in] model OrtModel containing new nodes, new initializers, and updated graph input and/or output info. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(ApplyModelToModelEditorSession, _Inout_ OrtSession* session, _In_ OrtModel* model); + + /** \brief Finalize the Model Editor session that was created using CreateModelEditorSession[FromArray]. + * + * Finalize the Model Editor session that augmented an ONNX model by adding new nodes. + * This will run optimizers and prepare the session for inferencing. + * + * \param[in] session OrtSession to finalize. Session must have been created using CreateModelEditorSession[FromArray]. + * \param[in] options OrtSessionOptions to use for the session. + * \param[in] prepacked_weights_container Optional OrtPrepackedWeightsContainer to use for the session. + Set to nullptr if not used. + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(FinalizeModelEditorSession, _Inout_ OrtSession* session, _In_ const OrtSessionOptions* options, + _In_opt_ OrtPrepackedWeightsContainer* prepacked_weights_container); +#endif // !defined(ORT_MINIMAL_BUILD) +}; + +/** + * ORT Compile API + */ + +/** \brief Flags representing options to enable when compiling a model. + */ +typedef enum OrtCompileApiFlags { + // Default. Do not enable any additional compilation options. + OrtCompileApiFlags_NONE = 0, + + // Force compilation to return an error (ORT_FAIL) if no nodes were compiled. + // Otherwise, a model with basic optimizations (ORT_ENABLE_BASIC) is still generated by default. + OrtCompileApiFlags_ERROR_IF_NO_NODES_COMPILED = 1 << 0, + + // Force compilation to return an error (ORT_FAIL) if a file with the same filename as the output model exists. + // Otherwise, compilation will automatically overwrite the output file if it exists. + OrtCompileApiFlags_ERROR_IF_OUTPUT_FILE_EXISTS = 1 << 1, +} OrtCompileApiFlags; + +/** + * \brief The OrtCompileApi struct provides functions to compile ONNX models. + * + * Execution providers that support compilation fuse a subgraph into an EPContext node that wraps a provider-specific + * binary representation of the subgraph. + * For more details about the EPContext design, refer to: + * \htmlonly + * EPContext design document. + * \endhtmlonly + * + * Example (error handling not shown): + * OrtStatus* status = NULL; + * OrtCompileApi* compile_api = ort_api->GetCompileApi(); + * OrtModelCompilationOptions* compile_options = NULL; + * + * status = compile_api->CreateModelCompilationOptionsFromSessionOptions(env, session_options, &compile_options); + * status = compile_api->ModelCompilationOptions_SetInputModelPath(compile_options, ORT_TSTR("model.onnx")); + * status = compile_api->ModelCompilationOptions_SetOutputModelPath(compile_options, ORT_TSTR("model.compiled.onnx")); + * status = compile_api->CompileModel(env, compile_options); + * compile_api->ReleaseModelCompilationOptions(compile_options); + * + * \since Version 1.22. + */ +struct OrtCompileApi { + /// \name OrtModelCompilationOptions + /// @{ + ORT_CLASS_RELEASE(ModelCompilationOptions); + + /** \brief Creates an OrtModelCompilationOptions object from an existing OrtSessionOptions object. + * + * An OrtModelCompilationOptions object contains the settings used to generate a compiled ONNX model. + * The OrtSessionOptions object has the execution providers with which the model will be compiled. + * + * ReleaseOrtModelCompilationsOptions must be called to free the OrtModelCompilationOptions after calling + * CompileModel. + * + * \note By default, the GraphOptimizationLevel is set to ORT_DISABLE_ALL. Use + * ModelCompilationOptions_SetGraphOptimizationLevel to enable graph optimizations. + * + * \param[in] env OrtEnv object. + * \param[in] session_options The OrtSessionOptions instance from which to create the OrtModelCompilationOptions. + * \param[out] out The created OrtModelCompilationOptions instance. + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(CreateModelCompilationOptionsFromSessionOptions, _In_ const OrtEnv* env, + _In_ const OrtSessionOptions* session_options, _Outptr_ OrtModelCompilationOptions** out); + + /** \brief Sets the file path to the input ONNX model to compile. + * + * The input model's location (e.g., file path or memory buffer) must be set with either + * ModelCompilationOptions_SetInputModelPath or ModelCompilationOptions_SetInputModelFromBuffer. + * + * \param[in] model_compile_options The OrtModelCompilationOptions instance. + * \param[in] input_model_path Null terminated string of the path (wchar on Windows, char otherwise). + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(ModelCompilationOptions_SetInputModelPath, _In_ OrtModelCompilationOptions* model_compile_options, + _In_ const ORTCHAR_T* input_model_path); + + /** \brief Sets the buffer that stores the bytes of the loaded ONNX model to compile. + * + * The input model's location (e.g., file path or memory buffer) must be set with either + * ModelCompilationOptions_SetInputModelPath or ModelCompilationOptions_SetInputModelFromBuffer. + * + * \param[in] model_compile_options The OrtModelCompilationOptions instance. + * \param[in] input_model_data Buffer containing the loaded ONNX model bytes. + * \param[in] input_model_data_size The number of bytes in the `input_model_data` buffer. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(ModelCompilationOptions_SetInputModelFromBuffer, + _In_ OrtModelCompilationOptions* model_compile_options, + _In_ const void* input_model_data, + size_t input_model_data_size); + + /** \brief Sets the file path for the output ONNX model generated by CompileModel. + * + * The output model's location (e.g., file path or memory buffer) can be set with either + * ModelCompilationOptions_SetOutputModelPath or ModelCompilationOptions_SetOutputModelBuffer. + * + * If the output model's location is not set, ONNX Runtime will generate an output file with a path based on + * the input model's file path. Examples: + * /Path/my_model.onnx -> /Path/my_model_ctx.onnx + * /Path/my_model -> /Path/my_model_ctx.onnx + * + * \param[in] model_compile_options The OrtModelCompilationOptions instance. + * \param[in] output_model_path Null terminated string of the path (wchar on Windows, char otherwise). + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(ModelCompilationOptions_SetOutputModelPath, _In_ OrtModelCompilationOptions* model_compile_options, + _In_ const ORTCHAR_T* output_model_path); + + /** \brief Optionally sets the file that should store external initializers for the compiled ONNX model. + * If not set, initializers are stored within the model. + * + * Only initializers for nodes that were not compiled are stored in the external initializers file. + * Compiled nodes contain their initializer data within the `ep_cache_context` attribute of EPContext nodes. + * Refer to ModelCompilationOptions_SetEpContextEmbedMode. + * + * \param[in] model_compile_options The OrtModelCompilationOptions instance. + * \param[in] external_initializers_file_path Null terminated string of the path to the file. + * \param[in] external_initializers_size_threshold Initializers larger than this threshold are stored in the file. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(ModelCompilationOptions_SetOutputModelExternalInitializersFile, + _In_ OrtModelCompilationOptions* model_compile_options, + _In_ const ORTCHAR_T* external_initializers_file_path, + size_t external_initializers_size_threshold); + + /** \brief Configures model compilation to store the output compiled ONNX model in a buffer. + * + * The caller passes an OrtAllocator that ONNX Runtime uses to allocate memory for the buffer. + * + * The output model's location (e.g., file path or memory buffer) can be set with either + * ModelCompilationOptions_SetOutputModelPath or ModelCompilationOptions_SetOutputModelBuffer. + * + * If the output model's location is not set, ONNX Runtime will generate an output file with a path based on + * the input model's file path. Examples: + * /Path/my_model.onnx -> /Path/my_model_ctx.onnx + * /Path/my_model -> /Path/my_model_ctx.onnx + * + * \param[in] model_compile_options The OrtModelCompilationOptions instance. + * \param[in] allocator The allocator used to allocate the buffer for the compiled model. + * \param[out] output_model_buffer_ptr Pointer to the buffer that stores the compiled model. + * \param[out] output_model_buffer_size_ptr Pointer set to the size of output model in bytes. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(ModelCompilationOptions_SetOutputModelBuffer, + _In_ OrtModelCompilationOptions* model_compile_options, + _Inout_ OrtAllocator* allocator, + _Outptr_ void** output_model_buffer_ptr, + _Out_ size_t* output_model_buffer_size_ptr); + + /** \brief Enables or disables the embedding of EPContext binary data into the `ep_cache_context` attribute + * of EPContext nodes. Defaults to false. + * + * If enabled, the `ep_cache_context` attribute of EPContext nodes will store the context binary data, which may + * include weights for compiled subgraphs. + * + * If disabled, the `ep_cache_context` attribute of EPContext nodes will contain the path to the file containing the + * context binary data. The path is set by the execution provider creating the EPContext node. + * + * More details relate to EPContext design refers to: + * \htmlonly + * EPContext design document. + * \endhtmlonly + * + * \param[in] model_compile_options The OrtModelCompilationOptions instance. + * \param[in] embed_ep_context_in_model True to embed EPContext binary data into the EPContext node + * `ep_cache_context` attributes. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(ModelCompilationOptions_SetEpContextEmbedMode, _In_ OrtModelCompilationOptions* model_compile_options, + bool embed_ep_context_in_model); + + /** \brief Compiles an input ONNX model with the given compilation options. + * + * \param[in] env OrtEnv object. + * \param[in] model_options The compilation options that defines compilation options for a model. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(CompileModel, _In_ const OrtEnv* env, _In_ const OrtModelCompilationOptions* model_options); + + /** \brief Sets flags from OrtCompileApiFlags that represent one or more boolean options to enable. + * + * \param[in] model_compile_options The OrtModelCompilationOptions instance. + * \param[in] flags bitwise OR of flags in OrtCompileApiFlags to enable. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(ModelCompilationOptions_SetFlags, _In_ OrtModelCompilationOptions* model_compile_options, + uint32_t flags); + + /** Sets information related to EP context binary file. + * + * EP uses this information to decide the location and context binary file name. + * Used while compiling model with input and output in memory buffer + * + * \param[in] model_compile_options The OrtModelCompilationOptions instance. + * \param[in] output_directory Null terminated string of the path (wchar on Windows, char otherwise). + * \param[in] model_name Null terminated string of the model name (wchar on Windows, char otherwise). + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(ModelCompilationOptions_SetEpContextBinaryInformation, + _In_ OrtModelCompilationOptions* model_compile_options, + _In_ const ORTCHAR_T* output_directory, + _In_ const ORTCHAR_T* model_name); + + /** Set the graph optimization level. + * + * \param[in] model_compile_options The OrtModelCompilationOptions instance. + * \param[in] graph_optimization_level The graph optimization level. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(ModelCompilationOptions_SetGraphOptimizationLevel, + _In_ OrtModelCompilationOptions* model_compile_options, + _In_ GraphOptimizationLevel graph_optimization_level); + + /** \brief Sets a OrtWriteBufferFunc function that is called by ORT to write out the output model's serialized + * ONNX bytes. + * + * The provided write function may be called repeatedly until then entire output model has been written out. Each call + * to the write function is expected to consume the entire input buffer. + * + * The output model's destination (e.g., file path, memory buffer, or stream) can be set with any of the functions + * that begin with ModelCompilationOptions_SetOutputModel____. + * + * \param[in] model_compile_options The OrtModelCompilationOptions instance. + * \param[in] write_func The OrtWriteBufferFunc function called by ORT when writing out the model. + * \param[in] state Opaque state passed as the first argument to OrtWriteBufferFunc. Can be NULL. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(ModelCompilationOptions_SetOutputModelWriteFunc, + _In_ OrtModelCompilationOptions* model_compile_options, + _In_ OrtWriteBufferFunc write_func, _In_ void* state); + + /** \brief Sets a OrtGetInitializerLocationFunc function that is called by ORT for every initializer in the generated + * model. Allows implementer to specify whether initializers should be stored within the model or externally. + * + * \param[in] model_compile_options The OrtModelCompilationOptions instance. + * \param[in] get_initializer_location_func The OrtGetInitializerLocationFunc function called by ORT when + * to determine the location of the initializer. + * \param[in] state Opaque state passed as the first argument to OrtGetInitializerLocationFunc. Can be NULL. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(ModelCompilationOptions_SetOutputModelGetInitializerLocationFunc, + _In_ OrtModelCompilationOptions* model_compile_options, + _In_ OrtGetInitializerLocationFunc get_initializer_location_func, _In_ void* state); +}; + +/** + * \brief The OrtInteropApi struct provides functions for external resource interop with execution providers. + * + * This API enables importing external GPU resources (memory and semaphores) for zero-copy sharing + * between ORT inference and other GPU workloads (e.g., D3D12 applications, media pipelines). + * + * The API is designed to be EP-agnostic and can be extended to support various GPU interop mechanisms + * (D3D12 shared handles, CUDA external memory, Vulkan, etc.). + * + * Example usage (error handling not shown): + * const OrtInteropApi* interop_api = ort_api->GetInteropApi(); + * OrtExternalResourceImporter* importer = NULL; + * + * status = interop_api->CreateExternalResourceImporterForDevice(ep_device, &importer); + * if (importer == nullptr) { + * // External resource import is optional for EPs to implement + * return; + * } + * bool can_import = false; + * status = interop_api->CanImportMemory(importer, ORT_EXTERNAL_MEMORY_HANDLE_TYPE_D3D12_RESOURCE, &can_import); + * if (can_import) { + * OrtExternalMemoryHandle* mem_handle = NULL; + * status = interop_api->ImportMemory(importer, &mem_desc, &mem_handle); + * // ... use mem_handle to create tensors ... + * interop_api->ReleaseExternalMemoryHandle(mem_handle); + * } + * interop_api->ReleaseExternalResourceImporter(importer); + * + * \since Version 1.24. + */ +struct OrtInteropApi { + /// \name OrtExternalResourceImporter + /// @{ + + /** \brief Create an external resource importer for a specific EP device. + * + * The external resource importer is a capability object that provides methods for importing + * external GPU memory and semaphores for zero-copy import with an execution provider. + * + * This is an optional EP capability. If the EP does not support external resource import, + * out_importer is set to nullptr and the function returns success (nullptr status). + * This allows callers to use the simple "if (status != nullptr) handle_error()" pattern + * and check out_importer separately for capability detection. + * + * \param[in] ep_device The OrtEpDevice instance to create the importer for. + * \param[out] out_importer Output parameter set to the created OrtExternalResourceImporter instance, + * or nullptr if the EP does not support external resource import. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(CreateExternalResourceImporterForDevice, + _In_ const OrtEpDevice* ep_device, + _Outptr_result_maybenull_ OrtExternalResourceImporter** out_importer); + + /** \brief Release an OrtExternalResourceImporter instance. + * + * \param[in] input The OrtExternalResourceImporter instance to release. May be nullptr. + * + * \since Version 1.24. + */ + ORT_CLASS_RELEASE(ExternalResourceImporter); + + /// @} + /// \name Memory Import + /// @{ + + /** \brief Check if the external resource importer can import a specific memory handle type. + * + * \param[in] importer The OrtExternalResourceImporter instance. + * \param[in] handle_type The type of external memory handle to check. + * \param[out] out_supported Set to true if the handle type is supported. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(CanImportMemory, + _In_ const OrtExternalResourceImporter* importer, + _In_ OrtExternalMemoryHandleType handle_type, + _Out_ bool* out_supported); + + /** \brief Import external memory into the execution provider. + * + * \param[in] importer The OrtExternalResourceImporter instance. + * \param[in] desc Descriptor containing the external memory handle and properties. + * \param[out] out_handle Output parameter set to the created OrtExternalMemoryHandle. + * The caller owns the returned handle and must call ReleaseExternalMemoryHandle to free it. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(ImportMemory, + _In_ OrtExternalResourceImporter* importer, + _In_ const OrtExternalMemoryDescriptor* desc, + _Outptr_ OrtExternalMemoryHandle** out_handle); + + /** \brief Release an OrtExternalMemoryHandle instance. + * + * \param[in] input The OrtExternalMemoryHandle instance to release. May be nullptr. + * + * \since Version 1.24. + */ + ORT_CLASS_RELEASE(ExternalMemoryHandle); + + /** \brief Create a tensor backed by imported external memory. + * + * The created tensor is a view over the imported memory and does not copy data. + * The OrtExternalMemoryHandle must remain valid for the lifetime of the tensor. + * + * \param[in] importer The OrtExternalResourceImporter instance. + * \param[in] mem_handle The imported external memory handle. + * \param[in] tensor_desc Descriptor specifying tensor element type, shape, and optional offset. + * \param[out] out_tensor Output parameter set to the created OrtValue containing the tensor. + * The caller owns the returned tensor and must call ReleaseValue to free it. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(CreateTensorFromMemory, + _In_ OrtExternalResourceImporter* importer, + _In_ const OrtExternalMemoryHandle* mem_handle, + _In_ const OrtExternalTensorDescriptor* tensor_desc, + _Outptr_ OrtValue** out_tensor); + + /// @} + /// \name Semaphore Import + /// @{ + + /** \brief Check if the external resource importer can import a specific semaphore type. + * + * \param[in] importer The OrtExternalResourceImporter instance. + * \param[in] type The type of external semaphore to check. + * \param[out] out_supported Set to true if the semaphore type is supported. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(CanImportSemaphore, + _In_ const OrtExternalResourceImporter* importer, + _In_ OrtExternalSemaphoreType type, + _Out_ bool* out_supported); + + /** \brief Import an external semaphore into the execution provider. + * + * The returned OrtExternalSemaphoreHandle can be used with WaitSemaphore and an OrtSyncStream + * to synchronize execution with external operations. + * + * \param[in] importer The OrtExternalResourceImporter instance. + * \param[in] desc Descriptor containing the external semaphore handle and type. + * \param[out] out_handle Output parameter set to the created OrtExternalSemaphoreHandle. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(ImportSemaphore, + _In_ OrtExternalResourceImporter* importer, + _In_ const OrtExternalSemaphoreDescriptor* desc, + _Outptr_ OrtExternalSemaphoreHandle** out_handle); + + /** \brief Release an OrtExternalSemaphoreHandle instance. + * + * \param[in] input The OrtExternalSemaphoreHandle instance to release. May be nullptr. + * + * \since Version 1.24. + */ + ORT_CLASS_RELEASE(ExternalSemaphoreHandle); + + /** \brief Wait on an external semaphore on the EP's stream. + * + * Inserts a wait operation into the EP's stream that blocks until the semaphore + * reaches the specified value. This is used to synchronize with external GPU work + * (e.g., D3D12 timeline fence). + * + * \param[in] importer The OrtExternalResourceImporter instance. + * \param[in] semaphore_handle The imported external semaphore. + * \param[in] stream The OrtSyncStream to wait on. + * \param[in] value The fence/semaphore value to wait for. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(WaitSemaphore, + _In_ OrtExternalResourceImporter* importer, + _In_ OrtExternalSemaphoreHandle* semaphore_handle, + _In_ OrtSyncStream* stream, + _In_ uint64_t value); + + /** \brief Signal an external semaphore from the EP's stream. + * + * Inserts a signal operation into the EP's stream that sets the semaphore + * to the specified value when reached. This is used to notify external GPU work + * (e.g., D3D12 timeline fence) that ORT inference is complete. + * + * \param[in] importer The OrtExternalResourceImporter instance. + * \param[in] semaphore_handle The imported external semaphore. + * \param[in] stream The OrtSyncStream to signal from. + * \param[in] value The fence/semaphore value to signal. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(SignalSemaphore, + _In_ OrtExternalResourceImporter* importer, + _In_ OrtExternalSemaphoreHandle* semaphore_handle, + _In_ OrtSyncStream* stream, + _In_ uint64_t value); + + /// @} +}; + +/* + * This is the old way to add the CUDA provider to the session, please use SessionOptionsAppendExecutionProvider_CUDA above to access the latest functionality + * This function always exists, but will only succeed if Onnxruntime was built with CUDA support and the CUDA provider shared library exists + * + * \param device_id CUDA device id, starts from zero. + */ +ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_CUDA, _In_ OrtSessionOptions* options, int device_id); + +/* + * This is the old way to add the ROCm provider to the session, please use + * SessionOptionsAppendExecutionProvider_ROCM above to access the latest functionality + * This function always exists, but will only succeed if Onnxruntime was built with + * HIP support and the ROCm provider shared library exists + * + * \param device_id HIP device id, starts from zero. + */ +ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_ROCM, _In_ OrtSessionOptions* options, int device_id); + +/* + * This is the old way to add the MIGraphX provider to the session, please use + * SessionOptionsAppendExecutionProvider_MIGraphX above to access the latest functionality + * This function always exists, but will only succeed if Onnxruntime was built with + * HIP support and the MIGraphX provider shared library exists + * + * \param device_id HIP device id, starts from zero. + */ +ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_MIGraphX, _In_ OrtSessionOptions* options, int device_id); + +/* + * This is the old way to add the oneDNN provider to the session, please use + * SessionOptionsAppendExecutionProvider_oneDNN above to access the latest functionality + * This function always exists, but will only succeed if Onnxruntime was built with + * oneDNN support and the oneDNN provider shared library exists + * + * \param use_arena zero: false. non-zero: true. + */ +ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_Dnnl, _In_ OrtSessionOptions* options, int use_arena); + +/* + * This is the old way to add the TensorRT provider to the session, please use SessionOptionsAppendExecutionProvider_TensorRT_V2 above to access the latest functionality + * This function always exists, but will only succeed if Onnxruntime was built with TensorRT support and the TensorRT provider shared library exists + * + * \param device_id CUDA device id, starts from zero. + */ +ORT_API_STATUS(OrtSessionOptionsAppendExecutionProvider_Tensorrt, _In_ OrtSessionOptions* options, int device_id); + +#ifdef __cplusplus +} +#endif +/// @} + +#include "onnxruntime_ep_c_api.h" diff --git a/dist/include/onnxruntime/onnxruntime_cxx_api.h b/dist/include/onnxruntime/onnxruntime_cxx_api.h new file mode 100644 index 0000000..5cf8cf8 --- /dev/null +++ b/dist/include/onnxruntime/onnxruntime_cxx_api.h @@ -0,0 +1,3534 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// Summary: The Ort C++ API is a header only wrapper around the Ort C API. +// +// The C++ API simplifies usage by returning values directly instead of error codes, throwing exceptions on errors +// and automatically releasing resources in the destructors. The primary purpose of C++ API is exception safety so +// all the resources follow RAII and do not leak memory. +// +// Each of the C++ wrapper classes holds only a pointer to the C internal object. Treat them like smart pointers. +// To create an empty object, pass 'nullptr' to the constructor (for example, Env e{nullptr};). However, you can't use them +// until you assign an instance that actually holds an underlying object. +// +// For Ort objects only move assignment between objects is allowed, there are no copy constructors. +// Some objects have explicit 'Clone' methods for this purpose. +// +// ConstXXXX types are copyable since they do not own the underlying C object, so you can pass them to functions as arguments +// by value or by reference. ConstXXXX types are restricted to const only interfaces. +// +// UnownedXXXX are similar to ConstXXXX but also allow non-const interfaces. +// +// The lifetime of the corresponding owning object must eclipse the lifetimes of the ConstXXXX/UnownedXXXX types. They exists so you do not +// have to fallback to C types and the API with the usual pitfalls. In general, do not use C API from your C++ code. + +#pragma once +#include "onnxruntime_c_api.h" +#include "onnxruntime_float16.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef ORT_NO_EXCEPTIONS +#include +#endif + +/** \brief All C++ Onnxruntime APIs are defined inside this namespace + * + */ +namespace Ort { + +/** \brief All C++ methods that can fail will throw an exception of this type + * + * If ORT_NO_EXCEPTIONS is defined, then any error will result in a call to abort() + */ +struct Exception : std::exception { + Exception(const std::string& string, OrtErrorCode code) : message_{string}, code_{code} {} + Exception(std::string&& string, OrtErrorCode code) : message_{std::move(string)}, code_{code} {} + + OrtErrorCode GetOrtErrorCode() const { return code_; } + const char* what() const noexcept override { return message_.c_str(); } + + private: + std::string message_; + OrtErrorCode code_; +}; + +#ifdef ORT_NO_EXCEPTIONS +// The #ifndef is for the very special case where the user of this library wants to define their own way of handling errors. +// NOTE: This header expects control flow to not continue after calling ORT_CXX_API_THROW +#ifndef ORT_CXX_API_THROW +#define ORT_CXX_API_THROW(string, code) \ + do { \ + std::cerr << Ort::Exception(string, code) \ + .what() \ + << std::endl; \ + abort(); \ + } while (false) +#endif +#else +#define ORT_CXX_API_THROW(string, code) \ + throw Ort::Exception(string, code) +#endif + +#ifdef ORT_API_MANUAL_INIT +// If the macro ORT_API_MANUAL_INIT is defined, no static initialization +// will be performed. Instead, users must call InitApi() before using the +// ORT C++ APIs.. +// +// InitApi() sets the global API object using the default initialization +// logic. Users call this to initialize the ORT C++ APIs at a time that +// makes sense in their program. +inline void InitApi() noexcept; + +// InitApi(const OrtApi*) is used by custom operator libraries that are not +// linked to onnxruntime. It sets the global API object, which is required +// by the ORT C++ APIs. +// +// Example mycustomop.cc: +// +// #define ORT_API_MANUAL_INIT +// #include +// #undef ORT_API_MANUAL_INIT +// +// OrtStatus* ORT_API_CALL RegisterCustomOps(OrtSessionOptions* options, const OrtApiBase* api_base) { +// Ort::InitApi(api_base->GetApi(ORT_API_VERSION)); +// // ... +// } +// +inline void InitApi(const OrtApi* api) noexcept; +#endif + +namespace detail { +// This is used internally by the C++ API. This class holds the global +// variable that points to the OrtApi. +struct Global { + static const OrtApi* Api(const OrtApi* newValue = nullptr) noexcept { + // This block-level static will be initialized once when this function is + // first executed, delaying the call to DefaultInit() until it is first needed. + // + // When ORT_API_MANUAL_INIT is not defined, DefaultInit() calls + // OrtGetApiBase()->GetApi(), which may result in a shared library being + // loaded. + // + // Using a block-level static instead of a class-level static helps + // avoid issues with static initialization order and dynamic libraries + // loading other dynamic libraries. + // + // This makes it safe to include the C++ API headers in a shared library + // that is delay loaded or delay loads its dependencies. + // + // This DOES NOT make it safe to _use_ arbitrary ORT C++ APIs when + // initializing static members, however. + static const OrtApi* api = DefaultInit(); + + if (newValue) { + api = newValue; + } + + return api; + } + + private: + // Has different definitions based on ORT_API_MANUAL_INIT + static const OrtApi* DefaultInit() noexcept; + +#ifdef ORT_API_MANUAL_INIT + // Public APIs to set the OrtApi* to use. + friend void ::Ort::InitApi() noexcept; + friend void ::Ort::InitApi(const OrtApi*) noexcept; +#endif +}; +} // namespace detail + +#ifdef ORT_API_MANUAL_INIT + +// See comments on declaration above for usage. +inline void InitApi(const OrtApi* api) noexcept { detail::Global::Api(api); } +inline void InitApi() noexcept { InitApi(OrtGetApiBase()->GetApi(ORT_API_VERSION)); } + +#ifdef _MSC_VER +// If you get a linker error about a mismatch here, you are trying to +// link two compilation units that have different definitions for +// ORT_API_MANUAL_INIT together. All compilation units must agree on the +// definition of ORT_API_MANUAL_INIT. +#pragma detect_mismatch("ORT_API_MANUAL_INIT", "enabled") +#endif + +inline const OrtApi* detail::Global::DefaultInit() noexcept { + // When ORT_API_MANUAL_INIT is defined, there's no default init that can + // be done. + return nullptr; +} + +#else // ORT_API_MANUAL_INIT + +#ifdef _MSC_VER +// If you get a linker error about a mismatch here, you are trying to link +// two compilation units that have different definitions for +// ORT_API_MANUAL_INIT together. All compilation units must agree on the +// definition of ORT_API_MANUAL_INIT. +#pragma detect_mismatch("ORT_API_MANUAL_INIT", "disabled") +#endif + +inline const OrtApi* detail::Global::DefaultInit() noexcept { + return OrtGetApiBase()->GetApi(ORT_API_VERSION); +} +#endif // ORT_API_MANUAL_INIT + +/// This returns a reference to the ORT C API. +inline const OrtApi& GetApi() noexcept { return *detail::Global::Api(); } + +/// +/// This function returns the onnxruntime version string +/// +/// version string major.minor.rev +std::string GetVersionString(); + +/// +/// This function returns the onnxruntime build information: including git branch, +/// git commit id, build type(Debug/Release/RelWithDebInfo) and cmake cpp flags. +/// +/// string +std::string GetBuildInfoString(); + +/// +/// This is a C++ wrapper for OrtApi::GetAvailableProviders() and +/// returns a vector of strings representing the available execution providers. +/// +/// vector of strings +std::vector GetAvailableProviders(); + +/// +/// This returns a reference to the ORT C Model Editor API. Used if building or augmenting a model at runtime. +/// +/// ORT C Model Editor API reference +inline const OrtModelEditorApi& GetModelEditorApi() { + auto* api = GetApi().GetModelEditorApi(); + if (api == nullptr) { + // minimal build + ORT_CXX_API_THROW("Model Editor API is not available in this build", ORT_FAIL); + } + + return *api; +} + +/// +/// This returns a reference to the ORT C Compile API. Used if compiling a model at runtime. +/// +/// ORT C Compile API reference +inline const OrtCompileApi& GetCompileApi() { + auto* api = GetApi().GetCompileApi(); + if (api == nullptr) { + // minimal build + ORT_CXX_API_THROW("Compile API is not available in this build", ORT_FAIL); + } + + return *api; +} + +/// +/// This returns a reference to the ORT C Interop API. Used for external resource import with EPs. +/// +/// ORT C Interop API reference +inline const OrtInteropApi& GetInteropApi() { + auto* api = GetApi().GetInteropApi(); + if (api == nullptr) { + // minimal build + ORT_CXX_API_THROW("Interop API is not available in this build", ORT_FAIL); + } + + return *api; +} + +/// +/// This returns a reference to the ORT C EP API. Used if authoring a plugin execution provider. +/// +/// ORT C EP API reference +inline const OrtEpApi& GetEpApi() { + auto* api = GetApi().GetEpApi(); + if (api == nullptr) { + // minimal build + ORT_CXX_API_THROW("EP API is not available in this build", ORT_FAIL); + } + + return *api; +} + +/** \brief IEEE 754 half-precision floating point data type + * + * \details This struct is used for converting float to float16 and back + * so the user could feed inputs and fetch outputs using these type. + * + * The size of the structure should align with uint16_t and one can freely cast + * uint16_t buffers to/from Ort::Float16_t to feed and retrieve data. + * + * \code{.unparsed} + * // This example demonstrates conversion from float to float16 + * constexpr float values[] = {1.f, 2.f, 3.f, 4.f, 5.f}; + * std::vector fp16_values; + * fp16_values.reserve(std::size(values)); + * std::transform(std::begin(values), std::end(values), std::back_inserter(fp16_values), + * [](float value) { return Ort::Float16_t(value); }); + * + * \endcode + */ +struct Float16_t : onnxruntime_float16::Float16Impl { + private: + /// + /// Constructor from a 16-bit representation of a float16 value + /// No conversion is done here. + /// + /// 16-bit representation + constexpr explicit Float16_t(uint16_t v) noexcept { val = v; } + + public: + using Base = onnxruntime_float16::Float16Impl; + + /// + /// Default constructor + /// + Float16_t() = default; + + /// + /// Explicit conversion to uint16_t representation of float16. + /// + /// uint16_t bit representation of float16 + /// new instance of Float16_t + constexpr static Float16_t FromBits(uint16_t v) noexcept { return Float16_t(v); } + + /// + /// __ctor from float. Float is converted into float16 16-bit representation. + /// + /// float value + explicit Float16_t(float v) noexcept { val = Base::ToUint16Impl(v); } + + /// + /// Converts float16 to float + /// + /// float representation of float16 value + float ToFloat() const noexcept { return Base::ToFloatImpl(); } + + /// + /// Checks if the value is negative + /// + /// true if negative + using Base::IsNegative; + + /// + /// Tests if the value is NaN + /// + /// true if NaN + using Base::IsNaN; + + /// + /// Tests if the value is finite + /// + /// true if finite + using Base::IsFinite; + + /// + /// Tests if the value represents positive infinity. + /// + /// true if positive infinity + using Base::IsPositiveInfinity; + + /// + /// Tests if the value represents negative infinity + /// + /// true if negative infinity + using Base::IsNegativeInfinity; + + /// + /// Tests if the value is either positive or negative infinity. + /// + /// True if absolute value is infinity + using Base::IsInfinity; + + /// + /// Tests if the value is NaN or zero. Useful for comparisons. + /// + /// True if NaN or zero. + using Base::IsNaNOrZero; + + /// + /// Tests if the value is normal (not zero, subnormal, infinite, or NaN). + /// + /// True if so + using Base::IsNormal; + + /// + /// Tests if the value is subnormal (denormal). + /// + /// True if so + using Base::IsSubnormal; + + /// + /// Creates an instance that represents absolute value. + /// + /// Absolute value + using Base::Abs; + + /// + /// Creates a new instance with the sign flipped. + /// + /// Flipped sign instance + using Base::Negate; + + /// + /// IEEE defines that positive and negative zero are equal, this gives us a quick equality check + /// for two values by or'ing the private bits together and stripping the sign. They are both zero, + /// and therefore equivalent, if the resulting value is still zero. + /// + /// first value + /// second value + /// True if both arguments represent zero + using Base::AreZero; + + /// + /// User defined conversion operator. Converts Float16_t to float. + /// + explicit operator float() const noexcept { return ToFloat(); } + + using Base::operator==; + using Base::operator!=; + using Base::operator<; +}; + +static_assert(sizeof(Float16_t) == sizeof(uint16_t), "Sizes must match"); + +/** \brief bfloat16 (Brain Floating Point) data type + * + * \details This struct is used for converting float to bfloat16 and back + * so the user could feed inputs and fetch outputs using these type. + * + * The size of the structure should align with uint16_t and one can freely cast + * uint16_t buffers to/from Ort::BFloat16_t to feed and retrieve data. + * + * \code{.unparsed} + * // This example demonstrates conversion from float to float16 + * constexpr float values[] = {1.f, 2.f, 3.f, 4.f, 5.f}; + * std::vector bfp16_values; + * bfp16_values.reserve(std::size(values)); + * std::transform(std::begin(values), std::end(values), std::back_inserter(bfp16_values), + * [](float value) { return Ort::BFloat16_t(value); }); + * + * \endcode + */ +struct BFloat16_t : onnxruntime_float16::BFloat16Impl { + private: + /// + /// Constructor from a uint16_t representation of bfloat16 + /// used in FromBits() to escape overload resolution issue with + /// constructor from float. + /// No conversion is done. + /// + /// 16-bit bfloat16 value + constexpr explicit BFloat16_t(uint16_t v) noexcept { val = v; } + + public: + using Base = onnxruntime_float16::BFloat16Impl; + + BFloat16_t() = default; + + /// + /// Explicit conversion to uint16_t representation of bfloat16. + /// + /// uint16_t bit representation of bfloat16 + /// new instance of BFloat16_t + static constexpr BFloat16_t FromBits(uint16_t v) noexcept { return BFloat16_t(v); } + + /// + /// __ctor from float. Float is converted into bfloat16 16-bit representation. + /// + /// float value + explicit BFloat16_t(float v) noexcept { val = Base::ToUint16Impl(v); } + + /// + /// Converts bfloat16 to float + /// + /// float representation of bfloat16 value + float ToFloat() const noexcept { return Base::ToFloatImpl(); } + + /// + /// Checks if the value is negative + /// + /// true if negative + using Base::IsNegative; + + /// + /// Tests if the value is NaN + /// + /// true if NaN + using Base::IsNaN; + + /// + /// Tests if the value is finite + /// + /// true if finite + using Base::IsFinite; + + /// + /// Tests if the value represents positive infinity. + /// + /// true if positive infinity + using Base::IsPositiveInfinity; + + /// + /// Tests if the value represents negative infinity + /// + /// true if negative infinity + using Base::IsNegativeInfinity; + + /// + /// Tests if the value is either positive or negative infinity. + /// + /// True if absolute value is infinity + using Base::IsInfinity; + + /// + /// Tests if the value is NaN or zero. Useful for comparisons. + /// + /// True if NaN or zero. + using Base::IsNaNOrZero; + + /// + /// Tests if the value is normal (not zero, subnormal, infinite, or NaN). + /// + /// True if so + using Base::IsNormal; + + /// + /// Tests if the value is subnormal (denormal). + /// + /// True if so + using Base::IsSubnormal; + + /// + /// Creates an instance that represents absolute value. + /// + /// Absolute value + using Base::Abs; + + /// + /// Creates a new instance with the sign flipped. + /// + /// Flipped sign instance + using Base::Negate; + + /// + /// IEEE defines that positive and negative zero are equal, this gives us a quick equality check + /// for two values by or'ing the private bits together and stripping the sign. They are both zero, + /// and therefore equivalent, if the resulting value is still zero. + /// + /// first value + /// second value + /// True if both arguments represent zero + using Base::AreZero; + + /// + /// User defined conversion operator. Converts BFloat16_t to float. + /// + explicit operator float() const noexcept { return ToFloat(); } + + // We do not have an inherited impl for the below operators + // as the internal class implements them a little differently + bool operator==(const BFloat16_t& rhs) const noexcept; + bool operator!=(const BFloat16_t& rhs) const noexcept { return !(*this == rhs); } + bool operator<(const BFloat16_t& rhs) const noexcept; +}; + +static_assert(sizeof(BFloat16_t) == sizeof(uint16_t), "Sizes must match"); + +/** \brief float8e4m3fn (Float8 Floating Point) data type + * \details It is necessary for type dispatching to make use of C++ API + * The type is implicitly convertible to/from uint8_t. + * See https://onnx.ai/onnx/technical/float8.html for further details. + */ +struct Float8E4M3FN_t { + uint8_t value; + constexpr Float8E4M3FN_t() noexcept : value(0) {} + constexpr Float8E4M3FN_t(uint8_t v) noexcept : value(v) {} + constexpr operator uint8_t() const noexcept { return value; } + // nan values are treated like any other value for operator ==, != + constexpr bool operator==(const Float8E4M3FN_t& rhs) const noexcept { return value == rhs.value; }; + constexpr bool operator!=(const Float8E4M3FN_t& rhs) const noexcept { return value != rhs.value; }; +}; + +static_assert(sizeof(Float8E4M3FN_t) == sizeof(uint8_t), "Sizes must match"); + +/** \brief float8e4m3fnuz (Float8 Floating Point) data type + * \details It is necessary for type dispatching to make use of C++ API + * The type is implicitly convertible to/from uint8_t. + * See https://onnx.ai/onnx/technical/float8.html for further details. + */ +struct Float8E4M3FNUZ_t { + uint8_t value; + constexpr Float8E4M3FNUZ_t() noexcept : value(0) {} + constexpr Float8E4M3FNUZ_t(uint8_t v) noexcept : value(v) {} + constexpr operator uint8_t() const noexcept { return value; } + // nan values are treated like any other value for operator ==, != + constexpr bool operator==(const Float8E4M3FNUZ_t& rhs) const noexcept { return value == rhs.value; }; + constexpr bool operator!=(const Float8E4M3FNUZ_t& rhs) const noexcept { return value != rhs.value; }; +}; + +static_assert(sizeof(Float8E4M3FNUZ_t) == sizeof(uint8_t), "Sizes must match"); + +/** \brief float8e5m2 (Float8 Floating Point) data type + * \details It is necessary for type dispatching to make use of C++ API + * The type is implicitly convertible to/from uint8_t. + * See https://onnx.ai/onnx/technical/float8.html for further details. + */ +struct Float8E5M2_t { + uint8_t value; + constexpr Float8E5M2_t() noexcept : value(0) {} + constexpr Float8E5M2_t(uint8_t v) noexcept : value(v) {} + constexpr operator uint8_t() const noexcept { return value; } + // nan values are treated like any other value for operator ==, != + constexpr bool operator==(const Float8E5M2_t& rhs) const noexcept { return value == rhs.value; }; + constexpr bool operator!=(const Float8E5M2_t& rhs) const noexcept { return value != rhs.value; }; +}; + +static_assert(sizeof(Float8E5M2_t) == sizeof(uint8_t), "Sizes must match"); + +/** \brief float8e5m2fnuz (Float8 Floating Point) data type + * \details It is necessary for type dispatching to make use of C++ API + * The type is implicitly convertible to/from uint8_t. + * See https://onnx.ai/onnx/technical/float8.html for further details. + */ +struct Float8E5M2FNUZ_t { + uint8_t value; + constexpr Float8E5M2FNUZ_t() noexcept : value(0) {} + constexpr Float8E5M2FNUZ_t(uint8_t v) noexcept : value(v) {} + constexpr operator uint8_t() const noexcept { return value; } + // nan values are treated like any other value for operator ==, != + constexpr bool operator==(const Float8E5M2FNUZ_t& rhs) const noexcept { return value == rhs.value; }; + constexpr bool operator!=(const Float8E5M2FNUZ_t& rhs) const noexcept { return value != rhs.value; }; +}; + +static_assert(sizeof(Float8E5M2FNUZ_t) == sizeof(uint8_t), "Sizes must match"); + +namespace detail { +// This is used internally by the C++ API. This macro is to make it easy to generate overloaded methods for all of the various OrtRelease* functions for every Ort* type +// This can't be done in the C API since C doesn't have function overloading. +#define ORT_DEFINE_RELEASE(NAME) \ + inline void OrtRelease(Ort##NAME* ptr) { GetApi().Release##NAME(ptr); } + +#define ORT_DEFINE_RELEASE_FROM_API_STRUCT(NAME, API_GETTER) \ + inline void OrtRelease(Ort##NAME* ptr) { API_GETTER().Release##NAME(ptr); } + +ORT_DEFINE_RELEASE(Allocator); +ORT_DEFINE_RELEASE(ArenaCfg); +ORT_DEFINE_RELEASE(CustomOpDomain); +ORT_DEFINE_RELEASE(Env); +ORT_DEFINE_RELEASE(ExternalInitializerInfo); +ORT_DEFINE_RELEASE(Graph); +ORT_DEFINE_RELEASE(IoBinding); +ORT_DEFINE_RELEASE(KernelInfo); +ORT_DEFINE_RELEASE(KeyValuePairs); +ORT_DEFINE_RELEASE(LoraAdapter); +ORT_DEFINE_RELEASE(MemoryInfo); +ORT_DEFINE_RELEASE(MapTypeInfo); +ORT_DEFINE_RELEASE(Model); +ORT_DEFINE_RELEASE(ModelMetadata); +ORT_DEFINE_RELEASE(Node); +ORT_DEFINE_RELEASE(Op); +ORT_DEFINE_RELEASE(OpAttr); +ORT_DEFINE_RELEASE(PrepackedWeightsContainer); +ORT_DEFINE_RELEASE(RunOptions); +ORT_DEFINE_RELEASE(Session); +ORT_DEFINE_RELEASE(SessionOptions); +ORT_DEFINE_RELEASE(SequenceTypeInfo); +ORT_DEFINE_RELEASE(Status); +ORT_DEFINE_RELEASE(SyncStream); +ORT_DEFINE_RELEASE(TensorTypeAndShapeInfo); +ORT_DEFINE_RELEASE(ThreadingOptions); +ORT_DEFINE_RELEASE(TypeInfo); +ORT_DEFINE_RELEASE(Value); +ORT_DEFINE_RELEASE(ValueInfo); + +ORT_DEFINE_RELEASE_FROM_API_STRUCT(ModelCompilationOptions, GetCompileApi); +ORT_DEFINE_RELEASE_FROM_API_STRUCT(EpDevice, GetEpApi); +ORT_DEFINE_RELEASE_FROM_API_STRUCT(KernelDef, GetEpApi); +ORT_DEFINE_RELEASE_FROM_API_STRUCT(KernelDefBuilder, GetEpApi); +ORT_DEFINE_RELEASE_FROM_API_STRUCT(KernelRegistry, GetEpApi); + +// This is defined explicitly since OrtTensorRTProviderOptionsV2 is not a C API type, +// but the struct has V2 in its name to indicate that it is the second version of the options. +inline void OrtRelease(OrtTensorRTProviderOptionsV2* ptr) { GetApi().ReleaseTensorRTProviderOptions(ptr); } +inline void OrtRelease(OrtCUDAProviderOptionsV2* ptr) { GetApi().ReleaseCUDAProviderOptions(ptr); } + +#undef ORT_DEFINE_RELEASE +#undef ORT_DEFINE_RELEASE_FROM_API_STRUCT + +/** \brief This is a tagging template type. Use it with Base to indicate that the C++ interface object + * has no ownership of the underlying C object. + */ +template +struct Unowned { + using Type = T; +}; + +/** \brief Used internally by the C++ API. C++ wrapper types inherit from this. + * This is a zero cost abstraction to wrap the C API objects and delete them on destruction. + * + * All of the C++ classes + * a) serve as containers for pointers to objects that are created by the underlying C API. + * Their size is just a pointer size, no need to dynamically allocate them. Use them by value. + * b) Each of struct XXXX, XXX instances function as smart pointers to the underlying C API objects. + * they would release objects owned automatically when going out of scope, they are move-only. + * c) ConstXXXX and UnownedXXX structs function as non-owning, copyable containers for the above pointers. + * ConstXXXX allow calling const interfaces only. They give access to objects that are owned by somebody else + * such as Onnxruntime or instances of XXXX classes. + * d) serve convenient interfaces that return C++ objects and further enhance exception and type safety so they can be used + * in C++ code. + * + */ + +/// +/// This is a non-const pointer holder that is move-only. Disposes of the pointer on destruction. +/// +template +struct Base { + using contained_type = T; + + constexpr Base() = default; + constexpr explicit Base(contained_type* p) noexcept : p_{p} {} + ~Base() { + OrtRelease(p_); + } + + Base(const Base&) = delete; + Base& operator=(const Base&) = delete; + + Base(Base&& v) noexcept : p_{v.p_} { v.p_ = nullptr; } + Base& operator=(Base&& v) noexcept { + OrtRelease(p_); + p_ = v.release(); + return *this; + } + + constexpr operator contained_type*() const noexcept { return p_; } + constexpr contained_type& operator*() const noexcept { return *p_; } + + /// \brief Relinquishes ownership of the contained C object pointer + /// The underlying object is not destroyed + contained_type* release() { + T* p = p_; + p_ = nullptr; + return p; + } + + protected: + contained_type* p_{}; +}; + +// Undefined. For const types use Base> +template +struct Base; + +/// +/// Covers unowned pointers owned by either the ORT +/// or some other instance of CPP wrappers. +/// Used for ConstXXX and UnownedXXXX types that are copyable. +/// Also convenient to wrap raw OrtXX pointers . +/// +/// +template +struct Base> { + using contained_type = typename Unowned::Type; + + constexpr Base() = default; + constexpr explicit Base(contained_type* p) noexcept : p_{p} {} + + ~Base() = default; + + Base(const Base&) = default; + Base& operator=(const Base&) = default; + + Base(Base&& v) noexcept : p_{v.p_} { v.p_ = nullptr; } + Base& operator=(Base&& v) noexcept { + p_ = nullptr; + std::swap(p_, v.p_); + return *this; + } + + constexpr operator contained_type*() const noexcept { return p_; } + constexpr contained_type& operator*() const noexcept { return *p_; } + + protected: + contained_type* p_{}; +}; + +// Light functor to release memory with OrtAllocator +struct AllocatedFree { + OrtAllocator* allocator_; + explicit AllocatedFree(OrtAllocator* allocator) + : allocator_(allocator) {} + void operator()(void* ptr) const { + if (ptr) allocator_->Free(allocator_, ptr); + } +}; + +} // namespace detail + +struct AllocatorWithDefaultOptions; +struct Env; +struct EpDevice; +struct ExternalInitializerInfo; +struct Graph; +struct Model; +struct Node; +struct ModelMetadata; +struct TypeInfo; +struct PrepackedWeightsContainer; +struct Session; +struct SessionOptions; +struct SyncStream; +struct TensorRTProviderOptions; +struct Value; +struct ValueInfo; + +/** \brief unique_ptr typedef used to own strings allocated by OrtAllocators + * and release them at the end of the scope. The lifespan of the given allocator + * must eclipse the lifespan of AllocatedStringPtr instance + */ +using AllocatedStringPtr = std::unique_ptr; + +/** \brief The Status that holds ownership of OrtStatus received from C API + * Use it to safely destroy OrtStatus* returned from the C API. Use appropriate + * constructors to construct an instance of a Status object from exceptions. + */ +struct Status : detail::Base { + Status() = default; // Same as with std::nullptr_t. But can be used in re-sizable containers and represent success. + explicit Status(std::nullptr_t) noexcept {} ///< Create an empty object, must be assigned a valid one to be used + explicit Status(OrtStatus* status) noexcept; ///< Takes ownership of OrtStatus instance returned from the C API. + explicit Status(const Exception&); ///< Creates status instance out of exception + explicit Status(const std::exception&); ///< Creates status instance out of exception + Status(const char* message, OrtErrorCode code); ///< Creates status instance out of null-terminated string message. + std::string GetErrorMessage() const; + OrtErrorCode GetErrorCode() const; + bool IsOK() const noexcept; ///< Returns true if instance represents an OK (non-error) status. +}; + +/** \brief The ThreadingOptions + * + * The ThreadingOptions used for set global threadpools' options of The Env. + */ +struct ThreadingOptions : detail::Base { + /// \brief Wraps OrtApi::CreateThreadingOptions + ThreadingOptions(); + + /// \brief Wraps OrtApi::SetGlobalIntraOpNumThreads + ThreadingOptions& SetGlobalIntraOpNumThreads(int intra_op_num_threads); + + /// \brief Wraps OrtApi::SetGlobalInterOpNumThreads + ThreadingOptions& SetGlobalInterOpNumThreads(int inter_op_num_threads); + + /// \brief Wraps OrtApi::SetGlobalSpinControl + ThreadingOptions& SetGlobalSpinControl(int allow_spinning); + + /// \brief Wraps OrtApi::SetGlobalDenormalAsZero + ThreadingOptions& SetGlobalDenormalAsZero(); + + /// \brief Wraps OrtApi::SetGlobalCustomCreateThreadFn + ThreadingOptions& SetGlobalCustomCreateThreadFn(OrtCustomCreateThreadFn ort_custom_create_thread_fn); + + /// \brief Wraps OrtApi::SetGlobalCustomThreadCreationOptions + ThreadingOptions& SetGlobalCustomThreadCreationOptions(void* ort_custom_thread_creation_options); + + /// \brief Wraps OrtApi::SetGlobalCustomJoinThreadFn + ThreadingOptions& SetGlobalCustomJoinThreadFn(OrtCustomJoinThreadFn ort_custom_join_thread_fn); +}; + +/** \brief The TensorRTOptions (V2) + * + * Used to pass options to TRT EP + */ +struct TensorRTProviderOptions : detail::Base { + TensorRTProviderOptions(std::nullptr_t) {} + /// \brief Wraps OrtApi::CreateTensorRTProviderOptionsV2 + TensorRTProviderOptions(); + ///< Wrapper around OrtApi::UpdateTensorRTProviderOptions + void Update(const std::unordered_map& options); + ///< Wrapper around OrtApi::UpdateTensorRTProviderOptions + void UpdateWithValue(const char* key, void* value); + + ///< Wrapper around OrtApi::GetTensorRTProviderOptionsByName + void* GetOptionByName(const char* name) const; + ///< Wrapper around OrtApi::GetTensorRTProviderOptionsAsString + std::string GetTensorRTProviderOptionsAsString() const; +}; + +/** \brief The CUDAProviderOptions (V2) + * + * Used to pass options to CUDA EP + */ +struct CUDAProviderOptions : detail::Base { + CUDAProviderOptions(std::nullptr_t) {} + /// \brief Wraps OrtApi::CreateCUDAProviderOptions + CUDAProviderOptions(); + ///< Wrapper around OrtApi::UpdateCUDAProviderOptions + void Update(const std::unordered_map& options); + ///< Wrapper around OrtApi::GetCUDAProviderOptionsAsString + std::string GetCUDAProviderOptionsAsString() const; + ///< Wrapper around OrtApi::UpdateCUDAProviderOptionsWithValue + void UpdateWithValue(const char* key, void* value); + ///< Wrapper around OrtApi::GetCUDAProviderOptionsByName + void* GetOptionByName(const char* name) const; +}; + +/** \brief The PrepackedWeightsContainer + * + * Create only and pass to Ort::Session constructor for multiple sessions + * to share pre-packed weights. + */ +struct PrepackedWeightsContainer : detail::Base { + using Base = detail::Base; + ///< No instance is created + explicit PrepackedWeightsContainer(std::nullptr_t) {} + ///< Take ownership of a pointer created by C API + explicit PrepackedWeightsContainer(OrtPrepackedWeightsContainer* p) : Base{p} {} + /// \brief Wraps OrtApi::CreatePrepackedWeightsContainer + PrepackedWeightsContainer(); +}; + +namespace detail { +template +struct ConstExternalInitializerInfoImpl : Base { + using B = Base; + using B::B; + + // Wraps OrtApi::ExternalInitializerInfo_GetFilePath + const std::basic_string GetFilePath() const; + // Wraps OrtApi::ExternalInitializerInfo_GetFileOffset + int64_t GetFileOffset() const; + // Wraps OrtApi::ExternalInitializerInfo_GetByteSize + size_t GetByteSize() const; +}; +} // namespace detail + +// Const object holder that does not own the underlying object +using ConstExternalInitializerInfo = + detail::ConstExternalInitializerInfoImpl>; + +/** \brief Wrapper around ::OrtExternalInitializerInfo + * + */ +struct ExternalInitializerInfo : detail::ConstExternalInitializerInfoImpl { + using Base = detail::ConstExternalInitializerInfoImpl; + using Base::Base; + + explicit ExternalInitializerInfo(std::nullptr_t) {} + explicit ExternalInitializerInfo(OrtExternalInitializerInfo* p) + : detail::ConstExternalInitializerInfoImpl{p} {} + + ConstExternalInitializerInfo GetConst() const { return ConstExternalInitializerInfo{this->p_}; } + + ///< Wraps OrtApi::CreateExternalInitializerInfo + ExternalInitializerInfo(const ORTCHAR_T* filepath, int64_t file_offset, size_t byte_size); + + ///< Wrapper around CreateExternalInitializerInfo that does not throw an exception. + static Status Create(const ORTCHAR_T* filepath, int64_t file_offset, size_t byte_size, + /*out*/ ExternalInitializerInfo& out); +}; + +namespace detail { +template +struct KeyValuePairsImpl : Ort::detail::Base { + using B = Ort::detail::Base; + using B::B; + + const char* GetValue(const char* key) const; + + // get the pairs in unordered_map. needs to copy to std::string so the hash works as expected + std::unordered_map GetKeyValuePairs() const; + // get the pairs in two vectors. entries will be 1:1 between keys and values. avoids copying to std::string + void GetKeyValuePairs(std::vector& keys, std::vector& values) const; +}; +} // namespace detail + +// Const object holder that does not own the underlying object +using ConstKeyValuePairs = detail::KeyValuePairsImpl>; + +/** \brief Wrapper around ::OrtKeyValuePairs */ +struct KeyValuePairs : detail::KeyValuePairsImpl { + explicit KeyValuePairs(std::nullptr_t) {} ///< No instance is created + /// Take ownership of a pointer created by C API + explicit KeyValuePairs(OrtKeyValuePairs* p) : KeyValuePairsImpl{p} {} + + /// \brief Wraps OrtApi::CreateKeyValuePairs + explicit KeyValuePairs(); + + /// \brief Wraps OrtApi::CreateKeyValuePairs and OrtApi::AddKeyValuePair + explicit KeyValuePairs(const std::unordered_map& kv_pairs); + + /// \brief Wraps OrtApi::AddKeyValuePair + void Add(const char* key, const char* value); + + /// \brief Wraps OrtApi::RemoveKeyValuePair + void Remove(const char* key); + + ConstKeyValuePairs GetConst() const { return ConstKeyValuePairs{this->p_}; } +}; + +namespace detail { +template +struct MemoryInfoImpl : Base { + using B = Base; + using B::B; + + std::string GetAllocatorName() const; ///< Wrapper MemoryInfoGetName + OrtAllocatorType GetAllocatorType() const; ///< Wrapper MemoryInfoGetType + int GetDeviceId() const; ///< Wrapper MemoryInfoGetId + OrtMemoryInfoDeviceType GetDeviceType() const; ///< Wrapper MemoryInfoGetDeviceType + OrtMemType GetMemoryType() const; ///< Wrapper MemoryInfoGetMemType + OrtDeviceMemoryType GetDeviceMemoryType() const; ///< Wrapper MemoryInfoGetDeviceMemType + uint32_t GetVendorId() const; ///< Wrapper MemoryInfoGetVendorId + + template + bool operator==(const MemoryInfoImpl& o) const; +}; +} // namespace detail + +// Const object holder that does not own the underlying object +using ConstMemoryInfo = detail::MemoryInfoImpl>; + +/** \brief Wrapper around ::OrtMemoryInfo + * + */ +struct MemoryInfo : detail::MemoryInfoImpl { + static MemoryInfo CreateCpu(OrtAllocatorType type, OrtMemType mem_type1); + explicit MemoryInfo(std::nullptr_t) {} ///< No instance is created + explicit MemoryInfo(OrtMemoryInfo* p) : MemoryInfoImpl{p} {} ///< Take ownership of a pointer created by C API + MemoryInfo(const char* name, OrtAllocatorType type, int id, OrtMemType mem_type); + MemoryInfo(const char* name, OrtMemoryInfoDeviceType device_type, uint32_t vendor_id, uint32_t device_id, + OrtDeviceMemoryType mem_type, size_t alignment, OrtAllocatorType allocator_type); ///< Wrapper around CreateMemoryInfo_V2 + ConstMemoryInfo GetConst() const { return ConstMemoryInfo{this->p_}; } +}; + +/// +/// Represents native memory allocation coming from one of the +/// OrtAllocators registered with OnnxRuntime. +/// Use it to wrap an allocation made by an allocator +/// so it can be automatically released when no longer needed. +/// +struct MemoryAllocation { + MemoryAllocation(OrtAllocator* allocator, void* p, size_t size); + ~MemoryAllocation(); + MemoryAllocation(const MemoryAllocation&) = delete; + MemoryAllocation& operator=(const MemoryAllocation&) = delete; + MemoryAllocation(MemoryAllocation&&) noexcept; + MemoryAllocation& operator=(MemoryAllocation&&) noexcept; + + void* get() { return p_; } + size_t size() const { return size_; } + + private: + OrtAllocator* allocator_; + void* p_; + size_t size_; +}; + +namespace detail { +template +struct AllocatorImpl : Base { + using B = Base; + using B::B; + + void* Alloc(size_t size); + MemoryAllocation GetAllocation(size_t size); + void Free(void* p); + ConstMemoryInfo GetInfo() const; + + /** \brief Function that returns the statistics of the allocator. + * + * \return A pointer to a KeyValuePairs object that will be filled with the allocator statistics. + */ + KeyValuePairs GetStats() const; +}; +} // namespace detail + +/** \brief Wrapper around ::OrtAllocator default instance that is owned by Onnxruntime + * + */ +struct AllocatorWithDefaultOptions : detail::AllocatorImpl> { + explicit AllocatorWithDefaultOptions(std::nullptr_t) {} ///< Convenience to create a class member and then replace with an instance + AllocatorWithDefaultOptions(); +}; + +/** \brief Wrapper around ::OrtAllocator + * + */ + +struct Allocator : detail::AllocatorImpl { + explicit Allocator(std::nullptr_t) {} ///< Convenience to create a class member and then replace with an instance + Allocator(const Session& session, const OrtMemoryInfo*); + + ///< Take ownership of a pointer created by C API + explicit Allocator(OrtAllocator* p) : AllocatorImpl{p} {} +}; + +using UnownedAllocator = detail::AllocatorImpl>; + +/** \brief Wrapper around ::OrtSyncStream + * + */ + +namespace detail { +template +struct SyncStreamImpl : Base { + using B = Base; + using B::B; + // For some reason this is not a const method on the stream + void* GetHandle(); ///< Wraps SyncStream_GetHandle +}; +} // namespace detail + +struct SyncStream : detail::SyncStreamImpl { + ///< Create an empty SyncStream object, must be assigned a valid one to be used + explicit SyncStream(std::nullptr_t) {} + ///< Take ownership of a pointer created by C API + explicit SyncStream(OrtSyncStream* p) : SyncStreamImpl{p} {} +}; + +using UnownedSyncStream = detail::SyncStreamImpl>; + +namespace detail { +template +struct HardwareDeviceImpl : Ort::detail::Base { + using B = Ort::detail::Base; + using B::B; + + OrtHardwareDeviceType Type() const; + uint32_t VendorId() const; + uint32_t DeviceId() const; + const char* Vendor() const; + ConstKeyValuePairs Metadata() const; +}; +} // namespace detail + +/** \brief Wrapper around ::OrtHardwareDevice + * \remarks HardwareDevice is always read-only for API users. + */ +using ConstHardwareDevice = detail::HardwareDeviceImpl>; + +namespace detail { +template +struct EpDeviceImpl : Ort::detail::Base { + using B = Ort::detail::Base; + using B::B; + + const char* EpName() const; + const char* EpVendor() const; + ConstKeyValuePairs EpMetadata() const; + ConstKeyValuePairs EpOptions() const; + ConstHardwareDevice Device() const; + ConstMemoryInfo GetMemoryInfo(OrtDeviceMemoryType memory_type) const; ///< Wraps EpDevice_MemoryInfo + SyncStream CreateSyncStream(ConstKeyValuePairs stream_options = {}) const; /// Wraps EpDevice_CreateSyncStream +}; +} // namespace detail + +/** \brief Wrapper around ::OrtEpDevice + * \remarks EpDevice is always read-only for ORT API users. + */ +using ConstEpDevice = detail::EpDeviceImpl>; + +/** \brief Mutable EpDevice that is created by EpApi users. + */ +struct EpDevice : detail::EpDeviceImpl { + explicit EpDevice(std::nullptr_t) {} ///< No instance is created + explicit EpDevice(OrtEpDevice* p) : EpDeviceImpl{p} {} ///< Take ownership of a pointer created by C API + + /// \brief Wraps OrtEpApi::CreateEpDevice + EpDevice(OrtEpFactory& ep_factory, ConstHardwareDevice& hardware_device, + ConstKeyValuePairs ep_metadata = {}, ConstKeyValuePairs ep_options = {}); +}; + +/** \brief Validate a compiled model's compatibility for one or more EP devices. + * + * Throws on error. Returns the resulting compatibility status. + * /// \param ep_devices The EP devices to check compatibility against. + * /// \param compatibility_info The compatibility string from the precompiled model to validate. + */ +OrtCompiledModelCompatibility GetModelCompatibilityForEpDevices( + const std::vector& ep_devices, + const char* compatibility_info); + +/** \brief Extract EP compatibility info from a precompiled model file. + * + * Parses the model file to extract the compatibility info string for a specific execution provider + * from the model's metadata properties. This is only applicable to models that have been precompiled + * for an EP. Standard ONNX models do not contain this information. + * + * \note This operation parses the full ONNX ModelProto from disk. + * + * \param model_path Path to the ONNX model file. + * \param ep_type The execution provider type string. Must be non-empty. + * Use ConstEpDevice::EpName() to get this value. + * \param allocator Allocator to use for the output string. + * \return The compatibility info string, or nullptr if not found for this EP. Caller must free via allocator. + * \throws Ort::Exception on error. + */ +AllocatedStringPtr GetCompatibilityInfoFromModelAllocated(const ORTCHAR_T* model_path, const char* ep_type, + OrtAllocator* allocator); + +/** \brief Extract EP compatibility info from precompiled model bytes in memory. + * + * Same as GetCompatibilityInfoFromModelAllocated but reads from a memory buffer. + * Useful when precompiled models are loaded from encrypted storage, network, or other non-file sources. + * + * \param model_data Pointer to the model data in memory. + * \param model_data_length Size of the model data in bytes. + * \param ep_type The execution provider type string. Must be non-empty. + * \param allocator Allocator to use for the output string. + * \return The compatibility info string, or nullptr if not found for this EP. Caller must free via allocator. + * \throws Ort::Exception on error. + */ +AllocatedStringPtr GetCompatibilityInfoFromModelBytesAllocated(const void* model_data, size_t model_data_length, + const char* ep_type, OrtAllocator* allocator); + +namespace detail { +template +struct EpAssignedNodeImpl : Ort::detail::Base { + using B = Ort::detail::Base; + using B::B; + + std::string GetName() const; + std::string GetDomain() const; + std::string GetOperatorType() const; +}; +} // namespace detail + +/** \brief Constant wrapper around ::OrtEpAssignedNode + * \remarks EpAssignedNode is always read-only for ORT API users. + */ +using ConstEpAssignedNode = detail::EpAssignedNodeImpl>; + +namespace detail { +template +struct EpAssignedSubgraphImpl : Ort::detail::Base { + using B = Ort::detail::Base; + using B::B; + + std::string GetEpName() const; + std::vector GetNodes() const; +}; +} // namespace detail + +/** \brief Constant wrapper around ::OrtEpAssignedSubgraph + * \remarks EpAssignedSubgraph is always read-only for ORT API users. + */ +using ConstEpAssignedSubgraph = detail::EpAssignedSubgraphImpl>; + +/** \brief The Env (Environment) + * + * The Env holds the logging state used by all other objects. + * Note: One Env must be created before using any other Onnxruntime functionality + */ +struct Env : detail::Base { + explicit Env(std::nullptr_t) {} ///< Create an empty Env object, must be assigned a valid one to be used + + /// \brief Wraps OrtApi::CreateEnv + Env(OrtLoggingLevel logging_level = ORT_LOGGING_LEVEL_WARNING, _In_ const char* logid = ""); + + /// \brief Wraps OrtApi::CreateEnvWithCustomLogger + Env(OrtLoggingLevel logging_level, const char* logid, OrtLoggingFunction logging_function, void* logger_param); + + /// \brief Wraps OrtApi::CreateEnvWithGlobalThreadPools + Env(const OrtThreadingOptions* tp_options, OrtLoggingLevel logging_level = ORT_LOGGING_LEVEL_WARNING, _In_ const char* logid = ""); + + /// \brief Wraps OrtApi::CreateEnvWithCustomLoggerAndGlobalThreadPools + Env(const OrtThreadingOptions* tp_options, OrtLoggingFunction logging_function, void* logger_param, + OrtLoggingLevel logging_level = ORT_LOGGING_LEVEL_WARNING, _In_ const char* logid = ""); + + /// \brief Wraps OrtApi::CreateEnvWithOptions + explicit Env(const OrtEnvCreationOptions* options); + + /// \brief C Interop Helper + explicit Env(OrtEnv* p) : Base{p} {} + + Env& EnableTelemetryEvents(); ///< Wraps OrtApi::EnableTelemetryEvents + Env& DisableTelemetryEvents(); ///< Wraps OrtApi::DisableTelemetryEvents + + Env& UpdateEnvWithCustomLogLevel(OrtLoggingLevel log_severity_level); ///< Wraps OrtApi::UpdateEnvWithCustomLogLevel + + Env& CreateAndRegisterAllocator(const OrtMemoryInfo* mem_info, const OrtArenaCfg* arena_cfg); ///< Wraps OrtApi::CreateAndRegisterAllocator + + Env& CreateAndRegisterAllocatorV2(const std::string& provider_type, const OrtMemoryInfo* mem_info, + const std::unordered_map& options, + const OrtArenaCfg* arena_cfg); ///< Wraps OrtApi::CreateAndRegisterAllocatorV2 + + Env& RegisterAllocator(OrtAllocator* allocator); ///< Wraps OrtApi::RegisterAllocator + + Env& UnregisterAllocator(const OrtMemoryInfo* mem_info); ///< Wraps OrtApi::UnregisterAllocator + + UnownedAllocator CreateSharedAllocator(const OrtEpDevice* ep_device, OrtDeviceMemoryType mem_type, + OrtAllocatorType allocator_type, + const OrtKeyValuePairs* allocator_options); ///< Wraps OrtApi::CreateSharedAllocator + + // Result may be nullptr + UnownedAllocator GetSharedAllocator(const OrtMemoryInfo* mem_info); ///< Wraps OrtApi::GetSharedAllocator + + void ReleaseSharedAllocator(const OrtEpDevice* ep_device, + OrtDeviceMemoryType mem_type); ///< Wraps OrtApi::ReleaseSharedAllocator + + Env& RegisterExecutionProviderLibrary(const char* registration_name, const std::basic_string& path); ///< Wraps OrtApi::RegisterExecutionProviderLibrary + Env& UnregisterExecutionProviderLibrary(const char* registration_name); ///< Wraps OrtApi::UnregisterExecutionProviderLibrary + + std::vector GetEpDevices() const; + + Status CopyTensors(const std::vector& src_tensors, + const std::vector& dst_tensors, + OrtSyncStream* stream) const; ///< Wraps OrtApi::CopyTensors +}; + +/** \brief Custom Op Domain + * + */ +struct CustomOpDomain : detail::Base { + using Base = detail::Base; + using Base::Base; + + explicit CustomOpDomain(std::nullptr_t) {} ///< Create an empty CustomOpDomain object, must be assigned a valid one to be used + + /// \brief Wraps OrtApi::CreateCustomOpDomain + explicit CustomOpDomain(const char* domain); + + // This does not take ownership of the op, simply registers it. + void Add(const OrtCustomOp* op); ///< Wraps CustomOpDomain_Add +}; + +/// \brief LoraAdapter holds a set of Lora Parameters loaded from a single file +struct LoraAdapter : detail::Base { + using Base = detail::Base; + using Base::Base; + + explicit LoraAdapter(std::nullptr_t) {} ///< Create an empty LoraAdapter object, must be assigned a valid one to be used + /// \brief Wraps OrtApi::CreateLoraAdapter + /// + /// The function attempts to load the adapter from the specified file + /// \param adapter_path The path to the Lora adapter + /// \param allocator optional pointer to a device allocator. If nullptr, the data stays on CPU. It would still + /// be copied to device if required by the model at inference time. + static LoraAdapter CreateLoraAdapter(const std::basic_string& adapter_path, + OrtAllocator* allocator); + + /// \brief Wraps OrtApi::CreateLoraAdapterFromArray + /// + /// The function attempts to load the adapter from the specified byte array. + /// \param bytes The byte array containing file LoraAdapter format + /// \param num_bytes The number of bytes in the byte array + /// \param allocator optional pointer to a device allocator. If nullptr, the data stays on CPU. It would still + /// be copied to device if required by the model at inference time. + static LoraAdapter CreateLoraAdapterFromArray(const void* bytes, size_t num_bytes, + OrtAllocator* allocator); +}; + +/** \brief RunOptions + * + */ +struct RunOptions : detail::Base { + explicit RunOptions(std::nullptr_t) {} ///< Create an empty RunOptions object, must be assigned a valid one to be used + RunOptions(); ///< Wraps OrtApi::CreateRunOptions + + RunOptions& SetRunLogVerbosityLevel(int); ///< Wraps OrtApi::RunOptionsSetRunLogVerbosityLevel + int GetRunLogVerbosityLevel() const; ///< Wraps OrtApi::RunOptionsGetRunLogVerbosityLevel + + RunOptions& SetRunLogSeverityLevel(int); ///< Wraps OrtApi::RunOptionsSetRunLogSeverityLevel + int GetRunLogSeverityLevel() const; ///< Wraps OrtApi::RunOptionsGetRunLogSeverityLevel + + RunOptions& SetRunTag(const char* run_tag); ///< wraps OrtApi::RunOptionsSetRunTag + const char* GetRunTag() const; ///< Wraps OrtApi::RunOptionsGetRunTag + + RunOptions& AddConfigEntry(const char* config_key, const char* config_value); ///< Wraps OrtApi::AddRunConfigEntry + const char* GetConfigEntry(const char* config_key); ///< Wraps OrtApi::GetRunConfigEntry + + /** \brief Terminates all currently executing Session::Run calls that were made using this RunOptions instance + * + * If a currently executing session needs to be force terminated, this can be called from another thread to force it to fail with an error + * Wraps OrtApi::RunOptionsSetTerminate + */ + RunOptions& SetTerminate(); + + /** \brief Clears the terminate flag so this RunOptions instance can be used in a new Session::Run call without it instantly terminating + * + * Wraps OrtApi::RunOptionsUnsetTerminate + */ + RunOptions& UnsetTerminate(); + + /** \brief Add the LoraAdapter to the list of active adapters. + * The setting does not affect RunWithBinding() calls. + * + * Wraps OrtApi::RunOptionsAddActiveLoraAdapter + * \param adapter The LoraAdapter to be used as the active adapter + */ + RunOptions& AddActiveLoraAdapter(const LoraAdapter& adapter); + + /** \brief Associate a sync stream with the run options. + * + * When set, the EP uses this stream for execution, enabling proper + * synchronization with imported external semaphores. Wraps OrtApi::RunOptionsSetSyncStream. + * + * \param stream The OrtSyncStream to associate with these run options. May be nullptr to clear. + */ + RunOptions& SetSyncStream(OrtSyncStream* stream); +}; + +namespace detail { +// Utility function that returns a SessionOption config entry key for a specific custom operator. +// Ex: custom_op.[custom_op_name].[config] +std::string MakeCustomOpConfigEntryKey(const char* custom_op_name, const char* config); +} // namespace detail + +/// +/// Class that represents session configuration entries for one or more custom operators. +/// +/// Example: +/// Ort::CustomOpConfigs op_configs; +/// op_configs.AddConfig("my_custom_op", "device_type", "CPU"); +/// +/// Passed to Ort::SessionOptions::RegisterCustomOpsLibrary. +/// +struct CustomOpConfigs { + CustomOpConfigs() = default; + ~CustomOpConfigs() = default; + CustomOpConfigs(const CustomOpConfigs&) = default; + CustomOpConfigs& operator=(const CustomOpConfigs&) = default; + CustomOpConfigs(CustomOpConfigs&& o) = default; + CustomOpConfigs& operator=(CustomOpConfigs&& o) = default; + + /** \brief Adds a session configuration entry/value for a specific custom operator. + * + * \param custom_op_name The name of the custom operator for which to add a configuration entry. + * Must match the name returned by the CustomOp's GetName() method. + * \param config_key The name of the configuration entry. + * \param config_value The value of the configuration entry. + * \return A reference to this object to enable call chaining. + */ + CustomOpConfigs& AddConfig(const char* custom_op_name, const char* config_key, const char* config_value); + + /** \brief Returns a flattened map of custom operator configuration entries and their values. + * + * The keys has been flattened to include both the custom operator name and the configuration entry key name. + * For example, a prior call to AddConfig("my_op", "key", "value") corresponds to the flattened key/value pair + * {"my_op.key", "value"}. + * + * \return An unordered map of flattened configurations. + */ + const std::unordered_map& GetFlattenedConfigs() const; + + private: + std::unordered_map flat_configs_; +}; + +/** \brief Options object used when creating a new Session object + * + * Wraps ::OrtSessionOptions object and methods + */ + +namespace detail { +// we separate const-only methods because passing const ptr to non-const methods +// is only discovered when inline methods are compiled which is counter-intuitive +template +struct ConstSessionOptionsImpl : Base { + using B = Base; + using B::B; + + SessionOptions Clone() const; ///< Creates and returns a copy of this SessionOptions object. Wraps OrtApi::CloneSessionOptions + + std::string GetConfigEntry(const char* config_key) const; ///< Wraps OrtApi::GetSessionConfigEntry + bool HasConfigEntry(const char* config_key) const; ///< Wraps OrtApi::HasSessionConfigEntry + std::string GetConfigEntryOrDefault(const char* config_key, const std::string& def) const; +}; + +template +struct SessionOptionsImpl : ConstSessionOptionsImpl { + using B = ConstSessionOptionsImpl; + using B::B; + + SessionOptionsImpl& SetIntraOpNumThreads(int intra_op_num_threads); ///< Wraps OrtApi::SetIntraOpNumThreads + SessionOptionsImpl& SetInterOpNumThreads(int inter_op_num_threads); ///< Wraps OrtApi::SetInterOpNumThreads + SessionOptionsImpl& SetGraphOptimizationLevel(GraphOptimizationLevel graph_optimization_level); ///< Wraps OrtApi::SetSessionGraphOptimizationLevel + SessionOptionsImpl& SetDeterministicCompute(bool value); ///< Wraps OrtApi::SetDeterministicCompute + + SessionOptionsImpl& EnableCpuMemArena(); ///< Wraps OrtApi::EnableCpuMemArena + SessionOptionsImpl& DisableCpuMemArena(); ///< Wraps OrtApi::DisableCpuMemArena + + SessionOptionsImpl& SetOptimizedModelFilePath(const ORTCHAR_T* optimized_model_file); ///< Wraps OrtApi::SetOptimizedModelFilePath + + SessionOptionsImpl& EnableProfiling(const ORTCHAR_T* profile_file_prefix); ///< Wraps OrtApi::EnableProfiling + SessionOptionsImpl& DisableProfiling(); ///< Wraps OrtApi::DisableProfiling + + SessionOptionsImpl& EnableOrtCustomOps(); ///< Wraps OrtApi::EnableOrtCustomOps + + SessionOptionsImpl& EnableMemPattern(); ///< Wraps OrtApi::EnableMemPattern + SessionOptionsImpl& DisableMemPattern(); ///< Wraps OrtApi::DisableMemPattern + + SessionOptionsImpl& SetExecutionMode(ExecutionMode execution_mode); ///< Wraps OrtApi::SetSessionExecutionMode + + SessionOptionsImpl& SetLoadCancellationFlag(bool value); ///< Wraps OrtApi::SessionOptionsSetLoadCancellationFlag + + SessionOptionsImpl& SetLogId(const char* logid); ///< Wraps OrtApi::SetSessionLogId + SessionOptionsImpl& SetLogSeverityLevel(int level); ///< Wraps OrtApi::SetSessionLogSeverityLevel + + SessionOptionsImpl& Add(OrtCustomOpDomain* custom_op_domain); ///< Wraps OrtApi::AddCustomOpDomain + + SessionOptionsImpl& DisablePerSessionThreads(); ///< Wraps OrtApi::DisablePerSessionThreads + + SessionOptionsImpl& AddConfigEntry(const char* config_key, const char* config_value); ///< Wraps OrtApi::AddSessionConfigEntry + + SessionOptionsImpl& AddInitializer(const char* name, const OrtValue* ort_val); ///< Wraps OrtApi::AddInitializer + SessionOptionsImpl& AddExternalInitializers(const std::vector& names, const std::vector& ort_values); ///< Wraps OrtApi::AddExternalInitializers + SessionOptionsImpl& AddExternalInitializersFromFilesInMemory(const std::vector>& external_initializer_file_names, + const std::vector& external_initializer_file_buffer_array, + const std::vector& external_initializer_file_lengths); ///< Wraps OrtApi::AddExternalInitializersFromFilesInMemory + + SessionOptionsImpl& AppendExecutionProvider_CPU(int use_arena); ///< Wraps OrtApi::SessionOptionsAppendExecutionProvider_CPU + SessionOptionsImpl& AppendExecutionProvider_CUDA(const OrtCUDAProviderOptions& provider_options); ///< Wraps OrtApi::SessionOptionsAppendExecutionProvider_CUDA + SessionOptionsImpl& AppendExecutionProvider_CUDA_V2(const OrtCUDAProviderOptionsV2& provider_options); ///< Wraps OrtApi::SessionOptionsAppendExecutionProvider_CUDA_V2 + SessionOptionsImpl& AppendExecutionProvider_ROCM(const OrtROCMProviderOptions& provider_options); ///< Wraps OrtApi::SessionOptionsAppendExecutionProvider_ROCM + SessionOptionsImpl& AppendExecutionProvider_OpenVINO(const OrtOpenVINOProviderOptions& provider_options); ///< Wraps OrtApi::SessionOptionsAppendExecutionProvider_OpenVINO + ///< Wraps OrtApi::SessionOptionsAppendExecutionProvider_OpenVINO_V2 + SessionOptionsImpl& AppendExecutionProvider_OpenVINO_V2(const std::unordered_map& provider_options = {}); + SessionOptionsImpl& AppendExecutionProvider_TensorRT(const OrtTensorRTProviderOptions& provider_options); ///< Wraps OrtApi::SessionOptionsAppendExecutionProvider_TensorRT + SessionOptionsImpl& AppendExecutionProvider_TensorRT_V2(const OrtTensorRTProviderOptionsV2& provider_options); ///< Wraps OrtApi::SessionOptionsAppendExecutionProvider_TensorRT + SessionOptionsImpl& AppendExecutionProvider_MIGraphX(const OrtMIGraphXProviderOptions& provider_options); ///< Wraps OrtApi::SessionOptionsAppendExecutionProvider_MIGraphX + /// Wraps OrtApi::SessionOptionsAppendExecutionProvider_CANN + SessionOptionsImpl& AppendExecutionProvider_CANN(const OrtCANNProviderOptions& provider_options); + /// Wraps OrtApi::SessionOptionsAppendExecutionProvider_Dnnl + SessionOptionsImpl& AppendExecutionProvider_Dnnl(const OrtDnnlProviderOptions& provider_options); + /// Wraps OrtApi::SessionOptionsAppendExecutionProvider. Currently supports QNN, SNPE and XNNPACK. + SessionOptionsImpl& AppendExecutionProvider(const std::string& provider_name, + const std::unordered_map& provider_options = {}); + + /// Append EPs that have been registered previously with the OrtEnv. + /// Wraps OrtApi::SessionOptionsAppendExecutionProvider_V2 + SessionOptionsImpl& AppendExecutionProvider_V2(Env& env, const std::vector& ep_devices, + const KeyValuePairs& ep_options); + /// Append EPs that have been registered previously with the OrtEnv. + /// Wraps OrtApi::SessionOptionsAppendExecutionProvider_V2 + SessionOptionsImpl& AppendExecutionProvider_V2(Env& env, const std::vector& ep_devices, + const std::unordered_map& ep_options); + + /// Wraps OrtApi::SessionOptionsSetEpSelectionPolicy + SessionOptionsImpl& SetEpSelectionPolicy(OrtExecutionProviderDevicePolicy policy); + + /// Wraps OrtApi::SessionOptionsSetEpSelectionPolicyDelegate + SessionOptionsImpl& SetEpSelectionPolicy(EpSelectionDelegate delegate, void* state = nullptr); + + SessionOptionsImpl& SetCustomCreateThreadFn(OrtCustomCreateThreadFn ort_custom_create_thread_fn); ///< Wraps OrtApi::SessionOptionsSetCustomCreateThreadFn + SessionOptionsImpl& SetCustomThreadCreationOptions(void* ort_custom_thread_creation_options); ///< Wraps OrtApi::SessionOptionsSetCustomThreadCreationOptions + SessionOptionsImpl& SetCustomJoinThreadFn(OrtCustomJoinThreadFn ort_custom_join_thread_fn); ///< Wraps OrtApi::SessionOptionsSetCustomJoinThreadFn + + ///< Registers the custom operator from the specified shared library via OrtApi::RegisterCustomOpsLibrary_V2. + ///< The custom operator configurations are optional. If provided, custom operator configs are set via + ///< OrtApi::AddSessionConfigEntry. + SessionOptionsImpl& RegisterCustomOpsLibrary(const ORTCHAR_T* library_name, const CustomOpConfigs& custom_op_configs = {}); + + SessionOptionsImpl& RegisterCustomOpsUsingFunction(const char* function_name); ///< Wraps OrtApi::RegisterCustomOpsUsingFunction + + ///< Wraps OrtApi::SessionOptionsAppendExecutionProvider_VitisAI + SessionOptionsImpl& AppendExecutionProvider_VitisAI(const std::unordered_map& provider_options = {}); + + ///< Wraps OrtApi::AddFreeDimensionOverride + SessionOptionsImpl& AddFreeDimensionOverride(const char* dim_denotation, int64_t dim_value); + + ///< Wraps OrtApi::AddFreeDimensionOverrideByName + SessionOptionsImpl& AddFreeDimensionOverrideByName(const char* dim_name, int64_t dim_value); +}; +} // namespace detail + +using UnownedSessionOptions = detail::SessionOptionsImpl>; +using ConstSessionOptions = detail::ConstSessionOptionsImpl>; + +/** \brief Wrapper around ::OrtSessionOptions + * + */ +struct SessionOptions : detail::SessionOptionsImpl { + explicit SessionOptions(std::nullptr_t) {} ///< Create an empty SessionOptions object, must be assigned a valid one to be used + SessionOptions(); ///< Wraps OrtApi::CreateSessionOptions + explicit SessionOptions(OrtSessionOptions* p) : SessionOptionsImpl{p} {} ///< Used for interop with the C API + UnownedSessionOptions GetUnowned() const { return UnownedSessionOptions{this->p_}; } + ConstSessionOptions GetConst() const { return ConstSessionOptions{this->p_}; } +}; + +/** \brief Options object used when compiling a model. + * + * Wraps ::OrtModelCompilationOptions object and methods + */ +struct ModelCompilationOptions : detail::Base { + using Base = detail::Base; + using Base::Base; + + explicit ModelCompilationOptions(std::nullptr_t) {} ///< Create an empty ModelCompilationOptions object, must be assigned a valid one to be used. + + ModelCompilationOptions(const Env& env, const SessionOptions& session_options); ///< Wraps OrtApi::CreateModelCompilationOptionsFromSessionOptions + ModelCompilationOptions(const Env& env, ConstSessionOptions session_options); ///< Wraps OrtApi::CreateModelCompilationOptionsFromSessionOptions + + ModelCompilationOptions& SetInputModelPath(const ORTCHAR_T* input_model_path); ///< Wraps OrtApi::ModelCompilationOptions_SetInputModelPath + ModelCompilationOptions& SetInputModelFromBuffer(const void* input_model_data, + size_t input_model_data_size); ///< Wraps OrtApi::ModelCompilationOptions_SetInputModelFromBuffer + ModelCompilationOptions& SetEpContextEmbedMode(bool embed_ep_context_in_model); ///< Wraps OrtApi::ModelCompilationOptions_SetEpContextEmbedMode + ModelCompilationOptions& SetOutputModelPath(const ORTCHAR_T* output_model_path); ///< Wraps OrtApi::ModelCompilationOptions_SetOutputModelPath + ModelCompilationOptions& SetOutputModelExternalInitializersFile(const ORTCHAR_T* file_path, + size_t initializer_size_threshold); ///< Wraps OrtApi::ModelCompilationOptions_SetOutputModelExternalInitializersFile + + ///< Wraps OrtApi::ModelCompilationOptions_SetOutputModelGetInitializerLocationFunc + ModelCompilationOptions& SetOutputModelGetInitializerLocationFunc( + OrtGetInitializerLocationFunc get_initializer_location_func, + void* state); + + ModelCompilationOptions& SetOutputModelBuffer(OrtAllocator* allocator, void** output_model_buffer_ptr, + size_t* output_model_buffer_size_ptr); ///< Wraps OrtApi::ModelCompilationOptions_SetOutputModelBuffer + + ///< Wraps OrtApi::ModelCompilationOptions_SetOutputModelWriteFunc + ModelCompilationOptions& SetOutputModelWriteFunc(OrtWriteBufferFunc write_func, void* state); + + ModelCompilationOptions& SetEpContextBinaryInformation(const ORTCHAR_T* output_directory, + const ORTCHAR_T* model_name); ///< Wraps OrtApi::ModelCompilationOptions_SetEpContextBinaryInformation + ModelCompilationOptions& SetFlags(uint32_t flags); ///< Wraps OrtApi::ModelCompilationOptions_SetFlags + + ModelCompilationOptions& SetGraphOptimizationLevel(GraphOptimizationLevel graph_optimization_level); ///< Wraps OrtApi::ModelCompilationOptions_SetGraphOptimizationLevel +}; + +/** \brief Compiles an input model to generate a model with EPContext nodes that execute EP-specific kernels. Wraps OrtApi::CompileModels. + * + * \param env: ORT environment object. + * \param model_compilation_options: Compilation options for a model. + * \return A Status indicating success or failure. + */ +Status CompileModel(const Env& env, const ModelCompilationOptions& model_compilation_options); + +/** \brief Wrapper around ::OrtModelMetadata + * + */ +struct ModelMetadata : detail::Base { + using Base = detail::Base; + using Base::Base; + + explicit ModelMetadata(std::nullptr_t) {} ///< Create an empty ModelMetadata object, must be assigned a valid one to be used + + /** \brief Returns a copy of the producer name. + * + * \param allocator to allocate memory for the copy of the name returned + * \return a instance of smart pointer that would deallocate the buffer when out of scope. + * The OrtAllocator instances must be valid at the point of memory release. + */ + AllocatedStringPtr GetProducerNameAllocated(OrtAllocator* allocator) const; ///< Wraps OrtApi::ModelMetadataGetProducerName + + /** \brief Returns a copy of the graph name. + * + * \param allocator to allocate memory for the copy of the name returned + * \return a instance of smart pointer that would deallocate the buffer when out of scope. + * The OrtAllocator instances must be valid at the point of memory release. + */ + AllocatedStringPtr GetGraphNameAllocated(OrtAllocator* allocator) const; ///< Wraps OrtApi::ModelMetadataGetGraphName + + /** \brief Returns a copy of the domain name. + * + * \param allocator to allocate memory for the copy of the name returned + * \return a instance of smart pointer that would deallocate the buffer when out of scope. + * The OrtAllocator instances must be valid at the point of memory release. + */ + AllocatedStringPtr GetDomainAllocated(OrtAllocator* allocator) const; ///< Wraps OrtApi::ModelMetadataGetDomain + + /** \brief Returns a copy of the description. + * + * \param allocator to allocate memory for the copy of the string returned + * \return a instance of smart pointer that would deallocate the buffer when out of scope. + * The OrtAllocator instances must be valid at the point of memory release. + */ + AllocatedStringPtr GetDescriptionAllocated(OrtAllocator* allocator) const; ///< Wraps OrtApi::ModelMetadataGetDescription + + /** \brief Returns a copy of the graph description. + * + * \param allocator to allocate memory for the copy of the string returned + * \return a instance of smart pointer that would deallocate the buffer when out of scope. + * The OrtAllocator instances must be valid at the point of memory release. + */ + AllocatedStringPtr GetGraphDescriptionAllocated(OrtAllocator* allocator) const; ///< Wraps OrtApi::ModelMetadataGetGraphDescription + + /** \brief Returns a vector of copies of the custom metadata keys. + * + * \param allocator to allocate memory for the copy of the string returned + * \return a instance std::vector of smart pointers that would deallocate the buffers when out of scope. + * The OrtAllocator instance must be valid at the point of memory release. + */ + std::vector GetCustomMetadataMapKeysAllocated(OrtAllocator* allocator) const; ///< Wraps OrtApi::ModelMetadataGetCustomMetadataMapKeys + + /** \brief Looks up a value by a key in the Custom Metadata map + * + * \param key zero terminated string key to lookup + * \param allocator to allocate memory for the copy of the string returned + * \return a instance of smart pointer that would deallocate the buffer when out of scope. + * maybe nullptr if key is not found. + * + * The OrtAllocator instances must be valid at the point of memory release. + */ + AllocatedStringPtr LookupCustomMetadataMapAllocated(const char* key, OrtAllocator* allocator) const; ///< Wraps OrtApi::ModelMetadataLookupCustomMetadataMap + + int64_t GetVersion() const; ///< Wraps OrtApi::ModelMetadataGetVersion +}; + +struct IoBinding; + +namespace detail { + +// we separate const-only methods because passing const ptr to non-const methods +// is only discovered when inline methods are compiled which is counter-intuitive +template +struct ConstSessionImpl : Base { + using B = Base; + using B::B; + + size_t GetInputCount() const; ///< Returns the number of model inputs + size_t GetOutputCount() const; ///< Returns the number of model outputs + size_t GetOverridableInitializerCount() const; ///< Returns the number of inputs that have defaults that can be overridden + + std::vector GetInputNames() const; + std::vector GetOutputNames() const; + std::vector GetOverridableInitializerNames() const; + + std::vector GetMemoryInfoForInputs() const; ///< Wrapper for OrtApi::SessionGetMemoryInfoForInputs + std::vector GetMemoryInfoForOutputs() const; ///< Wrapper for OrtApi::SessionGetMemoryInfoForOutputs + std::vector GetEpDeviceForInputs() const; ///< Wrapper for OrtApi::SessionGetEpDeviceForInputs + std::vector GetEpDeviceForOutputs() const; ///< Wrapper for OrtApi::SessionGetEpDeviceForOutputs + + /** \brief Returns a copy of input name at the specified index. + * + * \param index must less than the value returned by GetInputCount() + * \param allocator to allocate memory for the copy of the name returned + * \return a instance of smart pointer that would deallocate the buffer when out of scope. + * The OrtAllocator instances must be valid at the point of memory release. + */ + AllocatedStringPtr GetInputNameAllocated(size_t index, OrtAllocator* allocator) const; + + /** \brief Returns a copy of output name at then specified index. + * + * \param index must less than the value returned by GetOutputCount() + * \param allocator to allocate memory for the copy of the name returned + * \return a instance of smart pointer that would deallocate the buffer when out of scope. + * The OrtAllocator instances must be valid at the point of memory release. + */ + AllocatedStringPtr GetOutputNameAllocated(size_t index, OrtAllocator* allocator) const; + + /** \brief Returns a copy of the overridable initializer name at then specified index. + * + * \param index must less than the value returned by GetOverridableInitializerCount() + * \param allocator to allocate memory for the copy of the name returned + * \return a instance of smart pointer that would deallocate the buffer when out of scope. + * The OrtAllocator instances must be valid at the point of memory release. + */ + AllocatedStringPtr GetOverridableInitializerNameAllocated(size_t index, OrtAllocator* allocator) const; ///< Wraps OrtApi::SessionGetOverridableInitializerName + + uint64_t GetProfilingStartTimeNs() const; ///< Wraps OrtApi::SessionGetProfilingStartTimeNs + ModelMetadata GetModelMetadata() const; ///< Wraps OrtApi::SessionGetModelMetadata + + TypeInfo GetInputTypeInfo(size_t index) const; ///< Wraps OrtApi::SessionGetInputTypeInfo + TypeInfo GetOutputTypeInfo(size_t index) const; ///< Wraps OrtApi::SessionGetOutputTypeInfo + TypeInfo GetOverridableInitializerTypeInfo(size_t index) const; ///< Wraps OrtApi::SessionGetOverridableInitializerTypeInfo + + int GetOpset(const std::string& domain) const; ///< Wraps OrtApi::SessionGetOpsetForDomain + + std::vector GetInputs() const; + std::vector GetOutputs() const; + + /** \brief Returns information on the subgraph/nodes assigned to execution providers in the session. + * + * \return A list of ConstEpAssignedSubgraph instances. + */ + std::vector GetEpGraphAssignmentInfo() const; +}; + +template +struct SessionImpl : ConstSessionImpl { + using B = ConstSessionImpl; + using B::B; + + /** \brief Run the model returning results in an Ort allocated vector. + * + * Wraps OrtApi::Run + * + * The caller provides a list of inputs and a list of the desired outputs to return. + * + * See the output logs for more information on warnings/errors that occur while processing the model. + * Common errors are.. (TODO) + * + * \param[in] run_options + * \param[in] input_names Array of null terminated strings of length input_count that is the list of input names + * \param[in] input_values Array of Value objects of length input_count that is the list of input values + * \param[in] input_count Number of inputs (the size of the input_names & input_values arrays) + * \param[in] output_names Array of C style strings of length output_count that is the list of output names + * \param[in] output_count Number of outputs (the size of the output_names array) + * \return A std::vector of Value objects that directly maps to the output_names array (eg. output_name[0] is the first entry of the returned vector) + */ + std::vector Run(const RunOptions& run_options, const char* const* input_names, const Value* input_values, size_t input_count, + const char* const* output_names, size_t output_count); + + /** \brief Run the model returning results in user provided outputs + * Same as Run(const RunOptions&, const char* const*, const Value*, size_t,const char* const*, size_t) + */ + void Run(const RunOptions& run_options, const char* const* input_names, const Value* input_values, size_t input_count, + const char* const* output_names, Value* output_values, size_t output_count); + + void Run(const RunOptions& run_options, const IoBinding&); ///< Wraps OrtApi::RunWithBinding + + /** \brief Run the model asynchronously in a thread owned by intra op thread pool + * + * Wraps OrtApi::RunAsync + * + * \param[in] run_options + * \param[in] input_names Array of null terminated UTF8 encoded strings of the input names + * \param[in] input_values Array of Value objects of length input_count + * \param[in] input_count Number of elements in the input_names and inputs arrays + * \param[in] output_names Array of null terminated UTF8 encoded strings of the output names + * \param[out] output_values Array of provided Values to be filled with outputs. + * On calling RunAsync, output_values[i] could either be initialized by a null pointer or a preallocated OrtValue*. + * Later, on invoking the callback, each output_values[i] of null will be filled with an OrtValue* allocated by onnxruntime. + * Then, an OrtValue** pointer will be casted from output_values, and pass to the callback. + * NOTE: it is customer's duty to finally release output_values and each of its member, + * regardless of whether the member (Ort::Value) is allocated by onnxruntime or preallocated by the customer. + * \param[in] output_count Number of elements in the output_names and outputs array + * \param[in] callback Callback function on model run completion + * \param[in] user_data User data that pass back to the callback + */ + void RunAsync(const RunOptions& run_options, const char* const* input_names, const Value* input_values, size_t input_count, + const char* const* output_names, Value* output_values, size_t output_count, RunAsyncCallbackFn callback, void* user_data); + + /** \brief End profiling and return a copy of the profiling file name. + * + * \param allocator to allocate memory for the copy of the string returned + * \return a instance of smart pointer that would deallocate the buffer when out of scope. + * The OrtAllocator instances must be valid at the point of memory release. + */ + AllocatedStringPtr EndProfilingAllocated(OrtAllocator* allocator); ///< Wraps OrtApi::SessionEndProfiling + + /** \brief Set DynamicOptions for EPs (Execution Providers) + * + * Wraps OrtApi::SetEpDynamicOptions + * + * Valid options can be found in `include\onnxruntime\core\session\onnxruntime_session_options_config_keys.h` + * Look for `kOrtEpDynamicOptions` + * + * \param[in] keys Array of null terminated UTF8 encoded strings of EP dynamic option keys + * \param[in] values Array of null terminated UTF8 encoded string of EP dynamic option values + * \param[in] kv_len Number of elements in the keys and values arrays + */ + void SetEpDynamicOptions(const char* const* keys, const char* const* values, size_t kv_len); + + void FinalizeModelEditorSession(const Model& model, const SessionOptions& options, + OrtPrepackedWeightsContainer* prepacked_weights_container = nullptr); +}; + +} // namespace detail + +using ConstSession = detail::ConstSessionImpl>; +using UnownedSession = detail::SessionImpl>; + +/** \brief Wrapper around ::OrtSession + * + */ +struct Session : detail::SessionImpl { + /// Create an empty Session object, must be assigned a valid one to be used. Wraps OrtApi::CreateSession + explicit Session(std::nullptr_t) {} + explicit Session(OrtSession* p) : SessionImpl{p} {} ///< C API Interop + + Session(const Env& env, const ORTCHAR_T* model_path, const SessionOptions& options); + + /// Wraps OrtApi::CreateSessionWithPrepackedWeightsContainer + Session(const Env& env, const ORTCHAR_T* model_path, const SessionOptions& options, + OrtPrepackedWeightsContainer* prepacked_weights_container); + + /// Wraps OrtApi::CreateSessionFromArray + Session(const Env& env, const void* model_data, size_t model_data_length, const SessionOptions& options); + + /// Wraps OrtApi::CreateSessionFromArrayWithPrepackedWeightsContainer + Session(const Env& env, const void* model_data, size_t model_data_length, const SessionOptions& options, + OrtPrepackedWeightsContainer* prepacked_weights_container); + +#if !defined(ORT_MINIMAL_BUILD) + /// Wraps OrtModelEditorApi::CreateSessionFromModel + Session(const Env& env, const Model& model, const SessionOptions& options); + + /// Wraps OrtModelEditorApi::CreateModelEditorSession + static Session CreateModelEditorSession(const Env& env, const ORTCHAR_T* model_path, const SessionOptions& options); + + /// Wraps OrtModelEditorApi::CreateModelEditorSession + static Session CreateModelEditorSession(const Env& env, const void* model_data, size_t model_data_length, + const SessionOptions& options); +#endif // !defined(ORT_MINIMAL_BUILD) + + ConstSession GetConst() const { return ConstSession{this->p_}; } + UnownedSession GetUnowned() const { return UnownedSession{this->p_}; } +}; + +namespace detail { +template +struct TensorTypeAndShapeInfoImpl : Base { + using B = Base; + using B::B; + + ONNXTensorElementDataType GetElementType() const; ///< Wraps OrtApi::GetTensorElementType + size_t GetElementCount() const; ///< Wraps OrtApi::GetTensorShapeElementCount + + size_t GetDimensionsCount() const; ///< Wraps OrtApi::GetDimensionsCount + + /** \deprecated use GetShape() returning std::vector + * [[deprecated]] + * This interface is unsafe to use + */ + [[deprecated("use GetShape()")]] void GetDimensions(int64_t* values, size_t values_count) const; ///< Wraps OrtApi::GetDimensions + + void GetSymbolicDimensions(const char** values, size_t values_count) const; ///< Wraps OrtApi::GetSymbolicDimensions + std::vector GetSymbolicDimensions() const; + + bool HasShape() const; ///< Wraps OrtApi::TensorTypeAndShape_HasShape + std::vector GetShape() const; ///< Uses GetDimensionsCount & GetDimensions to return a std::vector of the shape +}; + +} // namespace detail + +using ConstTensorTypeAndShapeInfo = detail::TensorTypeAndShapeInfoImpl>; + +/** \brief Wrapper around ::OrtTensorTypeAndShapeInfo + * + */ +struct TensorTypeAndShapeInfo : detail::TensorTypeAndShapeInfoImpl { + using Base = detail::TensorTypeAndShapeInfoImpl; + using Base::Base; + + /// Create an empty TensorTypeAndShapeInfo object, must be assigned a valid one to be used + explicit TensorTypeAndShapeInfo(std::nullptr_t) {} + /// Used for interop with the C API + explicit TensorTypeAndShapeInfo(OrtTensorTypeAndShapeInfo* p) : TensorTypeAndShapeInfoImpl{p} {} + + // Create a TensorTypeAndShapeInfo object with the specified element type and dimensions + // symbolic_dims are optional, but should be 1:1 with dims. + // The value in symbolic_dims will be used for all entries in dims that are -1. + explicit TensorTypeAndShapeInfo(ONNXTensorElementDataType element_type, + const std::vector& dims, + const std::vector* symbolic_dims = nullptr); + + ConstTensorTypeAndShapeInfo GetConst() const { return ConstTensorTypeAndShapeInfo{this->p_}; } +}; + +namespace detail { +template +struct SequenceTypeInfoImpl : Base { + using B = Base; + using B::B; + TypeInfo GetSequenceElementType() const; ///< Wraps OrtApi::GetSequenceElementType +}; + +} // namespace detail + +using ConstSequenceTypeInfo = detail::SequenceTypeInfoImpl>; + +/** \brief Wrapper around ::OrtSequenceTypeInfo + * + */ +struct SequenceTypeInfo : detail::SequenceTypeInfoImpl { + using Base = detail::SequenceTypeInfoImpl; + using Base::Base; + + explicit SequenceTypeInfo(std::nullptr_t) {} ///< Create an empty SequenceTypeInfo object, must be assigned a valid one to be used + explicit SequenceTypeInfo(OrtSequenceTypeInfo* p) : SequenceTypeInfoImpl{p} {} ///< Used for interop with the C API + ConstSequenceTypeInfo GetConst() const { return ConstSequenceTypeInfo{this->p_}; } +}; + +namespace detail { +template +struct OptionalTypeInfoImpl : Base { + using B = Base; + using B::B; + TypeInfo GetOptionalElementType() const; ///< Wraps OrtApi::CastOptionalTypeToContainedTypeInfo +}; + +} // namespace detail + +// This is always owned by the TypeInfo and can only be obtained from it. +using ConstOptionalTypeInfo = detail::OptionalTypeInfoImpl>; + +namespace detail { +template +struct MapTypeInfoImpl : detail::Base { + using B = Base; + using B::B; + ONNXTensorElementDataType GetMapKeyType() const; ///< Wraps OrtApi::GetMapKeyType + TypeInfo GetMapValueType() const; ///< Wraps OrtApi::GetMapValueType +}; + +} // namespace detail + +using ConstMapTypeInfo = detail::MapTypeInfoImpl>; + +/** \brief Wrapper around ::OrtMapTypeInfo + * + */ +struct MapTypeInfo : detail::MapTypeInfoImpl { + using Base = detail::MapTypeInfoImpl; + using Base::Base; + + explicit MapTypeInfo(std::nullptr_t) {} ///< Create an empty MapTypeInfo object, must be assigned a valid one to be used + explicit MapTypeInfo(OrtMapTypeInfo* p) : MapTypeInfoImpl{p} {} ///< Used for interop with the C API + ConstMapTypeInfo GetConst() const { return ConstMapTypeInfo{this->p_}; } +}; + +namespace detail { +template +struct TypeInfoImpl : detail::Base { + using B = Base; + using B::B; + + ConstTensorTypeAndShapeInfo GetTensorTypeAndShapeInfo() const; ///< Wraps OrtApi::CastTypeInfoToTensorInfo + ConstSequenceTypeInfo GetSequenceTypeInfo() const; ///< Wraps OrtApi::CastTypeInfoToSequenceTypeInfo + ConstMapTypeInfo GetMapTypeInfo() const; ///< Wraps OrtApi::CastTypeInfoToMapTypeInfo + ConstOptionalTypeInfo GetOptionalTypeInfo() const; ///< wraps OrtApi::CastTypeInfoToOptionalTypeInfo + + ONNXType GetONNXType() const; +}; +} // namespace detail + +/// +/// Contains a constant, unowned OrtTypeInfo that can be copied and passed around by value. +/// Provides access to const OrtTypeInfo APIs. +/// +using ConstTypeInfo = detail::TypeInfoImpl>; + +/// +/// Type information that may contain either TensorTypeAndShapeInfo or +/// the information about contained sequence or map depending on the ONNXType. +/// +struct TypeInfo : detail::TypeInfoImpl { + using Base = detail::TypeInfoImpl; + using Base::Base; + + /// Create an empty TypeInfo object, must be assigned a valid one to be used + explicit TypeInfo(std::nullptr_t) {} + explicit TypeInfo(OrtTypeInfo* p) : TypeInfoImpl{p} {} ///< C API Interop + +#if !defined(ORT_MINIMAL_BUILD) + static TypeInfo CreateTensorInfo(ConstTensorTypeAndShapeInfo tensor_info); + static TypeInfo CreateSparseTensorInfo(ConstTensorTypeAndShapeInfo sparse_tensor_info); + static TypeInfo CreateSequenceTypeInfo(ConstTypeInfo sequence_type); + static TypeInfo CreateMapTypeInfo(ONNXTensorElementDataType key_type, ConstTypeInfo value_type); + static TypeInfo CreateOptionalTypeInfo(ConstTypeInfo contained_type); +#endif // !defined(ORT_MINIMAL_BUILD) + + ConstTypeInfo GetConst() const { return ConstTypeInfo{this->p_}; } +}; + +namespace detail { +// This structure is used to feed sparse tensor values +// information for use with FillSparseTensor() API +// if the data type for the sparse tensor values is numeric +// use data.p_data, otherwise, use data.str pointer to feed +// values. data.str is an array of const char* that are zero terminated. +// number of strings in the array must match shape size. +// For fully sparse tensors use shape {0} and set p_data/str +// to nullptr. +struct OrtSparseValuesParam { + const int64_t* values_shape; + size_t values_shape_len; + union { + const void* p_data; + const char** str; + } data; +}; + +// Provides a way to pass shape in a single +// argument +struct Shape { + const int64_t* shape; + size_t shape_len; +}; + +template +struct ConstValueImpl : Base { + using B = Base; + using B::B; + + /// + /// Obtains a pointer to a user defined data for experimental purposes + /// + template + void GetOpaqueData(const char* domain, const char* type_name, R&) const; ///< Wraps OrtApi::GetOpaqueValue + + bool IsTensor() const; ///< Returns true if Value is a tensor, false for other types like map/sequence/etc + bool HasValue() const; /// < Return true if OrtValue contains data and returns false if the OrtValue is a None + + size_t GetCount() const; // If a non tensor, returns 2 for map and N for sequence, where N is the number of elements + Value GetValue(int index, OrtAllocator* allocator) const; + + /// + /// This API returns a full length of string data contained within either a tensor or a sparse Tensor. + /// For sparse tensor it returns a full length of stored non-empty strings (values). The API is useful + /// for allocating necessary memory and calling GetStringTensorContent(). + /// + /// total length of UTF-8 encoded bytes contained. No zero terminators counted. + size_t GetStringTensorDataLength() const; + + /// + /// The API copies all of the UTF-8 encoded string data contained within a tensor or a sparse tensor + /// into a supplied buffer. Use GetStringTensorDataLength() to find out the length of the buffer to allocate. + /// The user must also allocate offsets buffer with the number of entries equal to that of the contained + /// strings. + /// + /// Strings are always assumed to be on CPU, no X-device copy. + /// + /// user allocated buffer + /// length in bytes of the allocated buffer + /// a pointer to the offsets user allocated buffer + /// count of offsets, must be equal to the number of strings contained. + /// that can be obtained from the shape of the tensor or from GetSparseTensorValuesTypeAndShapeInfo() + /// for sparse tensors + void GetStringTensorContent(void* buffer, size_t buffer_length, size_t* offsets, size_t offsets_count) const; + + /// + /// Returns a const typed pointer to the tensor contained data. + /// No type checking is performed, the caller must ensure the type matches the tensor type. + /// + /// + /// const pointer to data, no copies made + template + const R* GetTensorData() const; ///< Wraps OrtApi::GetTensorData /// + + /// + /// Returns a non-typed pointer to a tensor contained data. + /// + /// const pointer to data, no copies made + const void* GetTensorRawData() const; + + /// + /// The API returns type information for data contained in a tensor. For sparse + /// tensors it returns type information for contained non-zero values. + /// It returns dense shape for sparse tensors. + /// + /// TypeInfo + TypeInfo GetTypeInfo() const; + + /// + /// The API returns type information for data contained in a tensor. For sparse + /// tensors it returns type information for contained non-zero values. + /// It returns dense shape for sparse tensors. + /// + /// TensorTypeAndShapeInfo + TensorTypeAndShapeInfo GetTensorTypeAndShapeInfo() const; + + /// + /// This API returns information about the memory allocation used to hold data. + /// + /// Non owning instance of MemoryInfo + ConstMemoryInfo GetTensorMemoryInfo() const; + + /// + /// The API copies UTF-8 encoded bytes for the requested string element + /// contained within a tensor or a sparse tensor into a provided buffer. + /// Use GetStringTensorElementLength() to obtain the length of the buffer to allocate. + /// + /// + /// + /// + void GetStringTensorElement(size_t buffer_length, size_t element_index, void* buffer) const; + + /// + /// Returns string tensor UTF-8 encoded string element. + /// Use of this API is recommended over GetStringTensorElement() that takes void* buffer pointer. + /// + /// + /// std::string + std::string GetStringTensorElement(size_t element_index) const; + + /// + /// The API returns a byte length of UTF-8 encoded string element + /// contained in either a tensor or a spare tensor values. + /// + /// + /// byte length for the specified string element + size_t GetStringTensorElementLength(size_t element_index) const; + + /// + /// Returns the total size of the tensor data in bytes. Throws an exception if the OrtValue + /// does not contain a tensor or if it contains a tensor that contains strings. + /// For numeric tensors, this is sizeof(element_type) * total_element_count. + /// + /// The total size of the tensor data in bytes + size_t GetTensorSizeInBytes() const; ///< Wraps OrtApi::GetTensorSizeInBytes + +#if !defined(DISABLE_SPARSE_TENSORS) + /// + /// The API returns the sparse data format this OrtValue holds in a sparse tensor. + /// If the sparse tensor was not fully constructed, i.e. Use*() or Fill*() API were not used + /// the value returned is ORT_SPARSE_UNDEFINED. + /// + /// Format enum + OrtSparseFormat GetSparseFormat() const; + + /// + /// The API returns type and shape information for stored non-zero values of the + /// sparse tensor. Use GetSparseTensorValues() to obtain values buffer pointer. + /// + /// TensorTypeAndShapeInfo values information + TensorTypeAndShapeInfo GetSparseTensorValuesTypeAndShapeInfo() const; + + /// + /// The API returns type and shape information for the specified indices. Each supported + /// indices have their own enum values even if a give format has more than one kind of indices. + /// Use GetSparseTensorIndicesData() to obtain pointer to indices buffer. + /// + /// enum requested + /// type and shape information + TensorTypeAndShapeInfo GetSparseTensorIndicesTypeShapeInfo(OrtSparseIndicesFormat format) const; + + /// + /// The API retrieves a pointer to the internal indices buffer. The API merely performs + /// a convenience data type casting on the return type pointer. Make sure you are requesting + /// the right type, use GetSparseTensorIndicesTypeShapeInfo(); + /// + /// type to cast to + /// requested indices kind + /// number of indices entries + /// Pinter to the internal sparse tensor buffer containing indices. Do not free this pointer. + template + const R* GetSparseTensorIndicesData(OrtSparseIndicesFormat indices_format, size_t& num_indices) const; + + /// + /// Returns true if the OrtValue contains a sparse tensor + /// + /// + bool IsSparseTensor() const; + + /// + /// The API returns a pointer to an internal buffer of the sparse tensor + /// containing non-zero values. The API merely does casting. Make sure you + /// are requesting the right data type by calling GetSparseTensorValuesTypeAndShapeInfo() + /// first. + /// + /// numeric data types only. Use GetStringTensor*() to retrieve strings. + /// a pointer to the internal values buffer. Do not free this pointer. + template + const R* GetSparseTensorValues() const; + +#endif + + /// + /// Returns the tensor's element type and a reference to the tensor's internal shape data. The shape data is owned + /// by the Ort::Value and becomes invalid when the Ort::Value is destroyed or if the underlying shape data is + /// updated or reallocated. + /// + /// For a scalar, shape.shape is nullptr and shape.shape_len is 0. + /// + /// Wraps OrtApi::GetTensorElementTypeAndShapeDataReference. + /// + /// Output parameter set to the element's data type. + /// Output parameter set to the OrtValue instance's shape data and number of elements. + void GetTensorElementTypeAndShapeDataReference(ONNXTensorElementDataType& elem_type, Shape& shape) const; +}; + +template +struct ValueImpl : ConstValueImpl { + using B = ConstValueImpl; + using B::B; + + /// + /// Returns a non-const typed pointer to an OrtValue/Tensor contained buffer + /// No type checking is performed, the caller must ensure the type matches the tensor type. + /// + /// non-const pointer to data, no copies made + template + R* GetTensorMutableData(); + + /// + /// Returns a non-typed non-const pointer to a tensor contained data. + /// + /// pointer to data, no copies made + void* GetTensorMutableRawData(); + + /// + // Obtain a reference to an element of data at the location specified + /// by the vector of dims. + /// + /// + /// [in] expressed by a vector of dimensions offsets + /// + template + R& At(const std::vector& location); + + /// + /// Set all strings at once in a string tensor + /// + /// [in] An array of strings. Each string in this array must be null terminated. + /// [in] Count of strings in s (Must match the size of \p value's tensor shape) + void FillStringTensor(const char* const* s, size_t s_len); + + /// + /// Set a single string in a string tensor + /// + /// [in] A null terminated UTF-8 encoded string + /// [in] Index of the string in the tensor to set + void FillStringTensorElement(const char* s, size_t index); + + /// + /// Allocate if necessary and obtain a pointer to a UTF-8 + /// encoded string element buffer indexed by the flat element index, + /// of the specified length. + /// + /// This API is for advanced usage. It avoids a need to construct + /// an auxiliary array of string pointers, and allows to write data directly + /// (do not zero terminate). + /// + /// + /// + /// a pointer to a writable buffer + char* GetResizedStringTensorElementBuffer(size_t index, size_t buffer_length); + +#if !defined(DISABLE_SPARSE_TENSORS) + /// + /// Supplies COO format specific indices and marks the contained sparse tensor as being a COO format tensor. + /// Values are supplied with a CreateSparseTensor() API. The supplied indices are not copied and the user + /// allocated buffers lifespan must eclipse that of the OrtValue. + /// The location of the indices is assumed to be the same as specified by OrtMemoryInfo argument at the creation time. + /// + /// pointer to the user allocated buffer with indices. Use nullptr for fully sparse tensors. + /// number of indices entries. Use 0 for fully sparse tensors + void UseCooIndices(int64_t* indices_data, size_t indices_num); + + /// + /// Supplies CSR format specific indices and marks the contained sparse tensor as being a CSR format tensor. + /// Values are supplied with a CreateSparseTensor() API. The supplied indices are not copied and the user + /// allocated buffers lifespan must eclipse that of the OrtValue. + /// The location of the indices is assumed to be the same as specified by OrtMemoryInfo argument at the creation time. + /// + /// pointer to the user allocated buffer with inner indices or nullptr for fully sparse tensors + /// number of csr inner indices or 0 for fully sparse tensors + /// pointer to the user allocated buffer with outer indices or nullptr for fully sparse tensors + /// number of csr outer indices or 0 for fully sparse tensors + void UseCsrIndices(int64_t* inner_data, size_t inner_num, int64_t* outer_data, size_t outer_num); + + /// + /// Supplies BlockSparse format specific indices and marks the contained sparse tensor as being a BlockSparse format tensor. + /// Values are supplied with a CreateSparseTensor() API. The supplied indices are not copied and the user + /// allocated buffers lifespan must eclipse that of the OrtValue. + /// The location of the indices is assumed to be the same as specified by OrtMemoryInfo argument at the creation time. + /// + /// indices shape or a {0} for fully sparse + /// user allocated buffer with indices or nullptr for fully spare tensors + void UseBlockSparseIndices(const Shape& indices_shape, int32_t* indices_data); + + /// + /// The API will allocate memory using the allocator instance supplied to the CreateSparseTensor() API + /// and copy the values and COO indices into it. If data_mem_info specifies that the data is located + /// at difference device than the allocator, a X-device copy will be performed if possible. + /// + /// specified buffer memory description + /// values buffer information. + /// coo indices buffer or nullptr for fully sparse data + /// number of COO indices or 0 for fully sparse data + void FillSparseTensorCoo(const OrtMemoryInfo* data_mem_info, const OrtSparseValuesParam& values_param, + const int64_t* indices_data, size_t indices_num); + + /// + /// The API will allocate memory using the allocator instance supplied to the CreateSparseTensor() API + /// and copy the values and CSR indices into it. If data_mem_info specifies that the data is located + /// at difference device than the allocator, a X-device copy will be performed if possible. + /// + /// specified buffer memory description + /// values buffer information + /// csr inner indices pointer or nullptr for fully sparse tensors + /// number of csr inner indices or 0 for fully sparse tensors + /// pointer to csr indices data or nullptr for fully sparse tensors + /// number of csr outer indices or 0 + void FillSparseTensorCsr(const OrtMemoryInfo* data_mem_info, + const OrtSparseValuesParam& values, + const int64_t* inner_indices_data, size_t inner_indices_num, + const int64_t* outer_indices_data, size_t outer_indices_num); + + /// + /// The API will allocate memory using the allocator instance supplied to the CreateSparseTensor() API + /// and copy the values and BlockSparse indices into it. If data_mem_info specifies that the data is located + /// at difference device than the allocator, a X-device copy will be performed if possible. + /// + /// specified buffer memory description + /// values buffer information + /// indices shape. use {0} for fully sparse tensors + /// pointer to indices data or nullptr for fully sparse tensors + void FillSparseTensorBlockSparse(const OrtMemoryInfo* data_mem_info, + const OrtSparseValuesParam& values, + const Shape& indices_shape, + const int32_t* indices_data); + +#endif +}; + +} // namespace detail + +using ConstValue = detail::ConstValueImpl>; +using UnownedValue = detail::ValueImpl>; + +/** \brief Wrapper around ::OrtValue + * + */ +struct Value : detail::ValueImpl { + using Base = detail::ValueImpl; + using Base::Base; + using OrtSparseValuesParam = detail::OrtSparseValuesParam; + using Shape = detail::Shape; + + Value(std::nullptr_t) {} ///< Create an empty Value object, must be assigned a valid one to be used + Value(Value&&) = default; + Value& operator=(Value&&) = default; + + ConstValue GetConst() const { return ConstValue{this->p_}; } + UnownedValue GetUnowned() const { return UnownedValue{this->p_}; } + + /** \brief Creates a tensor with a user supplied buffer. Wraps OrtApi::CreateTensorWithDataAsOrtValue. + * \tparam T The numeric datatype. This API is not suitable for strings. + * \param info Memory description of where the p_data buffer resides (CPU vs GPU etc). + * \param p_data Pointer to the data buffer. + * \param p_data_element_count The number of elements in the data buffer. + * \param shape Pointer to the tensor shape dimensions. + * \param shape_len The number of tensor shape dimensions. + */ + template + static Value CreateTensor(const OrtMemoryInfo* info, T* p_data, size_t p_data_element_count, + const int64_t* shape, size_t shape_len); + + /** \brief Creates a tensor with a user supplied buffer. Wraps OrtApi::CreateTensorWithDataAsOrtValue. + * + * \param info Memory description of where the p_data buffer resides (CPU vs GPU etc). + * \param p_data Pointer to the data buffer. + * \param p_data_byte_count The number of bytes in the data buffer. + * \param shape Pointer to the tensor shape dimensions. + * \param shape_len The number of tensor shape dimensions. + * \param type The data type. + */ + static Value CreateTensor(const OrtMemoryInfo* info, void* p_data, size_t p_data_byte_count, + const int64_t* shape, size_t shape_len, + ONNXTensorElementDataType type); + + /** \brief Creates a tensor with a user supplied buffer. Wraps OrtApi::CreateTensorWithDataAndDeleterAsOrtValue. + * + * \param deleter OrtAllocator that will be used to free the buffer when no longer required. + * \param p_data Pointer to the data buffer. + * \param p_data_byte_count The number of bytes in the data buffer. + * \param shape Pointer to the tensor shape dimensions. + * \param shape_len The number of tensor shape dimensions. + * \param type The data type. + */ + static Value CreateTensor(OrtAllocator* deleter, void* p_data, size_t p_data_byte_count, + const int64_t* shape, size_t shape_len, + ONNXTensorElementDataType type); + + /** \brief Creates an OrtValue with a tensor using a supplied OrtAllocator. Wraps OrtApi::CreateTensorAsOrtValue. + * This overload will allocate the buffer for the tensor according to the supplied shape and data type. + * The allocated buffer will be owned by the returned OrtValue and will be freed when the OrtValue is released. + * The input data would need to be copied into the allocated buffer. + * This API is not suitable for strings. + * + * \tparam T The numeric datatype. This API is not suitable for strings. + * \param allocator The allocator to use. + * \param shape Pointer to the tensor shape dimensions. + * \param shape_len The number of tensor shape dimensions. + */ + template + static Value CreateTensor(OrtAllocator* allocator, const int64_t* shape, size_t shape_len); + + /** \brief Creates an OrtValue with a tensor using the supplied OrtAllocator. + * Wraps OrtApi::CreateTensorAsOrtValue. + * The allocated buffer will be owned by the returned OrtValue and will be freed when the OrtValue is released. + * The input data would need to be copied into the allocated buffer. + * This API is not suitable for strings. + * + * \param allocator The allocator to use. + * \param shape Pointer to the tensor shape dimensions. + * \param shape_len The number of tensor shape dimensions. + * \param type The data type. + */ + static Value CreateTensor(OrtAllocator* allocator, const int64_t* shape, size_t shape_len, + ONNXTensorElementDataType type); + + /** \brief Creates an OrtValue with a Map Onnx type representation. + * The API would ref-count the supplied OrtValues and they will be released + * when the returned OrtValue is released. The caller may release keys and values after the call + * returns. + * + * \param keys an OrtValue containing a tensor with primitive data type keys. + * \param values an OrtValue that may contain a tensor. Ort currently supports only primitive data type values. + */ + static Value CreateMap(const Value& keys, const Value& values); ///< Wraps OrtApi::CreateValue + + /** \brief Creates an OrtValue with a Sequence Onnx type representation. + * The API would ref-count the supplied OrtValues and they will be released + * when the returned OrtValue is released. The caller may release the values after the call + * returns. + * + * \param values a vector of OrtValues that must have the same Onnx value type. + */ + static Value CreateSequence(const std::vector& values); ///< Wraps OrtApi::CreateValue + + /** \brief Creates an OrtValue wrapping an Opaque type. + * This is used for experimental support of non-tensor types. + * + * \tparam T - the type of the value. + * \param domain - zero terminated utf-8 string. Domain of the type. + * \param type_name - zero terminated utf-8 string. Name of the type. + * \param value - the value to be wrapped. + */ + template + static Value CreateOpaque(const char* domain, const char* type_name, const T& value); ///< Wraps OrtApi::CreateOpaqueValue + +#if !defined(DISABLE_SPARSE_TENSORS) + /// + /// This is a simple forwarding method to the other overload that helps deducing + /// data type enum value from the type of the buffer. + /// + /// numeric datatype. This API is not suitable for strings. + /// Memory description where the user buffers reside (CPU vs GPU etc) + /// pointer to the user supplied buffer, use nullptr for fully sparse tensors + /// a would be dense shape of the tensor + /// non zero values shape. Use a single 0 shape for fully sparse tensors. + /// + template + static Value CreateSparseTensor(const OrtMemoryInfo* info, T* p_data, const Shape& dense_shape, + const Shape& values_shape); + + /// + /// Creates an OrtValue instance containing SparseTensor. This constructs + /// a sparse tensor that makes use of user allocated buffers. It does not make copies + /// of the user provided data and does not modify it. The lifespan of user provided buffers should + /// eclipse the life span of the resulting OrtValue. This call constructs an instance that only contain + /// a pointer to non-zero values. To fully populate the sparse tensor call UseIndices() API below + /// to supply a sparse format specific indices. + /// This API is not suitable for string data. Use CreateSparseTensor() with allocator specified so strings + /// can be properly copied into the allocated buffer. + /// + /// Memory description where the user buffers reside (CPU vs GPU etc) + /// pointer to the user supplied buffer, use nullptr for fully sparse tensors + /// a would be dense shape of the tensor + /// non zero values shape. Use a single 0 shape for fully sparse tensors. + /// data type + /// Ort::Value instance containing SparseTensor + static Value CreateSparseTensor(const OrtMemoryInfo* info, void* p_data, const Shape& dense_shape, + const Shape& values_shape, ONNXTensorElementDataType type); + + /// + /// This is a simple forwarding method to the below CreateSparseTensor. + /// This helps to specify data type enum in terms of C++ data type. + /// Use CreateSparseTensor + /// + /// numeric data type only. String data enum must be specified explicitly. + /// allocator to use + /// a would be dense shape of the tensor + /// Ort::Value + template + static Value CreateSparseTensor(OrtAllocator* allocator, const Shape& dense_shape); + + /// + /// Creates an instance of OrtValue containing sparse tensor. The created instance has no data. + /// The data must be supplied by on of the FillSparseTensor() methods that take both non-zero values + /// and indices. The data will be copied into a buffer that would be allocated using the supplied allocator. + /// Use this API to create OrtValues that contain sparse tensors with all supported data types including + /// strings. + /// + /// allocator to use. The allocator lifespan must eclipse that of the resulting OrtValue + /// a would be dense shape of the tensor + /// data type + /// an instance of Ort::Value + static Value CreateSparseTensor(OrtAllocator* allocator, const Shape& dense_shape, ONNXTensorElementDataType type); + +#endif // !defined(DISABLE_SPARSE_TENSORS) +}; + +namespace detail { +namespace binding_utils { +// Bring these out of template +std::vector GetOutputNamesHelper(const OrtIoBinding* binding, OrtAllocator*); +std::vector GetOutputValuesHelper(const OrtIoBinding* binding, OrtAllocator*); +} // namespace binding_utils + +template +struct ConstIoBindingImpl : Base { + using B = Base; + using B::B; + + std::vector GetOutputNames() const; + std::vector GetOutputNames(OrtAllocator*) const; + std::vector GetOutputValues() const; + std::vector GetOutputValues(OrtAllocator*) const; +}; + +template +struct IoBindingImpl : ConstIoBindingImpl { + using B = ConstIoBindingImpl; + using B::B; + + void BindInput(const char* name, const Value&); + void BindOutput(const char* name, const Value&); + void BindOutput(const char* name, const OrtMemoryInfo*); + void ClearBoundInputs(); + void ClearBoundOutputs(); + void SynchronizeInputs(); + void SynchronizeOutputs(); +}; + +} // namespace detail + +using ConstIoBinding = detail::ConstIoBindingImpl>; +using UnownedIoBinding = detail::IoBindingImpl>; + +/** \brief Wrapper around ::OrtIoBinding + * + */ +struct IoBinding : detail::IoBindingImpl { + explicit IoBinding(std::nullptr_t) {} ///< Create an empty object for convenience. Sometimes, we want to initialize members later. + explicit IoBinding(Session& session); + ConstIoBinding GetConst() const { return ConstIoBinding{this->p_}; } + UnownedIoBinding GetUnowned() const { return UnownedIoBinding{this->p_}; } +}; + +/*! \struct Ort::ArenaCfg + * \brief it is a structure that represents the configuration of an arena based allocator + * \details Please see docs/C_API.md for details + */ +struct ArenaCfg : detail::Base { + explicit ArenaCfg(std::nullptr_t) {} ///< Create an empty ArenaCfg object, must be assigned a valid one to be used + /** + * Wraps OrtApi::CreateArenaCfg + * \param max_mem - use 0 to allow ORT to choose the default + * \param arena_extend_strategy - use -1 to allow ORT to choose the default, 0 = kNextPowerOfTwo, 1 = kSameAsRequested + * \param initial_chunk_size_bytes - use -1 to allow ORT to choose the default + * \param max_dead_bytes_per_chunk - use -1 to allow ORT to choose the default + * See docs/C_API.md for details on what the following parameters mean and how to choose these values + */ + ArenaCfg(size_t max_mem, int arena_extend_strategy, int initial_chunk_size_bytes, int max_dead_bytes_per_chunk); + + /** + * Wraps Ort::CreateArenaCfgV2 + * See C API for details on what the following parameters mean and how to choose these values + */ + explicit ArenaCfg(const std::unordered_map& arena_config); +}; + +// +// Custom OPs (only needed to implement custom OPs) +// + +namespace detail { +// Need to define a templated ConstOpAttr with const members +template +struct ConstOpAttrImpl : Base { + using B = detail::Base; + using B::B; + + // Wraps OrtApi::OpAttr_GetName + std::string GetName() const; + // Wraps OrtApi::OpAttr_GetType + OrtOpAttrType GetType() const; + + // Wraps OrtApi::ReadAttr for a single value + // This does not support Tensor Attribute + // Call GetTensorAttributeAsOrtValue() instead. + template + Status GetValue(R& out) const; + + // Wraps OrtApi::ReadAttr for an array of values + template + Status GetValueArray(std::vector& out) const; + // Wraps OrtApi::OpAttr_GetTensorAttributeAsOrtValue + Status GetTensorAttributeAsOrtValue(Value&) const; +}; +} // namespace detail + +using ConstOpAttr = detail::ConstOpAttrImpl>; + +/// +/// This struct provides life time management for custom op attribute +/// +struct OpAttr : detail::ConstOpAttrImpl { + using Base = detail::ConstOpAttrImpl; + using Base::Base; + + OpAttr() = default; // Enable storing it in the container for resize() + explicit OpAttr(std::nullptr_t) {} + OpAttr(const char* name, const void* data, int len, OrtOpAttrType type); + + ConstOpAttr GetConst() const { return ConstOpAttr{this->p_}; } +}; + +/** + * Macro that logs a message using the provided logger. Throws an exception if OrtApi::Logger_LogMessage fails. + * Example: ORT_CXX_LOG(logger, ORT_LOGGING_LEVEL_INFO, "Log a message"); + * + * \param logger The Ort::Logger instance to use. Must be a value or reference. + * \param message_severity The logging severity level of the message. + * \param message A null-terminated UTF-8 message to log. + */ +#define ORT_CXX_LOG(logger, message_severity, message) \ + do { \ + if (message_severity >= logger.GetLoggingSeverityLevel()) { \ + Ort::ThrowOnError(logger.LogMessage(message_severity, ORT_FILE, __LINE__, \ + static_cast(__FUNCTION__), message)); \ + } \ + } while (false) + +/** + * Macro that logs a message using the provided logger. Can be used in noexcept code since errors are silently ignored. + * Example: ORT_CXX_LOG_NOEXCEPT(logger, ORT_LOGGING_LEVEL_INFO, "Log a message"); + * + * \param logger The Ort::Logger instance to use. Must be a value or reference. + * \param message_severity The logging severity level of the message. + * \param message A null-terminated UTF-8 message to log. + */ +#define ORT_CXX_LOG_NOEXCEPT(logger, message_severity, message) \ + do { \ + if (message_severity >= logger.GetLoggingSeverityLevel()) { \ + static_cast(logger.LogMessage(message_severity, ORT_FILE, __LINE__, \ + static_cast(__FUNCTION__), message)); \ + } \ + } while (false) + +/** + * Macro that logs a printf-like formatted message using the provided logger. Throws an exception if + * OrtApi::Logger_LogMessage fails or if a formatting error occurs. + * Example: ORT_CXX_LOGF(logger, ORT_LOGGING_LEVEL_INFO, "Log an int: %d", 12); + * + * \param logger The Ort::Logger instance to use. Must be a value or reference. + * \param message_severity The logging severity level of the message. + * \param format A null-terminated UTF-8 format string forwarded to a printf-like function. + * Refer to https://en.cppreference.com/w/cpp/io/c/fprintf for information on valid formats. + * \param ... Zero or more variadic arguments referenced by the format string. + */ +#define ORT_CXX_LOGF(logger, message_severity, /*format,*/...) \ + do { \ + if (message_severity >= logger.GetLoggingSeverityLevel()) { \ + Ort::ThrowOnError(logger.LogFormattedMessage(message_severity, ORT_FILE, __LINE__, \ + static_cast(__FUNCTION__), __VA_ARGS__)); \ + } \ + } while (false) + +/** + * Macro that logs a printf-like formatted message using the provided logger. Can be used in noexcept code since errors + * are silently ignored. + * Example: ORT_CXX_LOGF_NOEXCEPT(logger, ORT_LOGGING_LEVEL_INFO, "Log an int: %d", 12); + * + * \param logger The Ort::Logger instance to use. Must be a value or reference. + * \param message_severity The logging severity level of the message. + * \param format A null-terminated UTF-8 format string forwarded to a printf-like function. + * Refer to https://en.cppreference.com/w/cpp/io/c/fprintf for information on valid formats. + * \param ... Zero or more variadic arguments referenced by the format string. + */ +#define ORT_CXX_LOGF_NOEXCEPT(logger, message_severity, /*format,*/...) \ + do { \ + if (message_severity >= logger.GetLoggingSeverityLevel()) { \ + static_cast(logger.LogFormattedMessage(message_severity, ORT_FILE, __LINE__, \ + static_cast(__FUNCTION__), __VA_ARGS__)); \ + } \ + } while (false) + +/// +/// This class represents an ONNX Runtime logger that can be used to log information with an +/// associated severity level and source code location (file path, line number, function name). +/// +/// A Logger can be obtained from within custom operators by calling Ort::KernelInfo::GetLogger(). +/// Instances of Ort::Logger are the size of two pointers and can be passed by value. +/// +/// Use the ORT_CXX_LOG macros to ensure the source code location is set properly from the callsite +/// and to take advantage of a cached logging severity level that can bypass calls to the underlying C API. +/// +struct Logger { + /** + * Creates an empty Ort::Logger. Must be initialized from a valid Ort::Logger before use. + */ + Logger() = default; + + /** + * Creates an empty Ort::Logger. Must be initialized from a valid Ort::Logger before use. + */ + explicit Logger(std::nullptr_t) {} + + /** + * Creates a logger from an ::OrtLogger instance. Caches the logger's current severity level by calling + * OrtApi::Logger_GetLoggingSeverityLevel. Throws an exception if OrtApi::Logger_GetLoggingSeverityLevel fails. + * + * \param logger The ::OrtLogger to wrap. + */ + explicit Logger(const OrtLogger* logger); + + ~Logger() = default; + + Logger(const Logger&) = default; + Logger& operator=(const Logger&) = default; + + Logger(Logger&& v) noexcept = default; + Logger& operator=(Logger&& v) noexcept = default; + + /** + * Returns the logger's current severity level from the cached member. + * + * \return The current ::OrtLoggingLevel. + */ + OrtLoggingLevel GetLoggingSeverityLevel() const noexcept; + + /** + * Logs the provided message via OrtApi::Logger_LogMessage. Use the ORT_CXX_LOG or ORT_CXX_LOG_NOEXCEPT + * macros to properly set the source code location and to use the cached severity level to potentially bypass + * calls to the underlying C API. + * + * \param log_severity_level The message's logging severity level. + * \param file_path The filepath of the file in which the message is logged. Usually the value of ORT_FILE. + * \param line_number The file line number in which the message is logged. Usually the value of __LINE__. + * \param func_name The name of the function in which the message is logged. Usually the value of __FUNCTION__. + * \param message The message to log. + * \return A Ort::Status value to indicate error or success. + */ + Status LogMessage(OrtLoggingLevel log_severity_level, const ORTCHAR_T* file_path, int line_number, + const char* func_name, const char* message) const noexcept; + + /** + * Logs a printf-like formatted message via OrtApi::Logger_LogMessage. Use the ORT_CXX_LOGF or ORT_CXX_LOGF_NOEXCEPT + * macros to properly set the source code location and to use the cached severity level to potentially bypass + * calls to the underlying C API. Returns an error status if a formatting error occurs. + * + * \param log_severity_level The message's logging severity level. + * \param file_path The filepath of the file in which the message is logged. Usually the value of ORT_FILE. + * \param line_number The file line number in which the message is logged. Usually the value of __LINE__. + * \param func_name The name of the function in which the message is logged. Usually the value of __FUNCTION__. + * \param format A null-terminated UTF-8 format string forwarded to a printf-like function. + * Refer to https://en.cppreference.com/w/cpp/io/c/fprintf for information on valid formats. + * \param args Zero or more variadic arguments referenced by the format string. + * \return A Ort::Status value to indicate error or success. + */ + template + Status LogFormattedMessage(OrtLoggingLevel log_severity_level, const ORTCHAR_T* file_path, int line_number, + const char* func_name, const char* format, Args&&... args) const noexcept; + + private: + const OrtLogger* logger_{}; + OrtLoggingLevel cached_severity_level_{}; +}; + +/// +/// This class wraps a raw pointer OrtKernelContext* that is being passed +/// to the custom kernel Compute() method. Use it to safely access context +/// attributes, input and output parameters with exception safety guarantees. +/// See usage example in onnxruntime/test/testdata/custom_op_library/custom_op_library.cc +/// +struct KernelContext { + explicit KernelContext(OrtKernelContext* context); + size_t GetInputCount() const; + size_t GetOutputCount() const; + // If input is optional and is not present, the method returns an empty ConstValue + // which can be compared to nullptr. + ConstValue GetInput(size_t index) const; + // If output is optional and is not present, the method returns an empty UnownedValue + // which can be compared to nullptr. + UnownedValue GetOutput(size_t index, const int64_t* dim_values, size_t dim_count) const; + UnownedValue GetOutput(size_t index, const std::vector& dims) const; + void* GetGPUComputeStream() const; + Logger GetLogger() const; + Ort::Allocator GetAllocator(const OrtMemoryInfo& memory_info) const; + OrtKernelContext* GetOrtKernelContext() const { return ctx_; } + void ParallelFor(void (*fn)(void*, size_t), size_t total, size_t num_batch, void* usr_data) const; + + private: + OrtKernelContext* ctx_; +}; + +struct KernelInfo; + +namespace detail { +namespace attr_utils { +void GetAttr(const OrtKernelInfo* p, const char* name, float&); +void GetAttr(const OrtKernelInfo* p, const char* name, int64_t&); +void GetAttr(const OrtKernelInfo* p, const char* name, std::string&); +void GetAttrs(const OrtKernelInfo* p, const char* name, std::vector&); +void GetAttrs(const OrtKernelInfo* p, const char* name, std::vector&); +} // namespace attr_utils + +template +struct KernelInfoImpl : Base { + using B = Base; + using B::B; + + KernelInfo Copy() const; + + template // R is only implemented for float, int64_t, and string + R GetAttribute(const char* name) const { + R val; + attr_utils::GetAttr(this->p_, name, val); + return val; + } + + template // R is only implemented for std::vector, std::vector + std::vector GetAttributes(const char* name) const { + std::vector result; + attr_utils::GetAttrs(this->p_, name, result); + return result; + } + + Value GetTensorAttribute(const char* name, OrtAllocator* allocator) const; + + size_t GetInputCount() const; + size_t GetOutputCount() const; + + std::string GetInputName(size_t index) const; + std::string GetOutputName(size_t index) const; + + TypeInfo GetInputTypeInfo(size_t index) const; + TypeInfo GetOutputTypeInfo(size_t index) const; + + ConstValue GetTensorConstantInput(size_t index, int* is_constant) const; + + std::string GetNodeName() const; + Logger GetLogger() const; + + KeyValuePairs GetConfigEntries() const; + + std::string GetOperatorDomain() const; ///< Wraps OrtApi::KernelInfo_GetOperatorDomain + std::string GetOperatorType() const; ///< Wraps OrtApi::KernelInfo_GetOperatorType + int GetOperatorSinceVersion() const; ///< Wraps OrtApi::KernelInfo_GetOperatorSinceVersion + const OrtEp* GetEp() const; ///< Wraps OrtEpApi::KernelInfo_GetEp +}; + +} // namespace detail + +using ConstKernelInfo = detail::KernelInfoImpl>; + +/// +/// This struct owns the OrtKernInfo* pointer when a copy is made. +/// For convenient wrapping of OrtKernelInfo* passed to kernel constructor +/// and query attributes, warp the pointer with Ort::Unowned instance +/// so it does not destroy the pointer the kernel does not own. +/// +struct KernelInfo : detail::KernelInfoImpl { + using Base = detail::KernelInfoImpl; + using Base::Base; + explicit KernelInfo(std::nullptr_t) {} ///< Create an empty instance to initialize later + explicit KernelInfo(OrtKernelInfo* info); ///< Take ownership of the instance + ConstKernelInfo GetConst() const { return ConstKernelInfo{this->p_}; } +}; + +/// +/// Create and own custom defined operation. +/// +struct Op : detail::Base { + using Base = detail::Base; + using Base::Base; + + explicit Op(std::nullptr_t) {} ///< Create an empty Operator object, must be assigned a valid one to be used + + explicit Op(OrtOp*); ///< Take ownership of the OrtOp + + static Op Create(const OrtKernelInfo* info, const char* op_name, const char* domain, + int version, const char** type_constraint_names, + const ONNXTensorElementDataType* type_constraint_values, + size_t type_constraint_count, + const OpAttr* attr_values, + size_t attr_count, + size_t input_count, size_t output_count); + + void Invoke(const OrtKernelContext* context, + const Value* input_values, + size_t input_count, + Value* output_values, + size_t output_count); + + // For easier refactoring + void Invoke(const OrtKernelContext* context, + const OrtValue* const* input_values, + size_t input_count, + OrtValue* const* output_values, + size_t output_count); +}; + +/// +/// Provide access to per-node attributes and input shapes, so one could compute and set output shapes. +/// +struct ShapeInferContext { + struct SymbolicInteger { + SymbolicInteger(int64_t i) : i_(i), is_int_(true) {}; + SymbolicInteger(const char* s) : s_(s), is_int_(false) {}; + SymbolicInteger(const SymbolicInteger&) = default; + SymbolicInteger(SymbolicInteger&&) = default; + + SymbolicInteger& operator=(const SymbolicInteger&) = default; + SymbolicInteger& operator=(SymbolicInteger&&) = default; + + bool operator==(const SymbolicInteger& dim) const { + if (is_int_ == dim.is_int_) { + if (is_int_) { + return i_ == dim.i_; + } else { + return std::string{s_} == std::string{dim.s_}; + } + } + return false; + } + + bool IsInt() const { return is_int_; } + int64_t AsInt() const { return i_; } + const char* AsSym() const { return s_; } + + static constexpr int INVALID_INT_DIM = -2; + + private: + union { + int64_t i_; + const char* s_; + }; + bool is_int_; + }; + + using Shape = std::vector; + + ShapeInferContext(const OrtApi* ort_api, OrtShapeInferContext* ctx); + + const Shape& GetInputShape(size_t indice) const { return input_shapes_.at(indice); } + + size_t GetInputCount() const { return input_shapes_.size(); } + + Status SetOutputShape(size_t indice, const Shape& shape, ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT); + + int64_t GetAttrInt(const char* attr_name); + + using Ints = std::vector; + Ints GetAttrInts(const char* attr_name); + + float GetAttrFloat(const char* attr_name); + + using Floats = std::vector; + Floats GetAttrFloats(const char* attr_name); + + std::string GetAttrString(const char* attr_name); + + using Strings = std::vector; + Strings GetAttrStrings(const char* attr_name); + + private: + ConstOpAttr GetAttrHdl(const char* attr_name) const; + const OrtApi* ort_api_; + OrtShapeInferContext* ctx_; + std::vector input_shapes_; +}; + +using ShapeInferFn = Ort::Status (*)(Ort::ShapeInferContext&); + +#define MAX_CUSTOM_OP_END_VER (1UL << 31) - 1 + +template +struct CustomOpBase : OrtCustomOp { + CustomOpBase() { + OrtCustomOp::version = ORT_API_VERSION; + OrtCustomOp::GetName = [](const OrtCustomOp* this_) { return static_cast(this_)->GetName(); }; + + OrtCustomOp::GetExecutionProviderType = [](const OrtCustomOp* this_) { return static_cast(this_)->GetExecutionProviderType(); }; + + OrtCustomOp::GetInputTypeCount = [](const OrtCustomOp* this_) { return static_cast(this_)->GetInputTypeCount(); }; + OrtCustomOp::GetInputType = [](const OrtCustomOp* this_, size_t index) { return static_cast(this_)->GetInputType(index); }; + OrtCustomOp::GetInputMemoryType = [](const OrtCustomOp* this_, size_t index) { return static_cast(this_)->GetInputMemoryType(index); }; + + OrtCustomOp::GetOutputTypeCount = [](const OrtCustomOp* this_) { return static_cast(this_)->GetOutputTypeCount(); }; + OrtCustomOp::GetOutputType = [](const OrtCustomOp* this_, size_t index) { return static_cast(this_)->GetOutputType(index); }; + +#if defined(_MSC_VER) && !defined(__clang__) +#pragma warning(push) +#pragma warning(disable : 26409) +#endif + OrtCustomOp::KernelDestroy = [](void* op_kernel) { delete static_cast(op_kernel); }; +#if defined(_MSC_VER) && !defined(__clang__) +#pragma warning(pop) +#endif + OrtCustomOp::GetInputCharacteristic = [](const OrtCustomOp* this_, size_t index) { return static_cast(this_)->GetInputCharacteristic(index); }; + OrtCustomOp::GetOutputCharacteristic = [](const OrtCustomOp* this_, size_t index) { return static_cast(this_)->GetOutputCharacteristic(index); }; + + OrtCustomOp::GetVariadicInputMinArity = [](const OrtCustomOp* this_) { return static_cast(this_)->GetVariadicInputMinArity(); }; + OrtCustomOp::GetVariadicInputHomogeneity = [](const OrtCustomOp* this_) { return static_cast(static_cast(this_)->GetVariadicInputHomogeneity()); }; + OrtCustomOp::GetVariadicOutputMinArity = [](const OrtCustomOp* this_) { return static_cast(this_)->GetVariadicOutputMinArity(); }; + OrtCustomOp::GetVariadicOutputHomogeneity = [](const OrtCustomOp* this_) { return static_cast(static_cast(this_)->GetVariadicOutputHomogeneity()); }; +#ifdef __cpp_if_constexpr + if constexpr (WithStatus) { +#else + if (WithStatus) { +#endif + OrtCustomOp::CreateKernelV2 = [](const OrtCustomOp* this_, const OrtApi* api, const OrtKernelInfo* info, void** op_kernel) -> OrtStatusPtr { + return static_cast(this_)->CreateKernelV2(*api, info, op_kernel); + }; + OrtCustomOp::KernelComputeV2 = [](void* op_kernel, OrtKernelContext* context) -> OrtStatusPtr { + return static_cast(op_kernel)->ComputeV2(context); + }; + } else { + OrtCustomOp::CreateKernelV2 = nullptr; + OrtCustomOp::KernelComputeV2 = nullptr; + + OrtCustomOp::CreateKernel = [](const OrtCustomOp* this_, const OrtApi* api, const OrtKernelInfo* info) { return static_cast(this_)->CreateKernel(*api, info); }; + OrtCustomOp::KernelCompute = [](void* op_kernel, OrtKernelContext* context) { + static_cast(op_kernel)->Compute(context); + }; + } + + SetShapeInferFn(0); + + OrtCustomOp::GetStartVersion = [](const OrtCustomOp* this_) { + return static_cast(this_)->start_ver_; + }; + + OrtCustomOp::GetEndVersion = [](const OrtCustomOp* this_) { + return static_cast(this_)->end_ver_; + }; + + OrtCustomOp::GetMayInplace = nullptr; + OrtCustomOp::ReleaseMayInplace = nullptr; + OrtCustomOp::GetAliasMap = nullptr; + OrtCustomOp::ReleaseAliasMap = nullptr; + } + + // Default implementation of GetExecutionProviderType that returns nullptr to default to the CPU provider + const char* GetExecutionProviderType() const { return nullptr; } + + // Default implementations of GetInputCharacteristic() and GetOutputCharacteristic() below + // (inputs and outputs are required by default) + OrtCustomOpInputOutputCharacteristic GetInputCharacteristic(size_t /*index*/) const { + return OrtCustomOpInputOutputCharacteristic::INPUT_OUTPUT_REQUIRED; + } + + OrtCustomOpInputOutputCharacteristic GetOutputCharacteristic(size_t /*index*/) const { + return OrtCustomOpInputOutputCharacteristic::INPUT_OUTPUT_REQUIRED; + } + + // Default implementation of GetInputMemoryType() that returns OrtMemTypeDefault + OrtMemType GetInputMemoryType(size_t /*index*/) const { + return OrtMemTypeDefault; + } + + // Default implementation of GetVariadicInputMinArity() returns 1 to specify that a variadic input + // should expect at least 1 argument. + int GetVariadicInputMinArity() const { + return 1; + } + + // Default implementation of GetVariadicInputHomegeneity() returns true to specify that all arguments + // to a variadic input should be of the same type. + bool GetVariadicInputHomogeneity() const { + return true; + } + + // Default implementation of GetVariadicOutputMinArity() returns 1 to specify that a variadic output + // should produce at least 1 output value. + int GetVariadicOutputMinArity() const { + return 1; + } + + // Default implementation of GetVariadicOutputHomegeneity() returns true to specify that all output values + // produced by a variadic output should be of the same type. + bool GetVariadicOutputHomogeneity() const { + return true; + } + + // Declare list of session config entries used by this Custom Op. + // Implement this function in order to get configs from CustomOpBase::GetSessionConfigs(). + // This default implementation returns an empty vector of config entries. + std::vector GetSessionConfigKeys() const { + return std::vector{}; + } + + // Ort::CustomOpBase derived class should provide the following static method with the type/shape inferencing + // implementation if needed: + // static OrtStatusPtr InferOutputShape(Ort::ShapeInferContext& context) + template + decltype(&C::InferOutputShape) SetShapeInferFn(decltype(&C::InferOutputShape)) { + OrtCustomOp::InferOutputShapeFn = [](const OrtCustomOp*, OrtShapeInferContext* ort_ctx) -> OrtStatusPtr { + ShapeInferContext ctx(&GetApi(), ort_ctx); + return C::InferOutputShape(ctx); + }; + return {}; + } + + template + void SetShapeInferFn(...) { + OrtCustomOp::InferOutputShapeFn = {}; + } + + protected: + // Helper function that returns a map of session config entries specified by CustomOpBase::GetSessionConfigKeys. + void GetSessionConfigs(std::unordered_map& out, ConstSessionOptions options) const; + + int start_ver_ = 1; + int end_ver_ = MAX_CUSTOM_OP_END_VER; +}; + +// Forward declaration to resolve circular dependency +// on ConstNode +struct ValueInfoConsumerProducerInfo; + +namespace detail { +template +struct ConstValueInfoImpl : Base { + using B = Base; + using B::B; + + /// < A wrapper around OrtApi::GetValueInfoName + std::string GetName() const; + /// < A wrapper around OrtApi::GetValueInfoTypeInfo + ConstTypeInfo TypeInfo() const; + ///< Wraps OrtApi::ValueInfo_GetProducerNode + ValueInfoConsumerProducerInfo GetProducerNode() const; + /// < A wrapper around OrtApi::ValueInfo_GetValueConsumers + std::vector GetConsumers() const; + /// < A wrapper around OrtApi::ValueInfo_GetInitializerValue + Status GetInitializer(ConstValue& value) const; + /// < A wrapper around OrtApi::ValueInfo_GetExternalInitializerInfo + Status GetExternalInitializerInfo(ExternalInitializerInfo& info) const; + /// < A wrapper around OrtApi::ValueInfo_IsRequiredGraphInput + bool IsRequiredGraphInput() const; + /// < A wrapper around OrtApi::ValueInfo_IsOptionalGraphInput + bool IsOptionalGraphInput() const; + /// < A wrapper around OrtApi::ValueInfo_IsGraphOutput + bool IsGraphOutput() const; + /// < A wrapper around OrtApi::ValueInfo_IsConstantInitializer + bool IsConstantInitializer() const; + /// < A wrapper around OrtApi::ValueInfo_IsFromOuterScope + bool IsFromOuterScope() const; +}; +} // namespace detail + +// Const object holder that does not own the underlying object +using ConstValueInfo = detail::ConstValueInfoImpl>; + +/** \brief Wrapper around ::OrtValueInfo + * + */ +struct ValueInfo : detail::ConstValueInfoImpl { + ValueInfo() = default; // Same thing as with nullptr + explicit ValueInfo(std::nullptr_t) {} ///< No instance is created + /// Take ownership of a pointer created by C API + explicit ValueInfo(OrtValueInfo* p) : ConstValueInfoImpl{p} {} + +#if !defined(ORT_MINIMAL_BUILD) + // Create ValueInfo for a tensor + explicit ValueInfo(const std::string& name, const ConstTypeInfo& type_info); +#endif + ConstValueInfo GetConst() const { return ConstValueInfo{this->p_}; } +}; + +// Forward declaration +struct AttrNameSubgraph; + +namespace detail { +// Forward decl +template +struct ConstGraphImpl; + +template +struct ConstNodeImpl : Base { + using B = Base; + using B::B; + + // GetInputs() const; + // GetOutputs() const; + // GetImplicitInputs() const; + // GetAttributes() const; + // GetSubgraphs() const; + // > GetGraph() const; + // >; + +/** \brief Wrapper around ::OrtNode + * + */ +struct Node : detail::ConstNodeImpl { + Node() = default; // Same thing as with nullptr + explicit Node(std::nullptr_t) {} ///< No instance is created + explicit Node(OrtNode* p) : ConstNodeImpl{p} {} ///< Take ownership of a pointer created by C API + +#if !defined(ORT_MINIMAL_BUILD) + Node(const std::string& operator_name, const std::string& operator_domain, + const std::string& node_name, + const std::vector& input_names, + const std::vector& output_names); + + /// + /// Wraps CreateNode. Node takes ownership of attributes on success and updates the OpAttr in `attributes` to do so. + /// + Node(const std::string& operator_name, const std::string& operator_domain, + const std::string& node_name, + const std::vector& input_names, + const std::vector& output_names, + std::vector& attributes); + + private: + static void Init(const std::string& operator_name, const std::string& operator_domain, + const std::string& node_name, + const std::vector& input_names, + const std::vector& output_names, + std::vector& attributes, + OrtNode*& node); +#endif // !defined(ORT_MINIMAL_BUILD) +}; + +// Return struct for some of ValueInfo APIs. +// Must be declared after ConstNode is available. +struct ValueInfoConsumerProducerInfo { + ConstNode node; + // either producer output or consumer output index + // producer is unsigned only, output can be -1 + int64_t index; +}; + +// Represents a return value for Graph::GetOperatorSets() +struct OperatorSet { + std::string domain; + int64_t version; +}; + +namespace detail { +template +struct ConstGraphImpl : Base { + using B = Base; + using B::B; + + // GetModelPath() const; + // GetOperatorSets() const; + // GetInputs() const; + // GetOutputs() const; + // GetInitializers() const; + // GetNodes() const; + // & nodes) const; + // +struct GraphImpl : ConstGraphImpl { + using B = ConstGraphImpl; + using B::B; + +#if !defined(ORT_MINIMAL_BUILD) + // & inputs); + // & outputs); + // >; + +// Return value for Node API +// Must be declared after ConstGraph +struct AttrNameSubgraph { + std::string attr_name; + ConstGraph sub_graph; +}; + +/** \brief Wrapper around ::OrtGraph + * + */ +struct Graph : detail::GraphImpl { + explicit Graph(std::nullptr_t) {} ///< No instance is created + explicit Graph(OrtGraph* p) : GraphImpl{p} {} ///< Take ownership of a pointer created by C API +#if !defined(ORT_MINIMAL_BUILD) + // +struct ModelImpl : detail::Base { + using B = Ort::detail::Base; + using B::B; + +#if !defined(ORT_MINIMAL_BUILD) + // >; + +/** \brief Wrapper around ::OrtModel + * + */ +struct Model : detail::ModelImpl { + using DomainOpsetPair = std::pair; + + explicit Model(std::nullptr_t) {} ///< No instance is created + explicit Model(OrtModel* p) : ModelImpl{p} {} ///< Take ownership of a pointer created by C API + +#if !defined(ORT_MINIMAL_BUILD) + //< Wraps GetModelEditorApi().CreateModel() + explicit Model(const std::vector& opsets); +#endif +}; + +namespace detail { +template +struct ConstKernelDefImpl : Base { + using B = Base; + using B::B; + + ///< Wraps OrtEpApi::KernelDef_GetOperatorType + const char* GetOperatorType() const; + + ///< Wraps OrtEpApi::KernelDef_GetDomain + const char* GetDomain() const; + + ///< Wraps OrtEpApi::KernelDef_GetSinceVersion + std::pair GetSinceVersion() const; + + ///< Wraps OrtEpApi::KernelDef_GetExecutionProvider + const char* GetExecutionProvider() const; + + ///< Wraps OrtEpApi::KernelDef_GetInputMemType + OrtMemType GetInputMemType(size_t input_index) const; + + ///< Wraps OrtEpApi::KernelDef_GetOutputMemType + OrtMemType GetOutputMemType(size_t output_index) const; +}; +} // namespace detail + +using ConstKernelDef = detail::ConstKernelDefImpl>; + +struct KernelDef : detail::ConstKernelDefImpl { + using Base = detail::ConstKernelDefImpl; + using Base::Base; + + explicit KernelDef(std::nullptr_t) {} + explicit KernelDef(OrtKernelDef* p) : detail::ConstKernelDefImpl{p} {} + + ConstKernelDef GetConst() const { return ConstKernelDef{this->p_}; } +}; + +/** \brief Builder for OrtKernelDef. + * + * Used by plugin EPs to build a kernel definition. + */ +struct KernelDefBuilder : detail::Base { + KernelDefBuilder(); ///< Wraps OrtEpApi::CreateKernelDefBuilder + explicit KernelDefBuilder(std::nullptr_t) {} ///< Create an empty object, must be assigned a valid one to be used + explicit KernelDefBuilder(OrtKernelDefBuilder* ort_kernel_def_builder); + + KernelDefBuilder& SetOperatorType(const char* op_type); + KernelDefBuilder& SetDomain(const char* domain); + KernelDefBuilder& SetSinceVersion(int since_version_start, int since_version_end); + KernelDefBuilder& SetExecutionProvider(const char* ep_name); + KernelDefBuilder& SetInputMemType(size_t input_index, OrtMemType mem_type); + KernelDefBuilder& SetOutputMemType(size_t output_index, OrtMemType mem_type); + KernelDefBuilder& AddTypeConstraint(const char* arg_name, const OrtDataType* data_type); + KernelDefBuilder& AddTypeConstraint(const char* arg_name, const std::vector& data_types); + KernelDefBuilder& AddInputOutputAlias(int input_index, int output_index); + KernelDefBuilder& AddInputOutputAliases(const std::vector& input_indices, + const std::vector& output_indices); + KernelDefBuilder& AddInputOutputMutableAlias(int input_index, int output_index); + KernelDefBuilder& AddInputOutputMutableAliases(const std::vector& input_indices, + const std::vector& output_indices); + + KernelDef Build(); +}; + +/** \brief Registry for kernels supported by an EP. + * + * Used by plugin EPs to register definitions for supported kernels. + */ +struct KernelRegistry : detail::Base { + ///< Wrapper around OrtEpApi::CreateKernelRegistry + KernelRegistry(); + + ///< Create an empty object, must be assigned a valid one to be used + explicit KernelRegistry(std::nullptr_t) {} + + ///< Take ownership of a pointer created with the C API. + explicit KernelRegistry(OrtKernelRegistry* ort_kernel_registry); + + ///< Wraps KernelRegistry_AddKernel + Status AddKernel(const OrtKernelDef* kernel_def, OrtKernelCreateFunc kernel_create_func, + void* kernel_create_func_state); +}; + +namespace detail { +template +struct SharedPrePackedWeightCacheImpl : Ort::detail::Base { + using B = Ort::detail::Base; + using B::B; + + //< Wraps SharedPrePackedWeightCache_StoreWeightData + Status StoreWeightData(void** buffer_data_ptrs, size_t* buffer_sizes, size_t num_buffers); +}; +} // namespace detail + +/** \brief Convenience C++ wrapper class around a ::OrtSharedPrePackedWeightCache instance owned by ORT. + * + * An `OrtSharedPrePackedWeightCache*` instance is passed as an argument to OrtKernelImpl::PrePackWeight. + * Example use: + * OrtStatus* MyKernel::PrePackWeightImpl(OrtKernelImpl*, ..., OrtSharedPrePackedWeightCache* c_cache, ...) { + * ... + * if (c_cache != nullptr) { + * Ort::UnownedSharedPrePackedWeightCache cpp_cache(c_cache); + * Ort::Status status = cpp_cache.StoreWeightData(...); + * } + * ... + * } + * + * \remarks OrtSharedPrePackedWeightCache is always unowned, but mutable, for EpApi users. + */ +using UnownedSharedPrePackedWeightCache = + detail::SharedPrePackedWeightCacheImpl>; + +///< Wraps OrtEpApi::GetEnvConfigEntries() +Ort::KeyValuePairs GetEnvConfigEntries(); +} // namespace Ort +#include "onnxruntime_cxx_inline.h" diff --git a/dist/include/onnxruntime/onnxruntime_cxx_inline.h b/dist/include/onnxruntime/onnxruntime_cxx_inline.h new file mode 100644 index 0000000..1a3e491 --- /dev/null +++ b/dist/include/onnxruntime/onnxruntime_cxx_inline.h @@ -0,0 +1,3896 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// Do not include this file directly. Please include "onnxruntime_cxx_api.h" instead. +// If interested in trying out features of the new experimental C++ API, include "experimental_onnxruntime_cxx_api.h" instead. +// +// These are the inline implementations of the C++ header APIs. They're in this separate file as to not clutter +// the main C++ file with implementation details. + +#include +#include +#include +#include +#include +#include +#include + +// Convert OrtStatus to Ort::Status and return +// instead of throwing +#define ORT_CXX_RETURN_ON_API_FAIL(expression) \ + { \ + auto ort_status = (expression); \ + if (ort_status) { \ + return Ort::Status(ort_status); \ + } \ + } + +#ifdef __cpp_if_constexpr +#define ORT_CXX_IF_CONSTEXPR if constexpr +#else +#define ORT_CXX_IF_CONSTEXPR if +#endif + +namespace Ort { + +namespace detail { +inline void ThrowStatus(const Status& st) { + std::string error_message = st.GetErrorMessage(); + OrtErrorCode error_code = st.GetErrorCode(); + ORT_CXX_API_THROW(std::move(error_message), error_code); +} +} // namespace detail + +inline void ThrowOnError(OrtStatus* ort_status) { + if (ort_status) { + Ort::Status st(ort_status); + detail::ThrowStatus(st); + } +} + +inline void ThrowOnError(const Status& st) { + if (st) { + detail::ThrowStatus(st); + } +} + +inline Status::Status(OrtStatus* status) noexcept : detail::Base{status} { +} + +inline Status::Status(const std::exception& e) { + p_ = GetApi().CreateStatus(ORT_FAIL, e.what()); +} + +inline Status::Status(const Exception& e) { + p_ = GetApi().CreateStatus(e.GetOrtErrorCode(), e.what()); +} + +inline Status::Status(const char* message, OrtErrorCode code) { + p_ = GetApi().CreateStatus(code, message); +} + +inline std::string Status::GetErrorMessage() const { + std::string message(GetApi().GetErrorMessage(p_)); + return message; +} + +inline OrtErrorCode Status::GetErrorCode() const { + return GetApi().GetErrorCode(p_); +} + +inline bool Status::IsOK() const noexcept { + return (p_ == nullptr); +} + +// This template converts a C++ type into it's ONNXTensorElementDataType +template +struct TypeToTensorType; +template <> +struct TypeToTensorType { + static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT; +}; +template <> +struct TypeToTensorType { + static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16; +}; +template <> +struct TypeToTensorType { + static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16; +}; +template <> +struct TypeToTensorType { + static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE; +}; +template <> +struct TypeToTensorType { + static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8; +}; +template <> +struct TypeToTensorType { + static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16; +}; +template <> +struct TypeToTensorType { + static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32; +}; +template <> +struct TypeToTensorType { + static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64; +}; +template <> +struct TypeToTensorType { + static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8; +}; +template <> +struct TypeToTensorType { + static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16; +}; +template <> +struct TypeToTensorType { + static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32; +}; +template <> +struct TypeToTensorType { + static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64; +}; +template <> +struct TypeToTensorType { + static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL; +}; + +template <> +struct TypeToTensorType { + static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E4M3FN; +}; +template <> +struct TypeToTensorType { + static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E4M3FNUZ; +}; +template <> +struct TypeToTensorType { + static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E5M2; +}; +template <> +struct TypeToTensorType { + static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E5M2FNUZ; +}; + +inline bool BFloat16_t::operator==(const BFloat16_t& rhs) const noexcept { + if (IsNaN() || rhs.IsNaN()) { + // IEEE defines that NaN is not equal to anything, including itself. + return false; + } + return val == rhs.val; +} + +inline bool BFloat16_t::operator<(const BFloat16_t& rhs) const noexcept { + if (IsNaN() || rhs.IsNaN()) { + // IEEE defines that NaN is unordered with respect to everything, including itself. + return false; + } + + const bool left_is_negative = IsNegative(); + if (left_is_negative != rhs.IsNegative()) { + // When the signs of left and right differ, we know that left is less than right if it is + // the negative value. The exception to this is if both values are zero, in which case IEEE + // says they should be equal, even if the signs differ. + return left_is_negative && !AreZero(*this, rhs); + } + return (val != rhs.val) && ((val < rhs.val) ^ left_is_negative); +} + +inline MemoryAllocation::MemoryAllocation(OrtAllocator* allocator, void* p, size_t size) + : allocator_(allocator), p_(p), size_(size) { +} + +inline MemoryAllocation::~MemoryAllocation() { + if (p_ != nullptr) { + // We do not throw out of destructor + auto ret = GetApi().AllocatorFree(allocator_, p_); + static_cast(ret); + } +} + +inline MemoryAllocation::MemoryAllocation(MemoryAllocation&& o) noexcept : allocator_(nullptr), p_(nullptr), size_(0) { + *this = std::move(o); +} + +inline MemoryAllocation& MemoryAllocation::operator=(MemoryAllocation&& o) noexcept { + OrtAllocator* alloc = nullptr; + void* p = nullptr; + size_t sz = 0; + + // Swap out this + std::swap(alloc, allocator_); + std::swap(p, p_); + std::swap(sz, size_); + + // Swap with incoming + std::swap(allocator_, o.allocator_); + std::swap(p_, o.p_); + std::swap(size_, o.size_); + + // Destroy this instance if needed + MemoryAllocation this_alloc(alloc, p, sz); + return *this; +} + +namespace detail { + +template +inline void* AllocatorImpl::Alloc(size_t size) { + void* out; + ThrowOnError(GetApi().AllocatorAlloc(this->p_, size, &out)); + return out; +} + +template +inline MemoryAllocation AllocatorImpl::GetAllocation(size_t size) { + void* out; + ThrowOnError(GetApi().AllocatorAlloc(this->p_, size, &out)); + MemoryAllocation result(this->p_, out, size); + return result; +} + +template +inline void AllocatorImpl::Free(void* p) { + ThrowOnError(GetApi().AllocatorFree(this->p_, p)); +} + +template +inline ConstMemoryInfo AllocatorImpl::GetInfo() const { + const OrtMemoryInfo* out; + ThrowOnError(GetApi().AllocatorGetInfo(this->p_, &out)); + return ConstMemoryInfo{out}; +} + +template +inline KeyValuePairs AllocatorImpl::GetStats() const { + OrtKeyValuePairs* out; + ThrowOnError(GetApi().AllocatorGetStats(this->p_, &out)); + return KeyValuePairs(out); +} +} // namespace detail + +inline AllocatorWithDefaultOptions::AllocatorWithDefaultOptions() { + ThrowOnError(GetApi().GetAllocatorWithDefaultOptions(&this->p_)); +} + +inline Allocator::Allocator(const Session& sess, const OrtMemoryInfo* mem_info) { + ThrowOnError(GetApi().CreateAllocator(sess, mem_info, &this->p_)); +} + +namespace detail { + +template +inline std::string MemoryInfoImpl::GetAllocatorName() const { + const char* name = nullptr; + ThrowOnError(GetApi().MemoryInfoGetName(this->p_, &name)); + return std::string(name); +} + +template +inline OrtAllocatorType MemoryInfoImpl::GetAllocatorType() const { + OrtAllocatorType type; + ThrowOnError(GetApi().MemoryInfoGetType(this->p_, &type)); + return type; +} + +template +inline int MemoryInfoImpl::GetDeviceId() const { + int id = 0; + ThrowOnError(GetApi().MemoryInfoGetId(this->p_, &id)); + return id; +} + +template +inline OrtMemoryInfoDeviceType MemoryInfoImpl::GetDeviceType() const { + OrtMemoryInfoDeviceType type; + GetApi().MemoryInfoGetDeviceType(this->p_, &type); + return type; +} + +template +inline OrtMemType MemoryInfoImpl::GetMemoryType() const { + OrtMemType type; + ThrowOnError(GetApi().MemoryInfoGetMemType(this->p_, &type)); + return type; +} + +template +inline OrtDeviceMemoryType MemoryInfoImpl::GetDeviceMemoryType() const { + return GetApi().MemoryInfoGetDeviceMemType(this->p_); +} + +template +inline uint32_t MemoryInfoImpl::GetVendorId() const { + return GetApi().MemoryInfoGetVendorId(this->p_); +} + +template +template +inline bool MemoryInfoImpl::operator==(const MemoryInfoImpl& o) const { + int comp_result = 0; + ThrowOnError(Ort::GetApi().CompareMemoryInfo(this->p_, o, &comp_result)); + return comp_result == 0; +} + +} // namespace detail + +inline MemoryInfo MemoryInfo::CreateCpu(OrtAllocatorType type, OrtMemType mem_type) { + OrtMemoryInfo* p; + ThrowOnError(GetApi().CreateCpuMemoryInfo(type, mem_type, &p)); + return MemoryInfo(p); +} + +inline MemoryInfo::MemoryInfo(const char* name, OrtAllocatorType type, int id, OrtMemType mem_type) { + ThrowOnError(GetApi().CreateMemoryInfo(name, type, id, mem_type, &this->p_)); +} + +inline MemoryInfo::MemoryInfo(const char* name, OrtMemoryInfoDeviceType device_type, uint32_t vendor_id, uint32_t device_id, + OrtDeviceMemoryType mem_type, size_t alignment, OrtAllocatorType allocator_type) { + ThrowOnError(GetApi().CreateMemoryInfo_V2(name, device_type, vendor_id, device_id, mem_type, alignment, + allocator_type, &this->p_)); +} + +namespace detail { +template +inline std::vector ConstIoBindingImpl::GetOutputNames() const { + AllocatorWithDefaultOptions allocator; + return binding_utils::GetOutputNamesHelper(this->p_, allocator); +} + +template +inline std::vector ConstIoBindingImpl::GetOutputNames(OrtAllocator* allocator) const { + return binding_utils::GetOutputNamesHelper(this->p_, allocator); +} + +template +inline std::vector ConstIoBindingImpl::GetOutputValues() const { + AllocatorWithDefaultOptions allocator; + return binding_utils::GetOutputValuesHelper(this->p_, allocator); +} + +template +inline std::vector ConstIoBindingImpl::GetOutputValues(OrtAllocator* allocator) const { + return binding_utils::GetOutputValuesHelper(this->p_, allocator); +} + +template +inline void IoBindingImpl::BindInput(const char* name, const Value& value) { + ThrowOnError(GetApi().BindInput(this->p_, name, value)); +} + +template +inline void IoBindingImpl::BindOutput(const char* name, const Value& value) { + ThrowOnError(GetApi().BindOutput(this->p_, name, value)); +} + +template +inline void IoBindingImpl::BindOutput(const char* name, const OrtMemoryInfo* mem_info) { + ThrowOnError(GetApi().BindOutputToDevice(this->p_, name, mem_info)); +} + +template +inline void IoBindingImpl::ClearBoundInputs() { + GetApi().ClearBoundInputs(this->p_); +} + +template +inline void IoBindingImpl::ClearBoundOutputs() { + GetApi().ClearBoundOutputs(this->p_); +} + +template +inline void IoBindingImpl::SynchronizeInputs() { + ThrowOnError(GetApi().SynchronizeBoundInputs(this->p_)); +} + +template +inline void IoBindingImpl::SynchronizeOutputs() { + ThrowOnError(GetApi().SynchronizeBoundOutputs(this->p_)); +} + +namespace binding_utils { +inline std::vector GetOutputNamesHelper(const OrtIoBinding* binding, OrtAllocator* allocator) { + std::vector result; + auto free_fn = detail::AllocatedFree(allocator); + using Ptr = std::unique_ptr; + + char* buffer = nullptr; + size_t* lengths = nullptr; + size_t count = 0; + ThrowOnError(GetApi().GetBoundOutputNames(binding, allocator, &buffer, &lengths, &count)); + + if (count == 0) { + return result; + } + + Ptr buffer_g(buffer, free_fn); + Ptr lengths_g(lengths, free_fn); + + result.reserve(count); + for (size_t i = 0; i < count; ++i) { + auto sz = *lengths; + result.emplace_back(buffer, sz); + buffer += sz; + ++lengths; + } + return result; +} + +inline std::vector GetOutputValuesHelper(const OrtIoBinding* binding, OrtAllocator* allocator) { + std::vector result; + size_t output_count = 0; + + OrtValue** output_buffer = nullptr; + ThrowOnError(GetApi().GetBoundOutputValues(binding, allocator, &output_buffer, &output_count)); + if (output_count == 0) { + return result; + } + + std::unique_ptr buffer_g(output_buffer, AllocatedFree(allocator)); + + result.reserve(output_count); + for (size_t i = 0; i < output_count; ++i) { + result.emplace_back(output_buffer[i]); + } + return result; +} + +} // namespace binding_utils +} // namespace detail + +inline IoBinding::IoBinding(Session& session) { + ThrowOnError(GetApi().CreateIoBinding(session, &this->p_)); +} + +inline ArenaCfg::ArenaCfg(size_t max_mem, int arena_extend_strategy, int initial_chunk_size_bytes, int max_dead_bytes_per_chunk) { + ThrowOnError(GetApi().CreateArenaCfg(max_mem, arena_extend_strategy, initial_chunk_size_bytes, max_dead_bytes_per_chunk, &p_)); +} + +inline ArenaCfg::ArenaCfg(const std::unordered_map& arena_config) { + std::vector keys; + std::vector values; + keys.reserve(arena_config.size()); + values.reserve(arena_config.size()); + for (const auto& kv : arena_config) { + keys.push_back(kv.first.c_str()); + values.push_back(kv.second); + } + ThrowOnError(GetApi().CreateArenaCfgV2(keys.data(), values.data(), arena_config.size(), &p_)); +} + +inline ThreadingOptions::ThreadingOptions() { + ThrowOnError(GetApi().CreateThreadingOptions(&p_)); +} + +inline ThreadingOptions& ThreadingOptions::SetGlobalIntraOpNumThreads(int intra_op_num_threads) { + ThrowOnError(GetApi().SetGlobalIntraOpNumThreads(p_, intra_op_num_threads)); + return *this; +} + +inline ThreadingOptions& ThreadingOptions::SetGlobalInterOpNumThreads(int inter_op_num_threads) { + ThrowOnError(GetApi().SetGlobalInterOpNumThreads(p_, inter_op_num_threads)); + return *this; +} + +inline ThreadingOptions& ThreadingOptions::SetGlobalSpinControl(int allow_spinning) { + ThrowOnError(GetApi().SetGlobalSpinControl(p_, allow_spinning)); + return *this; +} + +inline ThreadingOptions& ThreadingOptions::SetGlobalDenormalAsZero() { + ThrowOnError(GetApi().SetGlobalDenormalAsZero(p_)); + return *this; +} + +inline ThreadingOptions& ThreadingOptions::SetGlobalCustomCreateThreadFn(OrtCustomCreateThreadFn ort_custom_create_thread_fn) { + ThrowOnError(GetApi().SetGlobalCustomCreateThreadFn(p_, ort_custom_create_thread_fn)); + return *this; +} + +inline ThreadingOptions& ThreadingOptions::SetGlobalCustomThreadCreationOptions(void* ort_custom_thread_creation_options) { + ThrowOnError(GetApi().SetGlobalCustomThreadCreationOptions(p_, ort_custom_thread_creation_options)); + return *this; +} + +inline ThreadingOptions& ThreadingOptions::SetGlobalCustomJoinThreadFn(OrtCustomJoinThreadFn ort_custom_join_thread_fn) { + ThrowOnError(GetApi().SetGlobalCustomJoinThreadFn(p_, ort_custom_join_thread_fn)); + return *this; +} + +inline TensorRTProviderOptions::TensorRTProviderOptions() { + ThrowOnError(GetApi().CreateTensorRTProviderOptions(&this->p_)); +} + +inline void TensorRTProviderOptions::Update(const std::unordered_map& options) { + std::vector keys; + std::vector values; + keys.reserve(options.size()); + values.reserve(options.size()); + for (const auto& kv : options) { + keys.push_back(kv.first.c_str()); + values.push_back(kv.second.c_str()); + } + ThrowOnError(GetApi().UpdateTensorRTProviderOptions(p_, keys.data(), values.data(), options.size())); +} + +inline void TensorRTProviderOptions::UpdateWithValue(const char* key, void* value) { + ThrowOnError(GetApi().UpdateTensorRTProviderOptionsWithValue(p_, key, value)); +} + +inline void* TensorRTProviderOptions::GetOptionByName(const char* name) const { + void* value = nullptr; + ThrowOnError(GetApi().GetTensorRTProviderOptionsByName(p_, name, &value)); + return value; +} + +inline std::string TensorRTProviderOptions::GetTensorRTProviderOptionsAsString() const { + AllocatorWithDefaultOptions allocator; + char* options_str = nullptr; + ThrowOnError(GetApi().GetTensorRTProviderOptionsAsString(p_, allocator, &options_str)); + std::unique_ptr options_str_g(options_str, detail::AllocatedFree(allocator)); + return std::string(options_str); +} + +inline CUDAProviderOptions::CUDAProviderOptions() { + ThrowOnError(GetApi().CreateCUDAProviderOptions(&this->p_)); +} + +inline void CUDAProviderOptions::Update(const std::unordered_map& options) { + std::vector keys; + std::vector values; + keys.reserve(options.size()); + values.reserve(options.size()); + for (const auto& kv : options) { + keys.push_back(kv.first.c_str()); + values.push_back(kv.second.c_str()); + } + ThrowOnError(GetApi().UpdateCUDAProviderOptions(p_, keys.data(), values.data(), options.size())); +} + +inline std::string CUDAProviderOptions::GetCUDAProviderOptionsAsString() const { + AllocatorWithDefaultOptions allocator; + char* options_str = nullptr; + ThrowOnError(GetApi().GetCUDAProviderOptionsAsString(p_, allocator, &options_str)); + std::unique_ptr options_str_g(options_str, detail::AllocatedFree(allocator)); + return std::string(options_str); +} + +inline void CUDAProviderOptions::UpdateWithValue(const char* key, void* value) { + ThrowOnError(GetApi().UpdateCUDAProviderOptionsWithValue(p_, key, value)); +} + +inline void* CUDAProviderOptions::GetOptionByName(const char* name) const { + void* value = nullptr; + ThrowOnError(GetApi().GetCUDAProviderOptionsByName(p_, name, &value)); + return value; +} + +inline PrepackedWeightsContainer::PrepackedWeightsContainer() { + ThrowOnError(GetApi().CreatePrepackedWeightsContainer(&this->p_)); +} + +namespace detail { + +template +inline const std::basic_string ConstExternalInitializerInfoImpl::GetFilePath() const { + return GetApi().ExternalInitializerInfo_GetFilePath(this->p_); +} + +template +inline int64_t ConstExternalInitializerInfoImpl::GetFileOffset() const { + return GetApi().ExternalInitializerInfo_GetFileOffset(this->p_); +} + +template +inline size_t ConstExternalInitializerInfoImpl::GetByteSize() const { + return GetApi().ExternalInitializerInfo_GetByteSize(this->p_); +} +} // namespace detail + +inline ExternalInitializerInfo::ExternalInitializerInfo(const ORTCHAR_T* filepath, int64_t file_offset, + size_t byte_size) { + ThrowOnError(GetApi().CreateExternalInitializerInfo(filepath, file_offset, byte_size, &this->p_)); +} + +inline Status ExternalInitializerInfo::Create(const ORTCHAR_T* filepath, int64_t file_offset, size_t byte_size, + /*out*/ ExternalInitializerInfo& out) { + OrtExternalInitializerInfo* info = nullptr; + OrtStatus* status = GetApi().CreateExternalInitializerInfo(filepath, file_offset, byte_size, &info); + if (status != nullptr) { + return Status{status}; + } + + out = ExternalInitializerInfo(info); + + return Status{nullptr}; +} + +namespace detail { +template +inline const char* KeyValuePairsImpl::GetValue(const char* key) const { + return GetApi().GetKeyValue(this->p_, key); +} + +template +inline std::unordered_map KeyValuePairsImpl::GetKeyValuePairs() const { + std::unordered_map out; + + size_t num_pairs = 0; + const char* const* keys = nullptr; + const char* const* values = nullptr; + GetApi().GetKeyValuePairs(this->p_, &keys, &values, &num_pairs); + if (num_pairs > 0) { + out.reserve(num_pairs); + for (size_t i = 0; i < num_pairs; ++i) { + out.emplace(keys[i], values[i]); + } + } + + return out; +} + +template +inline void KeyValuePairsImpl::GetKeyValuePairs(std::vector& keys, + std::vector& values) const { + keys.clear(); + values.clear(); + + size_t num_pairs = 0; + const char* const* keys_ptr = nullptr; + const char* const* values_ptr = nullptr; + GetApi().GetKeyValuePairs(this->p_, &keys_ptr, &values_ptr, &num_pairs); + if (num_pairs > 0) { + keys.resize(num_pairs); + values.resize(num_pairs); + std::copy(keys_ptr, keys_ptr + num_pairs, keys.begin()); + std::copy(values_ptr, values_ptr + num_pairs, values.begin()); + } +} +} // namespace detail + +inline KeyValuePairs::KeyValuePairs() { + GetApi().CreateKeyValuePairs(&p_); +} + +inline KeyValuePairs::KeyValuePairs(const std::unordered_map& kv_pairs) { + GetApi().CreateKeyValuePairs(&p_); + for (const auto& kv : kv_pairs) { + GetApi().AddKeyValuePair(this->p_, kv.first.c_str(), kv.second.c_str()); + } +} + +inline void KeyValuePairs::Add(const char* key, const char* value) { + GetApi().AddKeyValuePair(this->p_, key, value); +} + +inline void KeyValuePairs::Remove(const char* key) { + GetApi().RemoveKeyValuePair(this->p_, key); +} + +namespace detail { +template +inline void* SyncStreamImpl::GetHandle() { + return GetApi().SyncStream_GetHandle(this->p_); +} +} // namespace detail + +namespace detail { +template +inline OrtHardwareDeviceType HardwareDeviceImpl::Type() const { + return GetApi().HardwareDevice_Type(this->p_); +} + +template +inline uint32_t HardwareDeviceImpl::VendorId() const { + return GetApi().HardwareDevice_VendorId(this->p_); +} + +template +inline uint32_t HardwareDeviceImpl::DeviceId() const { + return GetApi().HardwareDevice_DeviceId(this->p_); +} + +template +inline const char* HardwareDeviceImpl::Vendor() const { + return GetApi().HardwareDevice_Vendor(this->p_); +} + +template +inline ConstKeyValuePairs HardwareDeviceImpl::Metadata() const { + return ConstKeyValuePairs{GetApi().HardwareDevice_Metadata(this->p_)}; +} + +template +inline const char* EpDeviceImpl::EpName() const { + return GetApi().EpDevice_EpName(this->p_); +} + +template +inline const char* EpDeviceImpl::EpVendor() const { + return GetApi().EpDevice_EpVendor(this->p_); +} + +template +inline ConstKeyValuePairs EpDeviceImpl::EpMetadata() const { + return ConstKeyValuePairs(GetApi().EpDevice_EpMetadata(this->p_)); +} + +template +inline ConstKeyValuePairs EpDeviceImpl::EpOptions() const { + return ConstKeyValuePairs(GetApi().EpDevice_EpOptions(this->p_)); +} + +template +inline ConstHardwareDevice EpDeviceImpl::Device() const { + return ConstHardwareDevice(GetApi().EpDevice_Device(this->p_)); +} + +template +inline ConstMemoryInfo EpDeviceImpl::GetMemoryInfo(OrtDeviceMemoryType memory_type) const { + const auto* mem_info = GetApi().EpDevice_MemoryInfo(this->p_, memory_type); + return ConstMemoryInfo{mem_info}; +} + +template +inline SyncStream EpDeviceImpl::CreateSyncStream(ConstKeyValuePairs stream_options) const { + OrtSyncStream* stream = nullptr; + ThrowOnError(GetApi().CreateSyncStreamForEpDevice(this->p_, stream_options, &stream)); + return SyncStream{stream}; +} +} // namespace detail + +inline EpDevice::EpDevice(OrtEpFactory& ep_factory, ConstHardwareDevice& hardware_device, + ConstKeyValuePairs ep_metadata, ConstKeyValuePairs ep_options) { + ThrowOnError(GetEpApi().CreateEpDevice(&ep_factory, hardware_device, ep_metadata, ep_options, &p_)); +} + +namespace detail { +template +inline std::string EpAssignedSubgraphImpl::GetEpName() const { + const char* ep_name = nullptr; + + // Returned null-terminated string will not be null if API function returns successfully. + ThrowOnError(GetApi().EpAssignedSubgraph_GetEpName(this->p_, &ep_name)); + return std::string(ep_name); +} + +template +inline std::vector EpAssignedSubgraphImpl::GetNodes() const { + size_t num_ep_nodes = 0; + const OrtEpAssignedNode* const* ep_node_ptrs = nullptr; + ThrowOnError(GetApi().EpAssignedSubgraph_GetNodes(this->p_, &ep_node_ptrs, &num_ep_nodes)); + + std::vector ep_nodes; + if (num_ep_nodes > 0) { + ep_nodes.reserve(num_ep_nodes); + for (size_t i = 0; i < num_ep_nodes; ++i) { + ep_nodes.emplace_back(ep_node_ptrs[i]); + } + } + + return ep_nodes; +} + +template +inline std::string EpAssignedNodeImpl::GetName() const { + const char* node_name = nullptr; + + // Returned null-terminated string will not be null if API function returns successfully. + ThrowOnError(GetApi().EpAssignedNode_GetName(this->p_, &node_name)); + return std::string(node_name); +} + +template +inline std::string EpAssignedNodeImpl::GetDomain() const { + const char* domain = nullptr; + + // Returned null-terminated string will not be null if API function returns successfully. + ThrowOnError(GetApi().EpAssignedNode_GetDomain(this->p_, &domain)); + return std::string(domain); +} + +template +inline std::string EpAssignedNodeImpl::GetOperatorType() const { + const char* op_type = nullptr; + + // Returned null-terminated string will not be null if API function returns successfully. + ThrowOnError(GetApi().EpAssignedNode_GetOperatorType(this->p_, &op_type)); + return std::string(op_type); +} +} // namespace detail + +inline Env::Env(OrtLoggingLevel logging_level, _In_ const char* logid) { + ThrowOnError(GetApi().CreateEnv(logging_level, logid, &p_)); + if (strcmp(logid, "onnxruntime-node") == 0) { + ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_NODEJS)); + } else { + ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_CPLUSPLUS)); + } +} + +inline Env::Env(OrtLoggingLevel logging_level, const char* logid, OrtLoggingFunction logging_function, void* logger_param) { + ThrowOnError(GetApi().CreateEnvWithCustomLogger(logging_function, logger_param, logging_level, logid, &p_)); + if (strcmp(logid, "onnxruntime-node") == 0) { + ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_NODEJS)); + } else { + ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_CPLUSPLUS)); + } +} + +inline Env::Env(const OrtThreadingOptions* tp_options, OrtLoggingLevel logging_level, _In_ const char* logid) { + ThrowOnError(GetApi().CreateEnvWithGlobalThreadPools(logging_level, logid, tp_options, &p_)); + if (strcmp(logid, "onnxruntime-node") == 0) { + ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_NODEJS)); + } else { + ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_CPLUSPLUS)); + } +} + +inline Env::Env(const OrtThreadingOptions* tp_options, OrtLoggingFunction logging_function, void* logger_param, + OrtLoggingLevel logging_level, _In_ const char* logid) { + ThrowOnError(GetApi().CreateEnvWithCustomLoggerAndGlobalThreadPools(logging_function, logger_param, logging_level, logid, tp_options, &p_)); + if (strcmp(logid, "onnxruntime-node") == 0) { + ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_NODEJS)); + } else { + ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_CPLUSPLUS)); + } +} + +inline Env::Env(const OrtEnvCreationOptions* options) { + ThrowOnError(GetApi().CreateEnvWithOptions(options, &p_)); + if (strcmp(options->log_id, "onnxruntime-node") == 0) { + ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_NODEJS)); + } else { + ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_CPLUSPLUS)); + } +} + +inline Env& Env::EnableTelemetryEvents() { + ThrowOnError(GetApi().EnableTelemetryEvents(p_)); + return *this; +} + +inline Env& Env::DisableTelemetryEvents() { + ThrowOnError(GetApi().DisableTelemetryEvents(p_)); + return *this; +} + +inline Env& Env::UpdateEnvWithCustomLogLevel(OrtLoggingLevel log_severity_level) { + ThrowOnError(GetApi().UpdateEnvWithCustomLogLevel(p_, log_severity_level)); + return *this; +} + +inline Env& Env::CreateAndRegisterAllocator(const OrtMemoryInfo* mem_info, const OrtArenaCfg* arena_cfg) { + ThrowOnError(GetApi().CreateAndRegisterAllocator(p_, mem_info, arena_cfg)); + return *this; +} + +inline Env& Env::CreateAndRegisterAllocatorV2(const std::string& provider_type, const OrtMemoryInfo* mem_info, const std::unordered_map& options, const OrtArenaCfg* arena_cfg) { + std::vector keys, values; + auto num_entries = options.size(); + if (num_entries > 0) { + keys.reserve(num_entries); + values.reserve(num_entries); + for (const auto& entry : options) { + keys.push_back(entry.first.c_str()); + values.push_back(entry.second.c_str()); + } + } + ThrowOnError(GetApi().CreateAndRegisterAllocatorV2(p_, provider_type.c_str(), mem_info, arena_cfg, keys.data(), values.data(), num_entries)); + return *this; +} + +inline Env& Env::RegisterAllocator(OrtAllocator* allocator) { + ThrowOnError(GetApi().RegisterAllocator(p_, allocator)); + return *this; +} + +inline Env& Env::UnregisterAllocator(const OrtMemoryInfo* mem_info) { + ThrowOnError(GetApi().UnregisterAllocator(p_, mem_info)); + return *this; +} + +inline Env& Env::RegisterExecutionProviderLibrary(const char* registration_name, + const std::basic_string& path) { + ThrowOnError(GetApi().RegisterExecutionProviderLibrary(p_, registration_name, path.c_str())); + return *this; +} + +inline Env& Env::UnregisterExecutionProviderLibrary(const char* registration_name) { + ThrowOnError(GetApi().UnregisterExecutionProviderLibrary(p_, registration_name)); + return *this; +} + +inline std::vector Env::GetEpDevices() const { + size_t num_devices = 0; + const OrtEpDevice* const* device_ptrs = nullptr; + ThrowOnError(GetApi().GetEpDevices(p_, &device_ptrs, &num_devices)); + + std::vector devices; + if (num_devices > 0) { + devices.reserve(num_devices); + for (size_t i = 0; i < num_devices; ++i) { + devices.emplace_back(device_ptrs[i]); + } + } + + return devices; +} + +inline Status Env::CopyTensors(const std::vector& src_tensors, + const std::vector& dst_tensors, + OrtSyncStream* stream) const { + if (src_tensors.size() != dst_tensors.size()) { + return Status("Source and destination tensor vectors must have the same size", ORT_INVALID_ARGUMENT); + } + if (src_tensors.empty()) { + return Status(nullptr); + } + + const OrtValue* const* src_tensors_ptr = reinterpret_cast(src_tensors.data()); + OrtValue* const* dst_tensors_ptr = reinterpret_cast(dst_tensors.data()); + OrtStatus* status = GetApi().CopyTensors(p_, src_tensors_ptr, dst_tensors_ptr, stream, src_tensors.size()); + return Status(status); +} + +inline UnownedAllocator Env::CreateSharedAllocator(const OrtEpDevice* ep_device, OrtDeviceMemoryType mem_type, + OrtAllocatorType allocator_type, + const OrtKeyValuePairs* allocator_options) { + OrtAllocator* p; + ThrowOnError(GetApi().CreateSharedAllocator(p_, ep_device, mem_type, allocator_type, allocator_options, &p)); + return UnownedAllocator{p}; +} + +inline UnownedAllocator Env::GetSharedAllocator(const OrtMemoryInfo* mem_info) { + OrtAllocator* p; + ThrowOnError(GetApi().GetSharedAllocator(p_, mem_info, &p)); + return UnownedAllocator{p}; +} + +inline void Env::ReleaseSharedAllocator(const OrtEpDevice* ep_device, + OrtDeviceMemoryType mem_type) { + ThrowOnError(GetApi().ReleaseSharedAllocator(p_, ep_device, mem_type)); +} + +inline CustomOpDomain::CustomOpDomain(const char* domain) { + ThrowOnError(GetApi().CreateCustomOpDomain(domain, &p_)); +} + +inline void CustomOpDomain::Add(const OrtCustomOp* op) { + ThrowOnError(GetApi().CustomOpDomain_Add(p_, op)); +} + +inline OrtCompiledModelCompatibility GetModelCompatibilityForEpDevices( + const std::vector& ep_devices, + const char* compatibility_info) { + if (ep_devices.empty()) { + ORT_CXX_API_THROW("ep_devices is empty", ORT_INVALID_ARGUMENT); + } + + std::vector ptrs; + ptrs.reserve(ep_devices.size()); + for (const auto& d : ep_devices) ptrs.push_back(d); + + OrtCompiledModelCompatibility status = OrtCompiledModelCompatibility_EP_NOT_APPLICABLE; + ThrowOnError(GetApi().GetModelCompatibilityForEpDevices( + reinterpret_cast(ptrs.data()), + ptrs.size(), + compatibility_info, + &status)); + return status; +} + +inline AllocatedStringPtr GetCompatibilityInfoFromModelAllocated(const ORTCHAR_T* model_path, const char* ep_type, + OrtAllocator* allocator) { + char* compat_info = nullptr; + ThrowOnError(GetApi().GetCompatibilityInfoFromModel(model_path, ep_type, allocator, &compat_info)); + return AllocatedStringPtr(compat_info, detail::AllocatedFree(allocator)); +} + +inline AllocatedStringPtr GetCompatibilityInfoFromModelBytesAllocated(const void* model_data, size_t model_data_length, + const char* ep_type, OrtAllocator* allocator) { + char* compat_info = nullptr; + ThrowOnError(GetApi().GetCompatibilityInfoFromModelBytes(model_data, model_data_length, ep_type, allocator, &compat_info)); + return AllocatedStringPtr(compat_info, detail::AllocatedFree(allocator)); +} + +inline LoraAdapter LoraAdapter::CreateLoraAdapter(const std::basic_string& adapter_path, + OrtAllocator* allocator) { + OrtLoraAdapter* p; + ThrowOnError(GetApi().CreateLoraAdapter(adapter_path.c_str(), allocator, &p)); + return LoraAdapter{p}; +} + +inline LoraAdapter LoraAdapter::CreateLoraAdapterFromArray(const void* bytes, size_t num_bytes, + OrtAllocator* allocator) { + OrtLoraAdapter* p; + ThrowOnError(GetApi().CreateLoraAdapterFromArray(bytes, num_bytes, allocator, &p)); + return LoraAdapter{p}; +} + +inline RunOptions::RunOptions() { + ThrowOnError(GetApi().CreateRunOptions(&p_)); +} + +inline RunOptions& RunOptions::SetRunLogVerbosityLevel(int level) { + ThrowOnError(GetApi().RunOptionsSetRunLogVerbosityLevel(p_, level)); + return *this; +} + +inline RunOptions& RunOptions::SetRunLogSeverityLevel(int level) { + ThrowOnError(GetApi().RunOptionsSetRunLogSeverityLevel(p_, level)); + return *this; +} + +inline int RunOptions::GetRunLogVerbosityLevel() const { + int out; + ThrowOnError(GetApi().RunOptionsGetRunLogVerbosityLevel(p_, &out)); + return out; +} + +inline int RunOptions::GetRunLogSeverityLevel() const { + int out; + ThrowOnError(GetApi().RunOptionsGetRunLogSeverityLevel(p_, &out)); + return out; +} + +inline RunOptions& RunOptions::SetRunTag(const char* run_tag) { + ThrowOnError(GetApi().RunOptionsSetRunTag(p_, run_tag)); + return *this; +} + +inline const char* RunOptions::GetRunTag() const { + const char* out; + ThrowOnError(GetApi().RunOptionsGetRunTag(p_, &out)); + return out; +} + +inline RunOptions& RunOptions::AddConfigEntry(const char* config_key, const char* config_value) { + ThrowOnError(GetApi().AddRunConfigEntry(p_, config_key, config_value)); + return *this; +} + +inline const char* RunOptions::GetConfigEntry(const char* config_key) { + return GetApi().GetRunConfigEntry(p_, config_key); +} + +inline RunOptions& RunOptions::SetTerminate() { + ThrowOnError(GetApi().RunOptionsSetTerminate(p_)); + return *this; +} + +inline RunOptions& RunOptions::UnsetTerminate() { + ThrowOnError(GetApi().RunOptionsUnsetTerminate(p_)); + return *this; +} + +inline RunOptions& RunOptions::AddActiveLoraAdapter(const LoraAdapter& adapter) { + ThrowOnError(GetApi().RunOptionsAddActiveLoraAdapter(p_, adapter)); + return *this; +} + +inline RunOptions& RunOptions::SetSyncStream(OrtSyncStream* stream) { + GetApi().RunOptionsSetSyncStream(p_, stream); + return *this; +} + +inline ModelCompilationOptions::ModelCompilationOptions(const Env& env, const SessionOptions& session_options) { + ThrowOnError(GetCompileApi().CreateModelCompilationOptionsFromSessionOptions(env, session_options, &this->p_)); +} + +inline ModelCompilationOptions::ModelCompilationOptions(const Env& env, ConstSessionOptions session_options) { + ThrowOnError(GetCompileApi().CreateModelCompilationOptionsFromSessionOptions(env, session_options, &this->p_)); +} + +inline Status CompileModel(const Env& env, const ModelCompilationOptions& model_compilation_options) { + return Ort::Status(GetCompileApi().CompileModel(env, model_compilation_options)); +} + +inline ModelCompilationOptions& ModelCompilationOptions::SetInputModelPath( + const ORTCHAR_T* input_model_path) { + Ort::ThrowOnError(GetCompileApi().ModelCompilationOptions_SetInputModelPath(this->p_, input_model_path)); + return *this; +} + +inline ModelCompilationOptions& ModelCompilationOptions::SetInputModelFromBuffer( + const void* input_model_data, size_t input_model_data_size) { + Ort::ThrowOnError(GetCompileApi().ModelCompilationOptions_SetInputModelFromBuffer(this->p_, input_model_data, + input_model_data_size)); + return *this; +} + +inline ModelCompilationOptions& ModelCompilationOptions::SetOutputModelPath( + const ORTCHAR_T* output_model_path) { + Ort::ThrowOnError(GetCompileApi().ModelCompilationOptions_SetOutputModelPath(this->p_, output_model_path)); + return *this; +} + +inline ModelCompilationOptions& ModelCompilationOptions::SetEpContextBinaryInformation( + const ORTCHAR_T* output_directory, const ORTCHAR_T* model_name) { + Ort::ThrowOnError(GetCompileApi().ModelCompilationOptions_SetEpContextBinaryInformation( + this->p_, + output_directory, + model_name)); + return *this; +} + +inline ModelCompilationOptions& ModelCompilationOptions::SetOutputModelExternalInitializersFile( + const ORTCHAR_T* file_path, size_t initializer_size_threshold) { + Ort::ThrowOnError(GetCompileApi().ModelCompilationOptions_SetOutputModelExternalInitializersFile( + this->p_, + file_path, + initializer_size_threshold)); + return *this; +} + +inline ModelCompilationOptions& +ModelCompilationOptions::SetOutputModelGetInitializerLocationFunc( + OrtGetInitializerLocationFunc get_initializer_location_func, void* state) { + Ort::ThrowOnError(GetCompileApi().ModelCompilationOptions_SetOutputModelGetInitializerLocationFunc( + this->p_, + get_initializer_location_func, + state)); + return *this; +} + +inline ModelCompilationOptions& ModelCompilationOptions::SetOutputModelBuffer( + OrtAllocator* allocator, void** output_model_buffer_ptr, size_t* output_model_buffer_size_ptr) { + Ort::ThrowOnError(GetCompileApi().ModelCompilationOptions_SetOutputModelBuffer(this->p_, allocator, + output_model_buffer_ptr, + output_model_buffer_size_ptr)); + return *this; +} + +inline ModelCompilationOptions& ModelCompilationOptions::SetOutputModelWriteFunc(OrtWriteBufferFunc write_func, + void* state) { + Ort::ThrowOnError(GetCompileApi().ModelCompilationOptions_SetOutputModelWriteFunc(this->p_, write_func, state)); + return *this; +} + +inline ModelCompilationOptions& ModelCompilationOptions::SetEpContextEmbedMode( + bool embed_ep_context_in_model) { + Ort::ThrowOnError(GetCompileApi().ModelCompilationOptions_SetEpContextEmbedMode( + this->p_, + embed_ep_context_in_model)); + return *this; +} + +inline ModelCompilationOptions& ModelCompilationOptions::SetFlags(uint32_t flags) { + Ort::ThrowOnError(GetCompileApi().ModelCompilationOptions_SetFlags(this->p_, flags)); + return *this; +} + +inline ModelCompilationOptions& ModelCompilationOptions::SetGraphOptimizationLevel( + GraphOptimizationLevel graph_optimization_level) { + Ort::ThrowOnError(GetCompileApi().ModelCompilationOptions_SetGraphOptimizationLevel(this->p_, + graph_optimization_level)); + return *this; +} + +namespace detail { + +template +inline Ort::SessionOptions ConstSessionOptionsImpl::Clone() const { + OrtSessionOptions* out; + ThrowOnError(GetApi().CloneSessionOptions(this->p_, &out)); + return SessionOptions{out}; +} + +template +inline std::string ConstSessionOptionsImpl::GetConfigEntry(const char* config_key) const { + size_t size = 0; + // Feed nullptr for the data buffer to query the true size of the string value + Ort::ThrowOnError(GetApi().GetSessionConfigEntry(this->p_, config_key, nullptr, &size)); + + std::string out; + out.resize(size); + Ort::ThrowOnError(GetApi().GetSessionConfigEntry(this->p_, config_key, &out[0], &size)); + out.resize(size - 1); // remove the terminating character '\0' + + return out; +} + +template +inline bool ConstSessionOptionsImpl::HasConfigEntry(const char* config_key) const { + int out = 0; + Ort::ThrowOnError(GetApi().HasSessionConfigEntry(this->p_, config_key, &out)); + return static_cast(out); +} + +template +inline std::string ConstSessionOptionsImpl::GetConfigEntryOrDefault(const char* config_key, + const std::string& def) const { + if (!this->HasConfigEntry(config_key)) { + return def; + } + + return this->GetConfigEntry(config_key); +} + +template +inline SessionOptionsImpl& SessionOptionsImpl::SetIntraOpNumThreads(int intra_op_num_threads) { + ThrowOnError(GetApi().SetIntraOpNumThreads(this->p_, intra_op_num_threads)); + return *this; +} + +template +inline SessionOptionsImpl& SessionOptionsImpl::SetInterOpNumThreads(int inter_op_num_threads) { + ThrowOnError(GetApi().SetInterOpNumThreads(this->p_, inter_op_num_threads)); + return *this; +} + +template +inline SessionOptionsImpl& SessionOptionsImpl::SetGraphOptimizationLevel(GraphOptimizationLevel graph_optimization_level) { + ThrowOnError(GetApi().SetSessionGraphOptimizationLevel(this->p_, graph_optimization_level)); + return *this; +} + +template +inline SessionOptionsImpl& SessionOptionsImpl::SetDeterministicCompute(bool value) { + ThrowOnError(GetApi().SetDeterministicCompute(this->p_, value)); + return *this; +} + +template +inline SessionOptionsImpl& SessionOptionsImpl::SetOptimizedModelFilePath(const ORTCHAR_T* optimized_model_filepath) { + ThrowOnError(GetApi().SetOptimizedModelFilePath(this->p_, optimized_model_filepath)); + return *this; +} + +template +inline SessionOptionsImpl& SessionOptionsImpl::EnableProfiling(const ORTCHAR_T* profile_file_prefix) { + ThrowOnError(GetApi().EnableProfiling(this->p_, profile_file_prefix)); + return *this; +} + +template +inline SessionOptionsImpl& SessionOptionsImpl::DisableProfiling() { + ThrowOnError(GetApi().DisableProfiling(this->p_)); + return *this; +} + +template +inline SessionOptionsImpl& SessionOptionsImpl::EnableOrtCustomOps() { + ThrowOnError(GetApi().EnableOrtCustomOps(this->p_)); + return *this; +} + +template +inline SessionOptionsImpl& SessionOptionsImpl::EnableMemPattern() { + ThrowOnError(GetApi().EnableMemPattern(this->p_)); + return *this; +} + +template +inline SessionOptionsImpl& SessionOptionsImpl::DisableMemPattern() { + ThrowOnError(GetApi().DisableMemPattern(this->p_)); + return *this; +} + +template +inline SessionOptionsImpl& SessionOptionsImpl::EnableCpuMemArena() { + ThrowOnError(GetApi().EnableCpuMemArena(this->p_)); + return *this; +} + +template +inline SessionOptionsImpl& SessionOptionsImpl::DisableCpuMemArena() { + ThrowOnError(GetApi().DisableCpuMemArena(this->p_)); + return *this; +} + +template +inline SessionOptionsImpl& SessionOptionsImpl::SetExecutionMode(ExecutionMode execution_mode) { + ThrowOnError(GetApi().SetSessionExecutionMode(this->p_, execution_mode)); + return *this; +} + +template +inline SessionOptionsImpl& SessionOptionsImpl::SetLoadCancellationFlag(bool value) { + ThrowOnError(GetApi().SessionOptionsSetLoadCancellationFlag(this->p_, value)); + return *this; +} + +template +inline SessionOptionsImpl& SessionOptionsImpl::SetLogId(const char* logid) { + ThrowOnError(GetApi().SetSessionLogId(this->p_, logid)); + return *this; +} + +template +inline SessionOptionsImpl& SessionOptionsImpl::SetLogSeverityLevel(int level) { + ThrowOnError(GetApi().SetSessionLogSeverityLevel(this->p_, level)); + return *this; +} + +template +inline SessionOptionsImpl& SessionOptionsImpl::Add(OrtCustomOpDomain* custom_op_domain) { + ThrowOnError(GetApi().AddCustomOpDomain(this->p_, custom_op_domain)); + return *this; +} + +template +inline SessionOptionsImpl& SessionOptionsImpl::AddConfigEntry(const char* config_key, const char* config_value) { + ThrowOnError(GetApi().AddSessionConfigEntry(this->p_, config_key, config_value)); + return *this; +} + +template +inline SessionOptionsImpl& SessionOptionsImpl::AddInitializer(const char* name, const OrtValue* ort_val) { + ThrowOnError(GetApi().AddInitializer(this->p_, name, ort_val)); + return *this; +} + +template +inline SessionOptionsImpl& SessionOptionsImpl::DisablePerSessionThreads() { + ThrowOnError(GetApi().DisablePerSessionThreads(this->p_)); + return *this; +} + +template +inline SessionOptionsImpl& SessionOptionsImpl::AddExternalInitializers(const std::vector& names, + const std::vector& ort_values) { + const size_t inputs_num = names.size(); + if (inputs_num != ort_values.size()) { + ORT_CXX_API_THROW("Expecting names and ort_values to have the same length", ORT_INVALID_ARGUMENT); + } + std::vector names_ptr; + std::vector ort_values_ptrs; + names_ptr.reserve(inputs_num); + ort_values_ptrs.reserve(inputs_num); + for (size_t i = 0; i < inputs_num; ++i) { + names_ptr.push_back(names[i].c_str()); + ort_values_ptrs.push_back(ort_values[i]); + } + ThrowOnError(GetApi().AddExternalInitializers(this->p_, names_ptr.data(), ort_values_ptrs.data(), inputs_num)); + return *this; +} + +template +inline SessionOptionsImpl& SessionOptionsImpl::AddExternalInitializersFromFilesInMemory(const std::vector>& file_names, + const std::vector& buffer_array, + const std::vector& file_lengths) { + const size_t inputs_num = file_names.size(); + if (inputs_num != buffer_array.size()) { + ORT_CXX_API_THROW("Expecting names and buffer_array to have the same length", ORT_INVALID_ARGUMENT); + } + if (inputs_num != file_lengths.size()) { + ORT_CXX_API_THROW("Expecting names and file_lengths to have the same length", ORT_INVALID_ARGUMENT); + } + std::vector names_ptr; + names_ptr.reserve(inputs_num); + for (size_t i = 0; i < inputs_num; ++i) { + names_ptr.push_back(file_names[i].c_str()); + } + ThrowOnError(GetApi().AddExternalInitializersFromFilesInMemory(this->p_, names_ptr.data(), buffer_array.data(), + file_lengths.data(), inputs_num)); + return *this; +} + +template +inline SessionOptionsImpl& SessionOptionsImpl::AppendExecutionProvider_CPU(int use_arena) { + ThrowOnError(OrtSessionOptionsAppendExecutionProvider_CPU(this->p_, use_arena)); + return *this; +} + +template +inline SessionOptionsImpl& SessionOptionsImpl::AppendExecutionProvider_CUDA(const OrtCUDAProviderOptions& provider_options) { + ThrowOnError(GetApi().SessionOptionsAppendExecutionProvider_CUDA(this->p_, &provider_options)); + return *this; +} + +template +inline SessionOptionsImpl& SessionOptionsImpl::AppendExecutionProvider_CUDA_V2(const OrtCUDAProviderOptionsV2& provider_options) { + ThrowOnError(GetApi().SessionOptionsAppendExecutionProvider_CUDA_V2(this->p_, &provider_options)); + return *this; +} + +template +inline SessionOptionsImpl& SessionOptionsImpl::AppendExecutionProvider_ROCM(const OrtROCMProviderOptions& provider_options) { + ThrowOnError(GetApi().SessionOptionsAppendExecutionProvider_ROCM(this->p_, &provider_options)); + return *this; +} + +template +inline SessionOptionsImpl& SessionOptionsImpl::AppendExecutionProvider_TensorRT(const OrtTensorRTProviderOptions& provider_options) { + ThrowOnError(GetApi().SessionOptionsAppendExecutionProvider_TensorRT(this->p_, &provider_options)); + return *this; +} + +template +inline SessionOptionsImpl& SessionOptionsImpl::AppendExecutionProvider_TensorRT_V2(const OrtTensorRTProviderOptionsV2& provider_options) { + ThrowOnError(GetApi().SessionOptionsAppendExecutionProvider_TensorRT_V2(this->p_, &provider_options)); + return *this; +} + +template +inline SessionOptionsImpl& SessionOptionsImpl::AppendExecutionProvider_MIGraphX(const OrtMIGraphXProviderOptions& provider_options) { + ThrowOnError(GetApi().SessionOptionsAppendExecutionProvider_MIGraphX(this->p_, &provider_options)); + return *this; +} + +template +inline SessionOptionsImpl& SessionOptionsImpl::AppendExecutionProvider_CANN(const OrtCANNProviderOptions& provider_options) { + ThrowOnError(GetApi().SessionOptionsAppendExecutionProvider_CANN(this->p_, &provider_options)); + return *this; +} + +template +inline SessionOptionsImpl& SessionOptionsImpl::AppendExecutionProvider_Dnnl(const OrtDnnlProviderOptions& provider_options) { + ThrowOnError(GetApi().SessionOptionsAppendExecutionProvider_Dnnl(this->p_, &provider_options)); + return *this; +} + +template +inline SessionOptionsImpl& SessionOptionsImpl::AppendExecutionProvider( + const std::string& provider_name, + const std::unordered_map& provider_options) { + auto num_entries = provider_options.size(); + std::vector keys, values; + if (num_entries > 0) { + keys.reserve(num_entries); + values.reserve(num_entries); + + for (const auto& entry : provider_options) { + keys.push_back(entry.first.c_str()); + values.push_back(entry.second.c_str()); + } + } + + ThrowOnError(GetApi().SessionOptionsAppendExecutionProvider(this->p_, provider_name.c_str(), + keys.data(), values.data(), num_entries)); + + return *this; +} + +namespace { +template +void SessionOptionsAppendEP(detail::SessionOptionsImpl& session_options, + Env& env, const std::vector& ep_devices, + const std::vector& ep_options_keys, + const std::vector& ep_options_values) { + std::vector ep_devices_ptrs; + ep_devices_ptrs.reserve(ep_devices.size()); + for (const auto& ep_device : ep_devices) { + ep_devices_ptrs.push_back(ep_device); + } + + ThrowOnError(GetApi().SessionOptionsAppendExecutionProvider_V2( + session_options, env, ep_devices_ptrs.data(), ep_devices_ptrs.size(), + ep_options_keys.data(), ep_options_values.data(), ep_options_keys.size())); +} +} // namespace + +template +inline SessionOptionsImpl& SessionOptionsImpl::AppendExecutionProvider_V2( + Env& env, const std::vector& ep_devices, const KeyValuePairs& ep_options) { + std::vector ep_options_keys, ep_options_values; + ep_options.GetKeyValuePairs(ep_options_keys, ep_options_values); + + SessionOptionsAppendEP(*this, env, ep_devices, ep_options_keys, ep_options_values); + + return *this; +} + +template +inline SessionOptionsImpl& SessionOptionsImpl::AppendExecutionProvider_V2( + Env& env, const std::vector& ep_devices, + const std::unordered_map& ep_options) { + std::vector ep_options_keys, ep_options_values; + ep_options_keys.reserve(ep_options.size()); + ep_options_values.reserve(ep_options.size()); + + for (const auto& [key, value] : ep_options) { + ep_options_keys.push_back(key.c_str()); + ep_options_values.push_back(value.c_str()); + } + + SessionOptionsAppendEP(*this, env, ep_devices, ep_options_keys, ep_options_values); + + return *this; +} + +template +inline SessionOptionsImpl& SessionOptionsImpl::SetEpSelectionPolicy(OrtExecutionProviderDevicePolicy policy) { + ThrowOnError(GetApi().SessionOptionsSetEpSelectionPolicy(this->p_, policy)); + return *this; +} + +template +inline SessionOptionsImpl& SessionOptionsImpl::SetEpSelectionPolicy(EpSelectionDelegate delegate, void* state) { + ThrowOnError(GetApi().SessionOptionsSetEpSelectionPolicyDelegate(this->p_, delegate, state)); + return *this; +} + +template +inline SessionOptionsImpl& SessionOptionsImpl::SetCustomCreateThreadFn(OrtCustomCreateThreadFn ort_custom_create_thread_fn) { + ThrowOnError(GetApi().SessionOptionsSetCustomCreateThreadFn(this->p_, ort_custom_create_thread_fn)); + return *this; +} + +template +inline SessionOptionsImpl& SessionOptionsImpl::SetCustomThreadCreationOptions(void* ort_custom_thread_creation_options) { + ThrowOnError(GetApi().SessionOptionsSetCustomThreadCreationOptions(this->p_, ort_custom_thread_creation_options)); + return *this; +} + +template +inline SessionOptionsImpl& SessionOptionsImpl::SetCustomJoinThreadFn(OrtCustomJoinThreadFn ort_custom_join_thread_fn) { + ThrowOnError(GetApi().SessionOptionsSetCustomJoinThreadFn(this->p_, ort_custom_join_thread_fn)); + return *this; +} + +template +inline SessionOptionsImpl& SessionOptionsImpl::AppendExecutionProvider_OpenVINO(const OrtOpenVINOProviderOptions& provider_options) { + ThrowOnError(GetApi().SessionOptionsAppendExecutionProvider_OpenVINO(this->p_, &provider_options)); + return *this; +} + +template +inline SessionOptionsImpl& SessionOptionsImpl::AppendExecutionProvider_OpenVINO_V2(const std::unordered_map& provider_options) { + auto num_entries = provider_options.size(); + std::vector keys, values; + if (num_entries > 0) { + keys.reserve(num_entries); + values.reserve(num_entries); + + for (const auto& entry : provider_options) { + keys.push_back(entry.first.c_str()); + values.push_back(entry.second.c_str()); + } + } + + ThrowOnError(GetApi().SessionOptionsAppendExecutionProvider_OpenVINO_V2(this->p_, + keys.data(), values.data(), num_entries)); + + return *this; +} + +template +inline SessionOptionsImpl& SessionOptionsImpl::AppendExecutionProvider_VitisAI(const std::unordered_map& provider_options) { + auto num_entries = provider_options.size(); + std::vector keys, values; + if (num_entries > 0) { + keys.reserve(num_entries); + values.reserve(num_entries); + + for (const auto& entry : provider_options) { + keys.push_back(entry.first.c_str()); + values.push_back(entry.second.c_str()); + } + } + + ThrowOnError(GetApi().SessionOptionsAppendExecutionProvider_VitisAI(this->p_, keys.data(), values.data(), num_entries)); + + return *this; +} + +template +inline SessionOptionsImpl& SessionOptionsImpl::RegisterCustomOpsLibrary(const ORTCHAR_T* library_name, + const CustomOpConfigs& custom_op_configs) { + // Add custom op config entries before registering the custom op library. Otherwise, the config entries _may_ be ignored by + // the custom op library. + for (const auto& config_iter : custom_op_configs.GetFlattenedConfigs()) { + AddConfigEntry(config_iter.first.c_str(), config_iter.second.c_str()); + } + + ThrowOnError(GetApi().RegisterCustomOpsLibrary_V2(this->p_, library_name)); + return *this; +} + +template +inline SessionOptionsImpl& SessionOptionsImpl::RegisterCustomOpsUsingFunction(const char* registration_function_name) { + ThrowOnError(GetApi().RegisterCustomOpsUsingFunction(this->p_, registration_function_name)); + return *this; +} + +template +inline SessionOptionsImpl& SessionOptionsImpl::AddFreeDimensionOverride(const char* dim_denotation, int64_t dim_value) { + ThrowOnError(GetApi().AddFreeDimensionOverrideByName(this->p_, dim_denotation, dim_value)); + return *this; +} + +template +inline SessionOptionsImpl& SessionOptionsImpl::AddFreeDimensionOverrideByName(const char* dim_name, int64_t dim_value) { + ThrowOnError(GetApi().AddFreeDimensionOverrideByName(this->p_, dim_name, dim_value)); + return *this; +} + +/// Session +template +inline size_t ConstSessionImpl::GetInputCount() const { + size_t out; + ThrowOnError(GetApi().SessionGetInputCount(this->p_, &out)); + return out; +} + +template +inline size_t ConstSessionImpl::GetOutputCount() const { + size_t out; + ThrowOnError(GetApi().SessionGetOutputCount(this->p_, &out)); + return out; +} + +template +inline size_t ConstSessionImpl::GetOverridableInitializerCount() const { + size_t out; + ThrowOnError(GetApi().SessionGetOverridableInitializerCount(this->p_, &out)); + return out; +} + +template +inline std::vector ConstSessionImpl::GetInputNames() const { + AllocatorWithDefaultOptions allocator; + + auto num_inputs = GetInputCount(); + std::vector input_names; + input_names.reserve(num_inputs); + + for (size_t i = 0; i < num_inputs; ++i) { + char* name; + ThrowOnError(GetApi().SessionGetInputName(this->p_, i, allocator, &name)); + input_names.emplace_back(name); + allocator.Free(name); + } + + return input_names; +} + +template +inline std::vector ConstSessionImpl::GetOutputNames() const { + AllocatorWithDefaultOptions allocator; + + auto num_inputs = GetOutputCount(); + std::vector output_names; + output_names.reserve(num_inputs); + + for (size_t i = 0; i < num_inputs; ++i) { + char* name; + ThrowOnError(GetApi().SessionGetOutputName(this->p_, i, allocator, &name)); + output_names.emplace_back(name); + allocator.Free(name); + } + + return output_names; +} + +template +inline std::vector ConstSessionImpl::GetOverridableInitializerNames() const { + AllocatorWithDefaultOptions allocator; + + auto num_initializers = GetOverridableInitializerCount(); + std::vector initializer_names; + initializer_names.reserve(num_initializers); + + for (size_t i = 0; i < num_initializers; ++i) { + char* name; + ThrowOnError(GetApi().SessionGetOverridableInitializerName(this->p_, i, allocator, &name)); + initializer_names.emplace_back(name); + } + + return initializer_names; +} + +template +inline std::vector ConstSessionImpl::GetMemoryInfoForInputs() const { + static_assert(sizeof(ConstMemoryInfo) == sizeof(OrtMemoryInfo*), + "ConstMemoryInfo must be compatible with OrtMemoryInfo*"); + + auto num_inputs = GetInputCount(); + std::vector mem_infos; + if (num_inputs > 0) { + mem_infos.resize(num_inputs); + + ThrowOnError(GetApi().SessionGetMemoryInfoForInputs(this->p_, + reinterpret_cast(mem_infos.data()), + num_inputs)); + } + + return mem_infos; +} + +template +inline std::vector ConstSessionImpl::GetMemoryInfoForOutputs() const { + static_assert(sizeof(ConstMemoryInfo) == sizeof(OrtMemoryInfo*), + "ConstMemoryInfo must be compatible with OrtMemoryInfo*"); + + auto num_outputs = GetOutputCount(); + std::vector mem_infos; + if (num_outputs > 0) { + mem_infos.resize(num_outputs); + + ThrowOnError(GetApi().SessionGetMemoryInfoForOutputs(this->p_, + reinterpret_cast(mem_infos.data()), + num_outputs)); + } + return mem_infos; +} + +template +inline AllocatedStringPtr ConstSessionImpl::GetInputNameAllocated(size_t index, OrtAllocator* allocator) const { + char* out; + ThrowOnError(GetApi().SessionGetInputName(this->p_, index, allocator, &out)); + return AllocatedStringPtr(out, detail::AllocatedFree(allocator)); +} + +template +inline AllocatedStringPtr ConstSessionImpl::GetOutputNameAllocated(size_t index, OrtAllocator* allocator) const { + char* out; + ThrowOnError(GetApi().SessionGetOutputName(this->p_, index, allocator, &out)); + return AllocatedStringPtr(out, detail::AllocatedFree(allocator)); +} + +template +inline AllocatedStringPtr ConstSessionImpl::GetOverridableInitializerNameAllocated(size_t index, OrtAllocator* allocator) const { + char* out; + ThrowOnError(GetApi().SessionGetOverridableInitializerName(this->p_, index, allocator, &out)); + return AllocatedStringPtr(out, detail::AllocatedFree(allocator)); +} + +template +inline std::vector ConstSessionImpl::GetEpDeviceForInputs() const { + auto num_inputs = GetInputCount(); + std::vector input_devices; + if (num_inputs > 0) { + input_devices.resize(num_inputs); + ThrowOnError(GetApi().SessionGetEpDeviceForInputs(this->p_, + reinterpret_cast(input_devices.data()), + num_inputs)); + } + return input_devices; +} + +template +inline std::vector ConstSessionImpl::GetEpDeviceForOutputs() const { + auto num_outputs = GetOutputCount(); + std::vector output_devices; + if (num_outputs > 0) { + output_devices.resize(num_outputs); + ThrowOnError(GetApi().SessionGetEpDeviceForOutputs(this->p_, + reinterpret_cast(output_devices.data()), + num_outputs)); + } + return output_devices; +} + +template +inline uint64_t ConstSessionImpl::GetProfilingStartTimeNs() const { + uint64_t out; + ThrowOnError(GetApi().SessionGetProfilingStartTimeNs(this->p_, &out)); + return out; +} + +template +inline ModelMetadata ConstSessionImpl::GetModelMetadata() const { + OrtModelMetadata* out; + ThrowOnError(GetApi().SessionGetModelMetadata(this->p_, &out)); + return ModelMetadata{out}; +} + +template +inline TypeInfo ConstSessionImpl::GetInputTypeInfo(size_t index) const { + OrtTypeInfo* out; + ThrowOnError(GetApi().SessionGetInputTypeInfo(this->p_, index, &out)); + return TypeInfo{out}; +} + +template +inline TypeInfo ConstSessionImpl::GetOutputTypeInfo(size_t index) const { + OrtTypeInfo* out; + ThrowOnError(GetApi().SessionGetOutputTypeInfo(this->p_, index, &out)); + return TypeInfo{out}; +} + +template +inline TypeInfo ConstSessionImpl::GetOverridableInitializerTypeInfo(size_t index) const { + OrtTypeInfo* out; + ThrowOnError(GetApi().SessionGetOverridableInitializerTypeInfo(this->p_, index, &out)); + return TypeInfo{out}; +} + +#if !defined(ORT_MINIMAL_BUILD) +template +inline int ConstSessionImpl::GetOpset(const std::string& domain) const { + int opset; + ThrowOnError(GetModelEditorApi().SessionGetOpsetForDomain(this->p_, domain.c_str(), &opset)); + return opset; +} +#endif // !defined(ORT_MINIMAL_BUILD) + +template +std::vector ConstSessionImpl::GetInputs() const { + const std::vector input_names = GetInputNames(); + + std::vector inputs; + inputs.reserve(input_names.size()); + + for (size_t i = 0; i < input_names.size(); ++i) { + auto type_info = GetInputTypeInfo(i); + inputs.emplace_back(ValueInfo{input_names[i], type_info.GetConst()}); + } + + return inputs; +} + +template +std::vector ConstSessionImpl::GetOutputs() const { + const std::vector output_names = GetOutputNames(); + + std::vector outputs; + outputs.reserve(output_names.size()); + + for (size_t i = 0; i < output_names.size(); ++i) { + auto type_info = GetOutputTypeInfo(i); + outputs.emplace_back(ValueInfo{output_names[i], type_info.GetConst()}); + } + + return outputs; +} + +template +inline std::vector ConstSessionImpl::GetEpGraphAssignmentInfo() const { + size_t num_ep_subgraphs = 0; + const OrtEpAssignedSubgraph* const* ep_subgraph_ptrs = nullptr; + ThrowOnError(GetApi().Session_GetEpGraphAssignmentInfo(this->p_, &ep_subgraph_ptrs, &num_ep_subgraphs)); + + std::vector ep_subgraphs; + if (num_ep_subgraphs > 0) { + ep_subgraphs.reserve(num_ep_subgraphs); + for (size_t i = 0; i < num_ep_subgraphs; ++i) { + ep_subgraphs.emplace_back(ep_subgraph_ptrs[i]); + } + } + + return ep_subgraphs; +} + +template +inline std::vector SessionImpl::Run(const RunOptions& run_options, const char* const* input_names, const Value* input_values, size_t input_count, + const char* const* output_names, size_t output_count) { + std::vector output_values; + output_values.reserve(output_count); + for (size_t i = 0; i < output_count; i++) + output_values.emplace_back(nullptr); + Run(run_options, input_names, input_values, input_count, output_names, output_values.data(), output_count); + return output_values; +} + +template +inline void SessionImpl::Run(const RunOptions& run_options, const char* const* input_names, const Value* input_values, size_t input_count, + const char* const* output_names, Value* output_values, size_t output_count) { + static_assert(sizeof(Value) == sizeof(OrtValue*), "Value is really just an array of OrtValue* in memory, so we can reinterpret_cast safely"); + auto ort_input_values = reinterpret_cast(input_values); + auto ort_output_values = reinterpret_cast(output_values); + ThrowOnError(GetApi().Run(this->p_, run_options, input_names, ort_input_values, input_count, output_names, output_count, ort_output_values)); +} + +template +inline void SessionImpl::Run(const RunOptions& run_options, const IoBinding& io_binding) { + ThrowOnError(GetApi().RunWithBinding(this->p_, run_options, io_binding)); +} + +template +inline void SessionImpl::RunAsync(const RunOptions& run_options, const char* const* input_names, const Value* input_values, size_t input_count, + const char* const* output_names, Value* output_values, size_t output_count, RunAsyncCallbackFn callback, void* user_data) { + auto ort_input_values = reinterpret_cast(input_values); + auto ort_output_values = reinterpret_cast(output_values); + ThrowOnError(GetApi().RunAsync(this->p_, run_options, input_names, + ort_input_values, input_count, output_names, output_count, + ort_output_values, callback, user_data)); +} + +template +inline AllocatedStringPtr SessionImpl::EndProfilingAllocated(OrtAllocator* allocator) { + char* out = nullptr; + ThrowOnError(GetApi().SessionEndProfiling(this->p_, allocator, &out)); + return AllocatedStringPtr(out, detail::AllocatedFree(allocator)); +} + +template +inline void SessionImpl::SetEpDynamicOptions(const char* const* keys, const char* const* values, size_t kv_len) { + ThrowOnError(GetApi().SetEpDynamicOptions(this->p_, keys, values, kv_len)); +} + +#if !defined(ORT_MINIMAL_BUILD) +template +inline void SessionImpl::FinalizeModelEditorSession(const Model& model, const SessionOptions& options, + OrtPrepackedWeightsContainer* prepacked_weights_container) { + ThrowOnError(GetModelEditorApi().ApplyModelToModelEditorSession(this->p_, model)); + ThrowOnError(GetModelEditorApi().FinalizeModelEditorSession(this->p_, options, prepacked_weights_container)); +} +#endif // #if !defined(ORT_MINIMAL_BUILD) + +} // namespace detail + +inline SessionOptions::SessionOptions() { + ThrowOnError(GetApi().CreateSessionOptions(&this->p_)); +} + +/// CustomOpConfigs +inline std::string detail::MakeCustomOpConfigEntryKey(const char* custom_op_name, const char* config) { + std::string config_key = "custom_op."; + + config_key += custom_op_name; + config_key += "."; + config_key += config; + + return config_key; +} + +inline CustomOpConfigs& CustomOpConfigs::AddConfig(const char* custom_op_name, const char* config_key, const char* config_value) { + const std::string full_flat_key = detail::MakeCustomOpConfigEntryKey(custom_op_name, config_key); + flat_configs_[full_flat_key] = config_value; + return *this; +} + +inline const std::unordered_map& CustomOpConfigs::GetFlattenedConfigs() const { + return flat_configs_; +} + +inline Session::Session(const Env& env, const ORTCHAR_T* model_path, const SessionOptions& options) { + ThrowOnError(GetApi().CreateSession(env, model_path, options, &this->p_)); +} + +inline Session::Session(const Env& env, const ORTCHAR_T* model_path, const SessionOptions& options, + OrtPrepackedWeightsContainer* prepacked_weights_container) { + ThrowOnError(GetApi().CreateSessionWithPrepackedWeightsContainer(env, model_path, options, prepacked_weights_container, &this->p_)); +} + +inline Session::Session(const Env& env, const void* model_data, size_t model_data_length, const SessionOptions& options) { + ThrowOnError(GetApi().CreateSessionFromArray(env, model_data, model_data_length, options, &this->p_)); +} + +inline Session::Session(const Env& env, const void* model_data, size_t model_data_length, + const SessionOptions& options, OrtPrepackedWeightsContainer* prepacked_weights_container) { + ThrowOnError(GetApi().CreateSessionFromArrayWithPrepackedWeightsContainer(env, model_data, model_data_length, options, + prepacked_weights_container, &this->p_)); +} + +#if !defined(ORT_MINIMAL_BUILD) +inline Session::Session(const Env& env, const Model& model, const SessionOptions& options) { + ThrowOnError(GetModelEditorApi().CreateSessionFromModel(env, model, options, &this->p_)); +} + +// static +inline Session Session::CreateModelEditorSession(const Env& env, const ORTCHAR_T* model_path, + const SessionOptions& options) { + OrtSession* session = nullptr; + ThrowOnError(GetModelEditorApi().CreateModelEditorSession(env, model_path, options, &session)); + return Session(session); +} + +// static +inline Session Session::CreateModelEditorSession(const Env& env, const void* model_data, size_t model_data_length, + const SessionOptions& options) { + OrtSession* session = nullptr; + ThrowOnError(GetModelEditorApi().CreateModelEditorSessionFromArray(env, model_data, model_data_length, options, + &session)); + return Session(session); +} + +void FinalizeModelEditorSession(const Model& model, const SessionOptions& options, + OrtPrepackedWeightsContainer* prepacked_weights_container); +#endif // #if !defined(ORT_MINIMAL_BUILD) + +inline AllocatedStringPtr ModelMetadata::GetProducerNameAllocated(OrtAllocator* allocator) const { + char* out; + ThrowOnError(GetApi().ModelMetadataGetProducerName(p_, allocator, &out)); + return AllocatedStringPtr(out, detail::AllocatedFree(allocator)); +} + +inline AllocatedStringPtr ModelMetadata::GetGraphNameAllocated(OrtAllocator* allocator) const { + char* out; + ThrowOnError(GetApi().ModelMetadataGetGraphName(p_, allocator, &out)); + return AllocatedStringPtr(out, detail::AllocatedFree(allocator)); +} + +inline AllocatedStringPtr ModelMetadata::GetDomainAllocated(OrtAllocator* allocator) const { + char* out; + ThrowOnError(GetApi().ModelMetadataGetDomain(p_, allocator, &out)); + return AllocatedStringPtr(out, detail::AllocatedFree(allocator)); +} + +inline AllocatedStringPtr Ort::ModelMetadata::GetDescriptionAllocated(OrtAllocator* allocator) const { + char* out; + ThrowOnError(GetApi().ModelMetadataGetDescription(p_, allocator, &out)); + return AllocatedStringPtr(out, detail::AllocatedFree(allocator)); +} + +inline AllocatedStringPtr ModelMetadata::GetGraphDescriptionAllocated(OrtAllocator* allocator) const { + char* out; + ThrowOnError(GetApi().ModelMetadataGetGraphDescription(p_, allocator, &out)); + return AllocatedStringPtr(out, detail::AllocatedFree(allocator)); +} + +inline AllocatedStringPtr ModelMetadata::LookupCustomMetadataMapAllocated(const char* key, OrtAllocator* allocator) const { + char* out; + ThrowOnError(GetApi().ModelMetadataLookupCustomMetadataMap(p_, allocator, key, &out)); + return AllocatedStringPtr(out, detail::AllocatedFree(allocator)); +} + +inline std::vector ModelMetadata::GetCustomMetadataMapKeysAllocated(OrtAllocator* allocator) const { + auto deletor = detail::AllocatedFree(allocator); + std::vector result; + + char** out = nullptr; + int64_t num_keys = 0; + ThrowOnError(GetApi().ModelMetadataGetCustomMetadataMapKeys(p_, allocator, &out, &num_keys)); + if (num_keys <= 0) { + return result; + } + + // array of pointers will be freed + std::unique_ptr array_guard(out, deletor); + // reserve may throw + auto strings_deletor = [&deletor, num_keys](char** out) { for(int64_t i = 0; i < num_keys; ++i) deletor(out[i]); }; + std::unique_ptr strings_guard(out, strings_deletor); + result.reserve(static_cast(num_keys)); + strings_guard.release(); + for (int64_t i = 0; i < num_keys; ++i) { + result.push_back(AllocatedStringPtr(out[i], deletor)); + } + + return result; +} + +inline int64_t ModelMetadata::GetVersion() const { + int64_t out; + ThrowOnError(GetApi().ModelMetadataGetVersion(p_, &out)); + return out; +} + +inline TensorTypeAndShapeInfo::TensorTypeAndShapeInfo(ONNXTensorElementDataType element_type, + const std::vector& dims, + const std::vector* symbolic_dims) { + ThrowOnError(GetApi().CreateTensorTypeAndShapeInfo(&p_)); + ThrowOnError(GetApi().SetTensorElementType(p_, element_type)); + ThrowOnError(GetApi().SetDimensions(p_, dims.data(), dims.size())); + + if (symbolic_dims) { + std::vector symbolic_dims_cstr; + symbolic_dims_cstr.reserve(symbolic_dims->size()); + std::transform(symbolic_dims->begin(), symbolic_dims->end(), std::back_inserter(symbolic_dims_cstr), + [](const std::string& s) { return s.c_str(); }); + ThrowOnError(GetApi().SetSymbolicDimensions(p_, symbolic_dims_cstr.data(), symbolic_dims_cstr.size())); + } +} + +#if !defined(ORT_MINIMAL_BUILD) +// static +inline TypeInfo TypeInfo::CreateTensorInfo(ConstTensorTypeAndShapeInfo tensor_type_and_shape_info) { + OrtTypeInfo* output = nullptr; + ThrowOnError(GetModelEditorApi().CreateTensorTypeInfo(tensor_type_and_shape_info, &output)); + return TypeInfo{output}; +} + +// static +inline TypeInfo TypeInfo::CreateSparseTensorInfo(ConstTensorTypeAndShapeInfo sparse_tensor_type_and_shape_info) { + OrtTypeInfo* output = nullptr; + ThrowOnError(GetModelEditorApi().CreateSparseTensorTypeInfo(sparse_tensor_type_and_shape_info, &output)); + return TypeInfo{output}; +} + +// static +inline TypeInfo TypeInfo::CreateSequenceTypeInfo(ConstTypeInfo sequence_type) { + OrtTypeInfo* output; + ThrowOnError(GetModelEditorApi().CreateSequenceTypeInfo(sequence_type, &output)); + return TypeInfo{output}; +} + +// static +inline TypeInfo TypeInfo::CreateMapTypeInfo(ONNXTensorElementDataType key_type, ConstTypeInfo value_type) { + OrtTypeInfo* output; + ThrowOnError(GetModelEditorApi().CreateMapTypeInfo(key_type, value_type, &output)); + return TypeInfo{output}; +} + +// static +inline TypeInfo TypeInfo::CreateOptionalTypeInfo(ConstTypeInfo contained_type) { + OrtTypeInfo* output; + ThrowOnError(GetModelEditorApi().CreateOptionalTypeInfo(contained_type, &output)); + return TypeInfo{output}; +} +#endif // #if !defined(ORT_MINIMAL_BUILD) + +namespace detail { + +template +inline ONNXTensorElementDataType TensorTypeAndShapeInfoImpl::GetElementType() const { + ONNXTensorElementDataType out; + ThrowOnError(GetApi().GetTensorElementType(this->p_, &out)); + return out; +} + +template +inline size_t TensorTypeAndShapeInfoImpl::GetElementCount() const { + size_t out; + ThrowOnError(GetApi().GetTensorShapeElementCount(this->p_, &out)); + return out; +} + +template +inline bool TensorTypeAndShapeInfoImpl::HasShape() const { + return GetApi().TensorTypeAndShape_HasShape(this->p_); +} + +template +inline size_t TensorTypeAndShapeInfoImpl::GetDimensionsCount() const { + size_t out; + ThrowOnError(GetApi().GetDimensionsCount(this->p_, &out)); + return out; +} + +template +inline void TensorTypeAndShapeInfoImpl::GetDimensions(int64_t* values, size_t values_count) const { + ThrowOnError(GetApi().GetDimensions(this->p_, values, values_count)); +} + +template +inline void TensorTypeAndShapeInfoImpl::GetSymbolicDimensions(const char** values, size_t values_count) const { + ThrowOnError(GetApi().GetSymbolicDimensions(this->p_, values, values_count)); +} + +template +inline std::vector TensorTypeAndShapeInfoImpl::GetSymbolicDimensions() const { + std::vector out; + size_t dim_count = GetDimensionsCount(); + if (dim_count > 0) { + out.resize(dim_count, nullptr); + ThrowOnError(GetApi().GetSymbolicDimensions(this->p_, out.data(), out.size())); + } + return out; +} + +template +inline std::vector TensorTypeAndShapeInfoImpl::GetShape() const { + std::vector out(GetDimensionsCount(), -1); + ThrowOnError(GetApi().GetDimensions(this->p_, out.data(), out.size())); + return out; +} + +template +inline ConstTensorTypeAndShapeInfo TypeInfoImpl::GetTensorTypeAndShapeInfo() const { + const OrtTensorTypeAndShapeInfo* out; + ThrowOnError(GetApi().CastTypeInfoToTensorInfo(this->p_, &out)); + return ConstTensorTypeAndShapeInfo{out}; +} + +template +inline ConstSequenceTypeInfo TypeInfoImpl::GetSequenceTypeInfo() const { + const OrtSequenceTypeInfo* out; + ThrowOnError(GetApi().CastTypeInfoToSequenceTypeInfo(this->p_, &out)); + return ConstSequenceTypeInfo{out}; +} + +template +inline ConstMapTypeInfo TypeInfoImpl::GetMapTypeInfo() const { + const OrtMapTypeInfo* out; + ThrowOnError(GetApi().CastTypeInfoToMapTypeInfo(this->p_, &out)); + return ConstMapTypeInfo{out}; +} + +template +inline ONNXType TypeInfoImpl::GetONNXType() const { + ONNXType out; + ThrowOnError(GetApi().GetOnnxTypeFromTypeInfo(this->p_, &out)); + return out; +} + +template +inline TypeInfo SequenceTypeInfoImpl::GetSequenceElementType() const { + OrtTypeInfo* output; + ThrowOnError(GetApi().GetSequenceElementType(this->p_, &output)); + return TypeInfo{output}; +} + +template +inline TypeInfo OptionalTypeInfoImpl::GetOptionalElementType() const { + OrtTypeInfo* info; + ThrowOnError(GetApi().GetOptionalContainedTypeInfo(this->p_, &info)); + return TypeInfo{info}; +} + +template +inline ONNXTensorElementDataType MapTypeInfoImpl::GetMapKeyType() const { + ONNXTensorElementDataType out; + ThrowOnError(GetApi().GetMapKeyType(this->p_, &out)); + return out; +} + +template +inline TypeInfo MapTypeInfoImpl::GetMapValueType() const { + OrtTypeInfo* output; + ThrowOnError(GetApi().GetMapValueType(this->p_, &output)); + return TypeInfo{output}; +} + +template +inline ConstOptionalTypeInfo TypeInfoImpl::GetOptionalTypeInfo() const { + const OrtOptionalTypeInfo* info; + ThrowOnError(GetApi().CastTypeInfoToOptionalTypeInfo(this->p_, &info)); + return ConstOptionalTypeInfo{info}; +} + +} // namespace detail + +namespace detail { + +template +template +inline void ConstValueImpl::GetOpaqueData(const char* domain, const char* type_name, R& out) const { + ThrowOnError(GetApi().GetOpaqueValue(domain, type_name, this->p_, &out, sizeof(R))); +} + +template +inline bool ConstValueImpl::IsTensor() const { + int out; + ThrowOnError(GetApi().IsTensor(this->p_, &out)); + return out != 0; +} + +template +inline bool ConstValueImpl::HasValue() const { + int out; + ThrowOnError(GetApi().HasValue(this->p_, &out)); + return out != 0; +} + +template +inline size_t ConstValueImpl::GetCount() const { + size_t out; + ThrowOnError(GetApi().GetValueCount(this->p_, &out)); + return out; +} + +template +inline Value ConstValueImpl::GetValue(int index, OrtAllocator* allocator) const { + OrtValue* out; + ThrowOnError(GetApi().GetValue(this->p_, index, allocator, &out)); + return Value{out}; +} + +template +inline size_t ConstValueImpl::GetStringTensorDataLength() const { + size_t out; + ThrowOnError(GetApi().GetStringTensorDataLength(this->p_, &out)); + return out; +} + +template +inline size_t ConstValueImpl::GetStringTensorElementLength(size_t element_index) const { + size_t out; + ThrowOnError(GetApi().GetStringTensorElementLength(this->p_, element_index, &out)); + return out; +} + +template +inline size_t ConstValueImpl::GetTensorSizeInBytes() const { + size_t out; + ThrowOnError(GetApi().GetTensorSizeInBytes(this->p_, &out)); + return out; +} + +template +template +inline const R* ConstValueImpl::GetTensorData() const { + const R* out; + ThrowOnError(GetApi().GetTensorData(this->p_, reinterpret_cast(&out))); + return out; +} + +template +inline const void* ConstValueImpl::GetTensorRawData() const { + const void* out; + ThrowOnError(GetApi().GetTensorData(this->p_, &out)); + return out; +} + +template +inline TypeInfo ConstValueImpl::GetTypeInfo() const { + OrtTypeInfo* output; + ThrowOnError(GetApi().GetTypeInfo(this->p_, &output)); + return TypeInfo{output}; +} + +template +inline TensorTypeAndShapeInfo ConstValueImpl::GetTensorTypeAndShapeInfo() const { + OrtTensorTypeAndShapeInfo* output; + ThrowOnError(GetApi().GetTensorTypeAndShape(this->p_, &output)); + return TensorTypeAndShapeInfo{output}; +} + +template +inline ConstMemoryInfo ConstValueImpl::GetTensorMemoryInfo() const { + const OrtMemoryInfo* mem_info; + ThrowOnError(GetApi().GetTensorMemoryInfo(this->p_, &mem_info)); + return ConstMemoryInfo(mem_info); +} + +template +inline void ConstValueImpl::GetStringTensorElement(size_t buffer_length, size_t element_index, void* buffer) const { + ThrowOnError(GetApi().GetStringTensorElement(this->p_, buffer_length, element_index, buffer)); +} + +template +inline std::string ConstValueImpl::GetStringTensorElement(size_t element_index) const { + size_t buffer_length; + ThrowOnError(GetApi().GetStringTensorElementLength(this->p_, element_index, &buffer_length)); + + std::string s; + s.resize(buffer_length); + ThrowOnError(GetApi().GetStringTensorElement(this->p_, buffer_length, element_index, &s[0])); + return s; +} + +template +inline void ConstValueImpl::GetStringTensorContent(void* buffer, size_t buffer_length, size_t* offsets, size_t offsets_count) const { + ThrowOnError(GetApi().GetStringTensorContent(this->p_, buffer, buffer_length, offsets, offsets_count)); +} + +#if !defined(DISABLE_SPARSE_TENSORS) +template +inline OrtSparseFormat ConstValueImpl::GetSparseFormat() const { + OrtSparseFormat format; + ThrowOnError(GetApi().GetSparseTensorFormat(this->p_, &format)); + return format; +} + +template +inline TensorTypeAndShapeInfo ConstValueImpl::GetSparseTensorValuesTypeAndShapeInfo() const { + OrtTensorTypeAndShapeInfo* output; + ThrowOnError(GetApi().GetSparseTensorValuesTypeAndShape(this->p_, &output)); + return TensorTypeAndShapeInfo{output}; +} + +template +inline TensorTypeAndShapeInfo ConstValueImpl::GetSparseTensorIndicesTypeShapeInfo(OrtSparseIndicesFormat indices_format) const { + OrtTensorTypeAndShapeInfo* output; + ThrowOnError(GetApi().GetSparseTensorIndicesTypeShape(this->p_, indices_format, &output)); + return TensorTypeAndShapeInfo{output}; +} + +template +template +inline const R* ConstValueImpl::GetSparseTensorIndicesData(OrtSparseIndicesFormat indices_format, size_t& num_indices) const { + const void* out; + ThrowOnError(GetApi().GetSparseTensorIndices(this->p_, indices_format, &num_indices, &out)); + return reinterpret_cast(out); +} + +template +inline bool ConstValueImpl::IsSparseTensor() const { + int out; + ThrowOnError(GetApi().IsSparseTensor(this->p_, &out)); + return out != 0; +} + +template +template +inline const R* ConstValueImpl::GetSparseTensorValues() const { + const void* out; + ThrowOnError(GetApi().GetSparseTensorValues(this->p_, &out)); + return reinterpret_cast(out); +} + +#endif + +template +void ConstValueImpl::GetTensorElementTypeAndShapeDataReference(ONNXTensorElementDataType& elem_type, + Shape& shape) const { + ThrowOnError(GetApi().GetTensorElementTypeAndShapeDataReference(this->p_, &elem_type, &shape.shape, + &shape.shape_len)); +} + +template +void ValueImpl::FillStringTensor(const char* const* s, size_t s_len) { + ThrowOnError(GetApi().FillStringTensor(this->p_, s, s_len)); +} + +template +void ValueImpl::FillStringTensorElement(const char* s, size_t index) { + ThrowOnError(GetApi().FillStringTensorElement(this->p_, s, index)); +} + +template +inline char* ValueImpl::GetResizedStringTensorElementBuffer(size_t index, size_t buffer_length) { + char* result; + ThrowOnError(GetApi().GetResizedStringTensorElementBuffer(this->p_, index, buffer_length, &result)); + return result; +} + +template +void* ValueImpl::GetTensorMutableRawData() { + void* out; + ThrowOnError(GetApi().GetTensorMutableData(this->p_, &out)); + return out; +} + +template +template +R* ValueImpl::GetTensorMutableData() { + R* out; + ThrowOnError(GetApi().GetTensorMutableData(this->p_, (void**)&out)); + return out; +} + +template +template +R& ValueImpl::At(const std::vector& location) { + static_assert(!std::is_same::value, "this api does not support std::string"); + R* out; + ThrowOnError(GetApi().TensorAt(this->p_, location.data(), location.size(), (void**)&out)); + return *out; +} + +#if !defined(DISABLE_SPARSE_TENSORS) +template +void ValueImpl::UseCooIndices(int64_t* indices_data, size_t indices_num) { + ThrowOnError(GetApi().UseCooIndices(this->p_, indices_data, indices_num)); +} + +template +void ValueImpl::UseCsrIndices(int64_t* inner_data, size_t inner_num, int64_t* outer_data, size_t outer_num) { + ThrowOnError(GetApi().UseCsrIndices(this->p_, inner_data, inner_num, outer_data, outer_num)); +} + +template +void ValueImpl::UseBlockSparseIndices(const Shape& indices_shape, int32_t* indices_data) { + ThrowOnError(GetApi().UseBlockSparseIndices(this->p_, indices_shape.shape, indices_shape.shape_len, indices_data)); +} + +template +void ValueImpl::FillSparseTensorCoo(const OrtMemoryInfo* mem_info, const OrtSparseValuesParam& values_param, + const int64_t* indices_data, size_t indices_num) { + ThrowOnError(GetApi().FillSparseTensorCoo(this->p_, mem_info, values_param.values_shape, + values_param.values_shape_len, values_param.data.p_data, + indices_data, indices_num)); +} + +template +void ValueImpl::FillSparseTensorCsr(const OrtMemoryInfo* data_mem_info, + const OrtSparseValuesParam& values, + const int64_t* inner_indices_data, size_t inner_indices_num, + const int64_t* outer_indices_data, size_t outer_indices_num) { + ThrowOnError(GetApi().FillSparseTensorCsr(this->p_, data_mem_info, values.values_shape, values.values_shape_len, values.data.p_data, + inner_indices_data, inner_indices_num, + outer_indices_data, outer_indices_num)); +} + +template +void ValueImpl::FillSparseTensorBlockSparse(const OrtMemoryInfo* data_mem_info, + const OrtSparseValuesParam& values, + const Shape& indices_shape, + const int32_t* indices_data) { + ThrowOnError(GetApi().FillSparseTensorBlockSparse(this->p_, data_mem_info, values.values_shape, values.values_shape_len, values.data.p_data, + indices_shape.shape, indices_shape.shape_len, + indices_data)); +} + +#endif // !defined(DISABLE_SPARSE_TENSORS) + +} // namespace detail + +template +inline Value Value::CreateTensor(const OrtMemoryInfo* info, T* p_data, size_t p_data_element_count, + const int64_t* shape, size_t shape_len) { + return CreateTensor(info, p_data, p_data_element_count * sizeof(T), shape, shape_len, TypeToTensorType::type); +} + +inline Value Value::CreateTensor(const OrtMemoryInfo* info, void* p_data, size_t p_data_byte_count, + const int64_t* shape, size_t shape_len, + ONNXTensorElementDataType type) { + OrtValue* out; + ThrowOnError(GetApi().CreateTensorWithDataAsOrtValue(info, p_data, p_data_byte_count, shape, shape_len, type, &out)); + return Value{out}; +} + +inline Value Value::CreateTensor(OrtAllocator* deleter, void* p_data, size_t p_data_byte_count, + const int64_t* shape, size_t shape_len, + ONNXTensorElementDataType type) { + OrtValue* out; + ThrowOnError(GetApi().CreateTensorWithDataAndDeleterAsOrtValue(deleter, p_data, p_data_byte_count, + shape, shape_len, type, &out)); + return Value{out}; +} + +template +inline Value Value::CreateTensor(OrtAllocator* allocator, const int64_t* shape, size_t shape_len) { + return CreateTensor(allocator, shape, shape_len, TypeToTensorType::type); +} + +inline Value Value::CreateTensor(OrtAllocator* allocator, const int64_t* shape, size_t shape_len, + ONNXTensorElementDataType type) { + OrtValue* out; + ThrowOnError(GetApi().CreateTensorAsOrtValue(allocator, shape, shape_len, type, &out)); + return Value{out}; +} + +#if !defined(DISABLE_SPARSE_TENSORS) + +template +inline Value Value::CreateSparseTensor(const OrtMemoryInfo* info, T* p_data, const Shape& dense_shape, + const Shape& values_shape) { + return CreateSparseTensor(info, p_data, dense_shape, values_shape, TypeToTensorType::type); +} + +inline Value Value::CreateSparseTensor(const OrtMemoryInfo* info, void* p_data, const Shape& dense_shape, + const Shape& values_shape, ONNXTensorElementDataType type) { + OrtValue* out; + ThrowOnError(GetApi().CreateSparseTensorWithValuesAsOrtValue(info, p_data, dense_shape.shape, dense_shape.shape_len, + values_shape.shape, values_shape.shape_len, type, + &out)); + return Value{out}; +} + +template +inline Value Value::CreateSparseTensor(OrtAllocator* allocator, const Shape& dense_shape) { + return CreateSparseTensor(allocator, dense_shape, TypeToTensorType::type); +} + +inline Value Value::CreateSparseTensor(OrtAllocator* allocator, const Shape& dense_shape, + ONNXTensorElementDataType type) { + OrtValue* out; + ThrowOnError(GetApi().CreateSparseTensorAsOrtValue(allocator, dense_shape.shape, dense_shape.shape_len, type, &out)); + return Value{out}; +} +#endif // !defined(DISABLE_SPARSE_TENSORS) + +inline Value Value::CreateMap(const Value& keys, const Value& values) { + OrtValue* out; + const OrtValue* inputs[2] = {keys, values}; + ThrowOnError(GetApi().CreateValue(inputs, 2, ONNX_TYPE_MAP, &out)); + return Value{out}; +} + +inline Value Value::CreateSequence(const std::vector& values) { + OrtValue* out; + std::vector values_ort{values.data(), values.data() + values.size()}; + ThrowOnError(GetApi().CreateValue(values_ort.data(), values_ort.size(), ONNX_TYPE_SEQUENCE, &out)); + return Value{out}; +} + +template +inline Value Value::CreateOpaque(const char* domain, const char* type_name, const T& data_container) { + OrtValue* out; + ThrowOnError(GetApi().CreateOpaqueValue(domain, type_name, &data_container, sizeof(T), &out)); + return Value{out}; +} + +// +// Custom OP Inlines +// +inline Logger::Logger(const OrtLogger* logger) : logger_(logger) { + Ort::ThrowOnError(GetApi().Logger_GetLoggingSeverityLevel(this->logger_, &this->cached_severity_level_)); +} + +inline OrtLoggingLevel Logger::GetLoggingSeverityLevel() const noexcept { + return cached_severity_level_; +} + +inline Status Logger::LogMessage(OrtLoggingLevel log_severity_level, const ORTCHAR_T* file_path, int line_number, + const char* func_name, const char* message) const noexcept { + OrtStatus* status = GetApi().Logger_LogMessage(logger_, log_severity_level, message, file_path, line_number, + func_name); + return Status{status}; +} + +// Disable warnings about the format string not being a literal (-Wformat-nonliteral and -Wformat-security) +// for gcc and clang. The alternative is to use actual C-style variadic parameters and apply +// __attribute__(format(printf...)), which does not work with variadic templates. +#if defined(__GNUC__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wformat-nonliteral" +#pragma GCC diagnostic ignored "-Wformat-security" +#elif defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wformat-nonliteral" +#pragma clang diagnostic ignored "-Wformat-security" +#endif +template +inline Status Logger::LogFormattedMessage(OrtLoggingLevel log_severity_level, const ORTCHAR_T* file_path, + int line_number, const char* func_name, const char* format, + Args&&... args) const noexcept { + int msg_len = std::snprintf(nullptr, 0U, format, std::forward(args)...); + + if (msg_len < 0) { // Formatting error + return Status("Failed to log message due to formatting error", OrtErrorCode::ORT_FAIL); + } + + OrtStatus* status = nullptr; + const size_t buffer_size = static_cast(msg_len) + 1U; + + constexpr size_t kStackBufferSize = 1024; + + if (buffer_size < kStackBufferSize) { + char buffer[kStackBufferSize]; + snprintf(buffer, kStackBufferSize, format, std::forward(args)...); + status = GetApi().Logger_LogMessage(logger_, log_severity_level, buffer, file_path, line_number, func_name); + } else { + // std::make_unique is only supported starting at C++14. +#if (__cplusplus >= 201402L) || (_MSC_VER >= 1900) + auto buffer = std::make_unique(buffer_size); +#else + std::unique_ptr buffer(new char[buffer_size]); +#endif + std::snprintf(buffer.get(), buffer_size, format, std::forward(args)...); + status = GetApi().Logger_LogMessage(logger_, log_severity_level, buffer.get(), file_path, line_number, func_name); + } + + return Status{status}; +} +// Re-enable -Wformat-nonliteral and -Wformat-security +#if defined(__GNUC__) +#pragma GCC diagnostic pop +#elif defined(__clang__) +#pragma clang diagnostic pop +#endif + +inline KernelContext::KernelContext(OrtKernelContext* context) : ctx_(context) { +} + +inline size_t KernelContext::GetInputCount() const { + size_t out = 0; + Ort::ThrowOnError(GetApi().KernelContext_GetInputCount(ctx_, &out)); + return out; +} + +inline size_t KernelContext::GetOutputCount() const { + size_t out = 0; + Ort::ThrowOnError(GetApi().KernelContext_GetOutputCount(ctx_, &out)); + return out; +} + +inline ConstValue KernelContext::GetInput(size_t index) const { + const OrtValue* out = nullptr; + Ort::ThrowOnError(GetApi().KernelContext_GetInput(ctx_, index, &out)); + return ConstValue{out}; +} + +inline UnownedValue KernelContext::GetOutput(size_t index, const int64_t* dim_values, size_t dim_count) const { + OrtValue* out = nullptr; + Ort::ThrowOnError(GetApi().KernelContext_GetOutput(ctx_, index, dim_values, dim_count, &out)); + return UnownedValue(out); +} + +inline UnownedValue KernelContext::GetOutput(size_t index, const std::vector& dims) const { + OrtValue* out = nullptr; + Ort::ThrowOnError(GetApi().KernelContext_GetOutput(ctx_, index, dims.data(), dims.size(), &out)); + return UnownedValue(out); +} + +inline void* KernelContext::GetGPUComputeStream() const { + void* out = nullptr; + Ort::ThrowOnError(GetApi().KernelContext_GetGPUComputeStream(ctx_, &out)); + return out; +} + +inline Ort::Allocator KernelContext::GetAllocator(const OrtMemoryInfo& memory_info) const { + OrtAllocator* out = nullptr; + Ort::ThrowOnError(GetApi().KernelContext_GetAllocator(ctx_, &memory_info, &out)); + return Ort::Allocator{out}; +} + +inline Logger KernelContext::GetLogger() const { + const OrtLogger* out = nullptr; + ThrowOnError(GetApi().KernelContext_GetLogger(this->ctx_, &out)); + return Logger{out}; +} + +inline void KernelContext::ParallelFor(void (*fn)(void*, size_t), size_t total, size_t num_batch, void* usr_data) const { + ThrowOnError(GetApi().KernelContext_ParallelFor(ctx_, fn, total, num_batch, usr_data)); +} + +namespace detail { + +template +constexpr OrtOpAttrType TypeToAttrType(); + +template <> +inline constexpr OrtOpAttrType TypeToAttrType() { + return OrtOpAttrType::ORT_OP_ATTR_INT; +} + +template <> +inline constexpr OrtOpAttrType TypeToAttrType() { + return OrtOpAttrType::ORT_OP_ATTR_FLOAT; +} + +template +inline constexpr OrtOpAttrType TypeToAttrsType(); + +template <> +inline constexpr OrtOpAttrType TypeToAttrsType() { + return OrtOpAttrType::ORT_OP_ATTR_INTS; +} + +template <> +inline constexpr OrtOpAttrType TypeToAttrsType() { + return OrtOpAttrType::ORT_OP_ATTR_FLOATS; +} + +inline Status CheckAttrType(const OrtOpAttr* attr, OrtOpAttrType requested_type) { + OrtOpAttrType type; + Ort::Status status(GetApi().OpAttr_GetType(attr, &type)); + if (!status.IsOK()) return status; + if (requested_type != type) { + std::string msg = "Attribute type mismatch: expected " + std::to_string(requested_type) + + ", but got " + std::to_string(type); + return Ort::Status(msg.c_str(), OrtErrorCode::ORT_INVALID_ARGUMENT); + } + return Ort::Status{}; +} + +inline size_t GetDataSize(const OrtOpAttr* attr, OrtOpAttrType attr_type) { + size_t result{}; + // Ignore the status here because we check the data type so the error should only be about + // the size + [[maybe_unused]] Status status{GetApi().ReadOpAttr(attr, attr_type, nullptr, 0, &result)}; + return result; +} + +template +Ort::Status GetNumericValue(const OrtOpAttr* attr, T& out) { + static_assert(std::is_arithmetic::value, "T must be an arithmetic type"); + size_t size{}; + return Ort::Status{GetApi().ReadOpAttr(attr, TypeToAttrType(), &out, sizeof(out), &size)}; +} + +template +struct GetValueImpl { + static Status GetValue(const OrtOpAttr* attr, T& out) { + return GetNumericValue(attr, out); + } + static Status GetValues(const OrtOpAttr* attr, std::vector& out) { + // Api deficiency when it comes to value arrays. It is not possible + // to tell if the error is due to the type mismatch or the size + // so we check the type first, and then ignore the status of the size check + constexpr auto deduced_type = TypeToAttrsType(); + auto status = CheckAttrType(attr, deduced_type); + if (!status.IsOK()) return status; + auto size = GetDataSize(attr, deduced_type); + std::vector result; + if (size > 0) { + result.resize(size / sizeof(T)); + status = Status{GetApi().ReadOpAttr( + attr, deduced_type, result.data(), size, &size)}; + if (!status.IsOK()) return status; + } + out.swap(result); + return status; + } +}; + +// Create GetValueImpl specializations for std::string +template <> +struct GetValueImpl { + static Status GetValue(const OrtOpAttr* attr, std::string& out) { + // Api deficiency when it comes to value arrays. It is not possible + // to tell if the error is due to the type mismatch or the size + // so we check the type first, and then ignore the status of the size check + auto status = CheckAttrType(attr, OrtOpAttrType::ORT_OP_ATTR_STRING); + if (!status.IsOK()) return status; + auto size = GetDataSize(attr, OrtOpAttrType::ORT_OP_ATTR_STRING); + std::string result; + if (size > 0) { + result.resize(size); + // some compilers in use do not support std::string::data() non-const + auto* buffer = &result[0]; + status = Status{GetApi().ReadOpAttr( + attr, OrtOpAttrType::ORT_OP_ATTR_STRING, buffer, size, &size)}; + if (!status.IsOK()) return status; + } + out.swap(result); + return status; + } + static Status GetValues(const OrtOpAttr* attr, std::vector& out) { + auto status = CheckAttrType(attr, OrtOpAttrType::ORT_OP_ATTR_STRINGS); + if (!status.IsOK()) return status; + + std::vector result; + size_t total_buffer_size = GetDataSize(attr, OrtOpAttrType::ORT_OP_ATTR_STRINGS); + if (total_buffer_size > 0) { + // Create a temporary buffer to hold the string data + std::vector buffer(total_buffer_size); + status = Status{GetApi().ReadOpAttr(attr, OrtOpAttrType::ORT_OP_ATTR_STRINGS, buffer.data(), + total_buffer_size, &total_buffer_size)}; + if (!status.IsOK()) return status; + + const char* data = buffer.data(); + const char* end = data + total_buffer_size; + while (data < end) { + result.emplace_back(data); + data += result.back().size() + 1; // Move past the null terminator + } + } + out.swap(result); + return status; + } +}; + +template +template +inline Status ConstOpAttrImpl::GetValue(R& out) const { + return GetValueImpl::GetValue(this->p_, out); +} + +template +template +inline Status ConstOpAttrImpl::GetValueArray(std::vector& out) const { + return GetValueImpl::GetValues(this->p_, out); +} + +template +inline Status ConstOpAttrImpl::GetTensorAttributeAsOrtValue(Value& out) const { + OrtValue* tensor_value = nullptr; + auto status = Status(GetApi().OpAttr_GetTensorAttributeAsOrtValue(this->p_, &tensor_value)); + if (!status.IsOK()) return status; + out = Value{tensor_value}; + return status; +} + +template +inline std::string ConstOpAttrImpl::GetName() const { + const char* name = nullptr; + ThrowOnError(GetApi().OpAttr_GetName(this->p_, &name)); + if (name != nullptr) { + return name; + } + return {}; +} + +template +inline OrtOpAttrType ConstOpAttrImpl::GetType() const { + OrtOpAttrType type; + ThrowOnError(GetApi().OpAttr_GetType(this->p_, &type)); + return type; +} +} // namespace detail + +inline OpAttr::OpAttr(const char* name, const void* data, int len, OrtOpAttrType type) { + Ort::ThrowOnError(GetApi().CreateOpAttr(name, data, len, type, &p_)); +} + +namespace detail { +template +inline KernelInfo KernelInfoImpl::Copy() const { + OrtKernelInfo* info_copy = nullptr; + Ort::ThrowOnError(GetApi().CopyKernelInfo(this->p_, &info_copy)); + return KernelInfo{info_copy}; +} + +template +inline size_t KernelInfoImpl::GetInputCount() const { + size_t out = 0; + ThrowOnError(GetApi().KernelInfo_GetInputCount(this->p_, &out)); + return out; +} + +template +inline size_t KernelInfoImpl::GetOutputCount() const { + size_t out = 0; + ThrowOnError(GetApi().KernelInfo_GetOutputCount(this->p_, &out)); + return out; +} + +template +inline std::string KernelInfoImpl::GetInputName(size_t index) const { + size_t size = 0; + + // Feed nullptr for the data buffer to query the true size of the string value + Ort::ThrowOnError(GetApi().KernelInfo_GetInputName(this->p_, index, nullptr, &size)); + + std::string out; + out.resize(size); + Ort::ThrowOnError(GetApi().KernelInfo_GetInputName(this->p_, index, &out[0], &size)); + out.resize(size - 1); // remove the terminating character '\0' + + return out; +} + +template +inline std::string KernelInfoImpl::GetOutputName(size_t index) const { + size_t size = 0; + + // Feed nullptr for the data buffer to query the true size of the string value + Ort::ThrowOnError(GetApi().KernelInfo_GetOutputName(this->p_, index, nullptr, &size)); + + std::string out; + out.resize(size); + Ort::ThrowOnError(GetApi().KernelInfo_GetOutputName(this->p_, index, &out[0], &size)); + out.resize(size - 1); // remove the terminating character '\0' + + return out; +} + +template +inline TypeInfo KernelInfoImpl::GetInputTypeInfo(size_t index) const { + OrtTypeInfo* out = nullptr; + ThrowOnError(GetApi().KernelInfo_GetInputTypeInfo(this->p_, index, &out)); + return TypeInfo{out}; +} + +template +inline TypeInfo KernelInfoImpl::GetOutputTypeInfo(size_t index) const { + OrtTypeInfo* out = nullptr; + ThrowOnError(GetApi().KernelInfo_GetOutputTypeInfo(this->p_, index, &out)); + return TypeInfo{out}; +} + +template +inline Value KernelInfoImpl::GetTensorAttribute(const char* name, OrtAllocator* allocator) const { + OrtValue* out = nullptr; + ThrowOnError(GetApi().KernelInfoGetAttribute_tensor(this->p_, name, allocator, &out)); + return Value{out}; +} + +template +inline ConstValue KernelInfoImpl::GetTensorConstantInput(size_t index, int* is_constant) const { + const OrtValue* out = nullptr; + ThrowOnError(GetApi().KernelInfoGetConstantInput_tensor(this->p_, index, is_constant, &out)); + return ConstValue{out}; +} + +template +inline std::string KernelInfoImpl::GetNodeName() const { + size_t size = 0; + + // Feed nullptr for the data buffer to query the true size of the string value + Ort::ThrowOnError(GetApi().KernelInfo_GetNodeName(this->p_, nullptr, &size)); + + std::string out; + out.resize(size); + Ort::ThrowOnError(GetApi().KernelInfo_GetNodeName(this->p_, &out[0], &size)); + out.resize(size - 1); // remove the terminating character '\0' + + return out; +} + +template +inline Logger KernelInfoImpl::GetLogger() const { + const OrtLogger* out = nullptr; + ThrowOnError(GetApi().KernelInfo_GetLogger(this->p_, &out)); + return Logger{out}; +} + +template +inline KeyValuePairs KernelInfoImpl::GetConfigEntries() const { + OrtKeyValuePairs* out = nullptr; + Ort::ThrowOnError(GetApi().KernelInfo_GetConfigEntries(this->p_, &out)); + return KeyValuePairs{out}; +} + +template +inline std::string KernelInfoImpl::GetOperatorDomain() const { + size_t size = 0; + + // Feed nullptr for the data buffer to query the true size of the string value + Ort::ThrowOnError(GetApi().KernelInfo_GetOperatorDomain(this->p_, nullptr, &size)); + + std::string out; + out.resize(size); + Ort::ThrowOnError(GetApi().KernelInfo_GetOperatorDomain(this->p_, &out[0], &size)); + out.resize(size - 1); // remove the terminating character '\0' + + return out; +} + +template +inline std::string KernelInfoImpl::GetOperatorType() const { + size_t size = 0; + + // Feed nullptr for the data buffer to query the true size of the string value + Ort::ThrowOnError(GetApi().KernelInfo_GetOperatorType(this->p_, nullptr, &size)); + + std::string out; + out.resize(size); + Ort::ThrowOnError(GetApi().KernelInfo_GetOperatorType(this->p_, &out[0], &size)); + out.resize(size - 1); // remove the terminating character '\0' + + return out; +} + +template +inline int KernelInfoImpl::GetOperatorSinceVersion() const { + int out = 0; + Ort::ThrowOnError(GetApi().KernelInfo_GetOperatorSinceVersion(this->p_, &out)); + return out; +} + +template +inline const OrtEp* KernelInfoImpl::GetEp() const { + const OrtEp* ep = nullptr; + Ort::ThrowOnError(GetEpApi().KernelInfo_GetEp(this->p_, &ep)); + return ep; +} + +inline void attr_utils::GetAttr(const OrtKernelInfo* p, const char* name, float& out) { + Ort::ThrowOnError(GetApi().KernelInfoGetAttribute_float(p, name, &out)); +} + +inline void attr_utils::GetAttr(const OrtKernelInfo* p, const char* name, int64_t& out) { + Ort::ThrowOnError(GetApi().KernelInfoGetAttribute_int64(p, name, &out)); +} + +inline void attr_utils::GetAttr(const OrtKernelInfo* p, const char* name, std::string& result) { + size_t size = 0; + // Feed nullptr for the data buffer to query the true size of the string attribute + Ort::ThrowOnError(GetApi().KernelInfoGetAttribute_string(p, name, nullptr, &size)); + + std::string out; + out.resize(size); + Ort::ThrowOnError(GetApi().KernelInfoGetAttribute_string(p, name, &out[0], &size)); + out.resize(size - 1); // remove the terminating character '\0' + out.swap(result); +} + +inline void attr_utils::GetAttrs(const OrtKernelInfo* p, const char* name, std::vector& result) { + size_t size = 0; + // Feed nullptr for the data buffer to query the true size of the attribute + Ort::ThrowOnError(GetApi().KernelInfoGetAttributeArray_float(p, name, nullptr, &size)); + + std::vector out; + out.resize(size); + Ort::ThrowOnError(GetApi().KernelInfoGetAttributeArray_float(p, name, out.data(), &size)); + out.swap(result); +} + +inline void attr_utils::GetAttrs(const OrtKernelInfo* p, const char* name, std::vector& result) { + size_t size = 0; + + // Feed nullptr for the data buffer to query the true size of the attribute + Ort::ThrowOnError(GetApi().KernelInfoGetAttributeArray_int64(p, name, nullptr, &size)); + + std::vector out; + out.resize(size); + Ort::ThrowOnError(GetApi().KernelInfoGetAttributeArray_int64(p, name, out.data(), &size)); + out.swap(result); +} +} // namespace detail + +inline KernelInfo::KernelInfo(OrtKernelInfo* info) : detail::KernelInfoImpl{info} {} + +inline Op::Op(OrtOp* p) : detail::Base(p) {} + +inline Op Op::Create(const OrtKernelInfo* info, const char* op_name, const char* domain, int version, + const char** type_constraint_names, + const ONNXTensorElementDataType* type_constraint_values, + size_t type_constraint_count, + const OpAttr* attr_values, size_t attr_count, + size_t input_count, size_t output_count) { + static_assert(sizeof(OpAttr) == sizeof(OrtOpAttr*), + "OpAttr's is expected to be just an array of OrtOpAttr in memory so we can reinterpret safely"); + auto attr_input_values = reinterpret_cast(attr_values); + OrtOp* op; + Ort::ThrowOnError(GetApi().CreateOp(info, op_name, domain, version, type_constraint_names, type_constraint_values, + static_cast(type_constraint_count), + attr_input_values, + static_cast(attr_count), + static_cast(input_count), + static_cast(output_count), &op)); + return Op{op}; +} + +inline void Op::Invoke(const OrtKernelContext* context, + const Value* input_values, + size_t input_count, + Value* output_values, + size_t output_count) { + static_assert(sizeof(Value) == sizeof(OrtValue*), + "Value is really just an array of OrtValue* in memory, so we can reinterpret_cast safely"); + auto ort_input_values = reinterpret_cast(input_values); + auto ort_output_values = reinterpret_cast(output_values); + Ort::ThrowOnError(GetApi().InvokeOp(context, p_, ort_input_values, static_cast(input_count), + ort_output_values, static_cast(output_count))); +} + +inline void Op::Invoke(const OrtKernelContext* context, + const OrtValue* const* input_values, + size_t input_count, + OrtValue* const* output_values, + size_t output_count) { + Ort::ThrowOnError(GetApi().InvokeOp(context, p_, input_values, static_cast(input_count), + output_values, static_cast(output_count))); +} + +inline std::string GetVersionString() { + return OrtGetApiBase()->GetVersionString(); +} + +inline std::string GetBuildInfoString() { + return GetApi().GetBuildInfoString(); +} + +inline std::vector GetAvailableProviders() { + char** providers; + int len; + + auto release_fn = [&len](char** providers) { + // This should always return nullptr. + ThrowOnError(GetApi().ReleaseAvailableProviders(providers, len)); + }; + + ThrowOnError(GetApi().GetAvailableProviders(&providers, &len)); + std::unique_ptr guard(providers, release_fn); + std::vector available_providers; + available_providers.reserve(static_cast(len)); + for (int i = 0; i < len; ++i) { + available_providers.emplace_back(providers[i]); + } + return available_providers; +} + +template +void CustomOpBase::GetSessionConfigs(std::unordered_map& out, + ConstSessionOptions options) const { + const TOp* derived = static_cast(this); + std::vector keys = derived->GetSessionConfigKeys(); + + out.reserve(keys.size()); + + std::string config_entry_key = detail::MakeCustomOpConfigEntryKey(derived->GetName(), ""); + const size_t prefix_size = config_entry_key.length(); + + for (const auto& key : keys) { + config_entry_key.resize(prefix_size); + config_entry_key.append(key); + out[key] = options.GetConfigEntryOrDefault(config_entry_key.c_str(), ""); + } +} + +inline ShapeInferContext::ShapeInferContext(const OrtApi* ort_api, + OrtShapeInferContext* ctx) : ort_api_(ort_api), ctx_(ctx) { + size_t input_count = 0; + Ort::ThrowOnError(ort_api_->ShapeInferContext_GetInputCount(ctx_, &input_count)); + for (size_t ith_input = 0; ith_input < input_count; ++ith_input) { + OrtTensorTypeAndShapeInfo* info{}; + Ort::ThrowOnError(ort_api_->ShapeInferContext_GetInputTypeShape(ctx, ith_input, &info)); + TensorTypeAndShapeInfo type_shape_info(info); + auto integer_shape = type_shape_info.GetShape(); + std::vector symbolic_shape(integer_shape.size(), {}); + if (!integer_shape.empty()) { + type_shape_info.GetSymbolicDimensions(&symbolic_shape[0], integer_shape.size()); + } + Shape shape; + for (size_t ith = 0; ith < integer_shape.size(); ++ith) { + if (symbolic_shape[ith] && std::string{symbolic_shape[ith]}.size() > 0) { + shape.emplace_back(symbolic_shape[ith]); + } else { + shape.emplace_back(integer_shape[ith]); + } + } + input_shapes_.push_back(std::move(shape)); + type_shape_info.release(); + } +} + +inline Status ShapeInferContext::SetOutputShape(size_t indice, const Shape& shape, ONNXTensorElementDataType type) { + OrtTensorTypeAndShapeInfo* info = {}; + ORT_CXX_RETURN_ON_API_FAIL(ort_api_->CreateTensorTypeAndShapeInfo(&info)); + ORT_CXX_RETURN_ON_API_FAIL(ort_api_->SetTensorElementType(info, type)); + + using InfoPtr = std::unique_ptr>; + + InfoPtr info_ptr(info, [this](OrtTensorTypeAndShapeInfo* obj) { + ort_api_->ReleaseTensorTypeAndShapeInfo(obj); + }); + + std::vector integer_dims; + std::vector symbolic_dims; + + for (const auto dim : shape) { + if (dim.IsInt()) { + integer_dims.push_back(dim.AsInt()); + symbolic_dims.push_back(""); + } else { + if (!dim.AsSym() || std::string{dim.AsSym()}.empty()) { + ORT_CXX_API_THROW("Symbolic dim must not be an empty string", ORT_INVALID_ARGUMENT); + } + integer_dims.push_back(SymbolicInteger::INVALID_INT_DIM); + symbolic_dims.push_back(dim.AsSym()); + } + } + + ORT_CXX_RETURN_ON_API_FAIL(ort_api_->SetDimensions(info, integer_dims.data(), integer_dims.size())); + ORT_CXX_RETURN_ON_API_FAIL(ort_api_->SetSymbolicDimensions(info, symbolic_dims.data(), symbolic_dims.size())); + ORT_CXX_RETURN_ON_API_FAIL(ort_api_->ShapeInferContext_SetOutputTypeShape(ctx_, indice, info)); + return Status{nullptr}; +} + +inline int64_t ShapeInferContext::GetAttrInt(const char* attr_name) { + auto attr = GetAttrHdl(attr_name); + int64_t value; + Status status = attr.GetValue(value); + if (!status.IsOK()) { + ORT_CXX_API_THROW("Getting int attribute failed: " + status.GetErrorMessage(), status.GetErrorCode()); + } + return value; +} + +inline ShapeInferContext::Ints ShapeInferContext::GetAttrInts(const char* attr_name) { + auto attr = GetAttrHdl(attr_name); + ShapeInferContext::Ints result; + auto status = attr.GetValueArray(result); + if (!status.IsOK()) { + ORT_CXX_API_THROW("Getting ints attribute failed: " + status.GetErrorMessage(), status.GetErrorCode()); + } + return result; +} + +inline float ShapeInferContext::GetAttrFloat(const char* attr_name) { + auto attr = GetAttrHdl(attr_name); + float value; + Status status = attr.GetValue(value); + if (!status.IsOK()) { + ORT_CXX_API_THROW("Getting float attribute failed: " + status.GetErrorMessage(), status.GetErrorCode()); + } + return value; +} + +inline ShapeInferContext::Floats ShapeInferContext::GetAttrFloats(const char* attr_name) { + auto attr = GetAttrHdl(attr_name); + ShapeInferContext::Floats result; + auto status = attr.GetValueArray(result); + if (!status.IsOK()) { + ORT_CXX_API_THROW("Getting floats attribute failed: " + status.GetErrorMessage(), status.GetErrorCode()); + } + return result; +} + +inline std::string ShapeInferContext::GetAttrString(const char* attr_name) { + auto attr = GetAttrHdl(attr_name); + std::string value; + Status status = attr.GetValue(value); + if (!status.IsOK()) { + ORT_CXX_API_THROW("Getting string attribute failed: " + status.GetErrorMessage(), status.GetErrorCode()); + } + return value; +} + +inline ShapeInferContext::Strings ShapeInferContext::GetAttrStrings(const char* attr_name) { + auto attr = GetAttrHdl(attr_name); + ShapeInferContext::Strings result; + auto status = attr.GetValueArray(result); + if (!status.IsOK()) { + ORT_CXX_API_THROW("Getting strings attribute failed: " + status.GetErrorMessage(), status.GetErrorCode()); + } + return result; +} + +inline ConstOpAttr ShapeInferContext::GetAttrHdl(const char* attr_name) const { + const OrtOpAttr* attr_hdl = {}; + Ort::ThrowOnError(ort_api_->ShapeInferContext_GetAttribute(ctx_, attr_name, &attr_hdl)); + return ConstOpAttr{attr_hdl}; +} + +namespace detail { +inline std::vector StringsToCharPtrs(const std::vector& strings) { + std::vector ptrs; + ptrs.reserve(strings.size()); + std::transform(strings.begin(), strings.end(), std::back_inserter(ptrs), + [](const std::string& s) { return s.c_str(); }); + + return ptrs; +} +} // namespace detail + +namespace detail { +template +inline size_t ConstNodeImpl::GetId() const { + size_t id; + ThrowOnError(GetApi().Node_GetId(this->p_, &id)); + return id; +} + +template +inline std::string ConstNodeImpl::GetName() const { + const char* name; + ThrowOnError(GetApi().Node_GetName(this->p_, &name)); + return std::string(name); +} + +template +inline std::string ConstNodeImpl::GetOperatorType() const { + const char* type; + ThrowOnError(GetApi().Node_GetOperatorType(this->p_, &type)); + return std::string(type); +} + +template +inline std::string ConstNodeImpl::GetDomain() const { + const char* domain; + ThrowOnError(GetApi().Node_GetDomain(this->p_, &domain)); + return std::string(domain); +} + +template +inline int ConstNodeImpl::GetSinceVersion() const { + int since_version; + ThrowOnError(GetApi().Node_GetSinceVersion(this->p_, &since_version)); + return since_version; +} + +template +inline std::vector ConstNodeImpl::GetInputs() const { + static_assert(sizeof(const OrtValueInfo*) == sizeof(ConstValueInfo)); + size_t num_vi; + ThrowOnError(GetApi().Node_GetNumInputs(this->p_, &num_vi)); + std::vector result; + if (num_vi > 0) { + result.resize(num_vi); + ThrowOnError(GetApi().Node_GetInputs(this->p_, reinterpret_cast(result.data()), num_vi)); + } + return result; +} + +template +inline std::vector ConstNodeImpl::GetOutputs() const { + static_assert(sizeof(const OrtValueInfo*) == sizeof(ConstValueInfo)); + size_t num_vi; + ThrowOnError(GetApi().Node_GetNumOutputs(this->p_, &num_vi)); + std::vector result; + if (num_vi > 0) { + result.resize(num_vi); + ThrowOnError(GetApi().Node_GetOutputs(this->p_, reinterpret_cast(result.data()), num_vi)); + } + return result; +} + +template +inline std::vector ConstNodeImpl::GetImplicitInputs() const { + static_assert(sizeof(const OrtValueInfo*) == sizeof(ConstValueInfo)); + size_t num_vi; + ThrowOnError(GetApi().Node_GetNumImplicitInputs(this->p_, &num_vi)); + std::vector result; + if (num_vi > 0) { + result.resize(num_vi); + ThrowOnError(GetApi().Node_GetImplicitInputs(this->p_, reinterpret_cast(result.data()), + num_vi)); + } + return result; +} + +template +inline std::vector ConstNodeImpl::GetAttributes() const { + static_assert(sizeof(const OrtOpAttr*) == sizeof(ConstOpAttr), "Must be the same size"); + size_t num_attrs; + ThrowOnError(GetApi().Node_GetNumAttributes(this->p_, &num_attrs)); + std::vector attrs; + if (num_attrs > 0) { + attrs.resize(num_attrs); + ThrowOnError(GetApi().Node_GetAttributes(this->p_, reinterpret_cast(attrs.data()), num_attrs)); + } + return attrs; +} + +template +inline Status ConstNodeImpl::GetAttributeByName(const std::string& name, ConstOpAttr& out) const { + const OrtOpAttr* attr = nullptr; + auto status = Status(GetApi().Node_GetAttributeByName(this->p_, name.c_str(), &attr)); + out = ConstOpAttr{attr}; + return status; +} + +template +inline std::vector ConstNodeImpl::GetSubgraphs() const { + size_t num_graphs; + ThrowOnError(GetApi().Node_GetNumSubgraphs(this->p_, &num_graphs)); + std::vector result; + if (num_graphs > 0) { + std::vector sub_graphs(num_graphs); + std::vector attr_names(num_graphs); + ThrowOnError(GetApi().Node_GetSubgraphs(this->p_, sub_graphs.data(), num_graphs, attr_names.data())); + result.reserve(num_graphs); + for (size_t i = 0; i < num_graphs; ++i) { + result.push_back({std::string(attr_names[i]), ConstGraph{sub_graphs[i]}}); + } + } + return result; +} + +template +inline ConstGraph ConstNodeImpl::GetGraph() const { + const OrtGraph* graph; + ThrowOnError(GetApi().Node_GetGraph(this->p_, &graph)); + return ConstGraph{graph}; +} + +template +inline std::string ConstNodeImpl::GetEpName() const { + const char* name; + ThrowOnError(GetApi().Node_GetEpName(this->p_, &name)); + return std::string(name); +} + +} // namespace detail + +#if !defined(ORT_MINIMAL_BUILD) +// static +inline void Node::Init(const std::string& operator_name, const std::string& operator_domain, + const std::string& node_name, + const std::vector& input_names, + const std::vector& output_names, + std::vector& attributes, + OrtNode*& node) { + auto inputs = detail::StringsToCharPtrs(input_names); + auto outputs = detail::StringsToCharPtrs(output_names); + + std::vector attributes_ptrs; + attributes_ptrs.reserve(attributes.size()); + std::transform(attributes.begin(), attributes.end(), std::back_inserter(attributes_ptrs), + [](OpAttr& attr) -> OrtOpAttr* { return attr; }); + + ThrowOnError(GetModelEditorApi().CreateNode(operator_name.c_str(), operator_domain.c_str(), node_name.c_str(), + inputs.data(), inputs.size(), + outputs.data(), outputs.size(), + attributes_ptrs.data(), attributes_ptrs.size(), + &node)); + + // Node now owns the attributes + std::for_each(attributes.begin(), attributes.end(), [](OpAttr& attr) { attr.release(); }); +} + +inline Node::Node(const std::string& operator_name, const std::string& operator_domain, + const std::string& node_name, + const std::vector& input_names, + const std::vector& output_names, + std::vector& attributes) { + Init(operator_name, operator_domain, node_name, input_names, output_names, attributes, p_); +} + +inline Node::Node(const std::string& operator_name, const std::string& operator_domain, + const std::string& node_name, + const std::vector& input_names, + const std::vector& output_names) { + std::vector empty_attributes; + Init(operator_name, operator_domain, node_name, input_names, output_names, empty_attributes, p_); +} +inline ValueInfo::ValueInfo(const std::string& name, const ConstTypeInfo& type_info) { + ThrowOnError(GetModelEditorApi().CreateValueInfo(name.c_str(), type_info, &p_)); +} +#endif // !defined(ORT_MINIMAL_BUILD) + +namespace detail { +template +inline std::string ConstValueInfoImpl::GetName() const { + const char* p = nullptr; + ThrowOnError(GetApi().GetValueInfoName(this->p_, &p)); + return std::string(p); +} + +template +inline ConstTypeInfo ConstValueInfoImpl::TypeInfo() const { + const OrtTypeInfo* type_info = nullptr; + ThrowOnError(GetApi().GetValueInfoTypeInfo(this->p_, &type_info)); + return ConstTypeInfo{type_info}; +} + +template +inline ValueInfoConsumerProducerInfo ConstValueInfoImpl::GetProducerNode() const { + ValueInfoConsumerProducerInfo info; + const OrtNode* producer; + size_t index; + ThrowOnError(GetApi().ValueInfo_GetValueProducer(this->p_, &producer, &index)); + info.node = ConstNode(producer); + info.index = static_cast(index); + return info; +} + +template +inline std::vector ConstValueInfoImpl::GetConsumers() const { + size_t num = 0; + ThrowOnError(GetApi().ValueInfo_GetValueNumConsumers(this->p_, &num)); + std::vector out; + if (num > 0) { + std::vector nodes(num); + std::vector indices(num); + ThrowOnError(GetApi().ValueInfo_GetValueConsumers(this->p_, nodes.data(), indices.data(), num)); + out.reserve(num); + for (size_t i = 0; i < num; ++i) { + out.push_back({ConstNode{nodes[i]}, indices[i]}); + } + } + return out; +} + +template +inline Status ConstValueInfoImpl::GetInitializer(ConstValue& value) const { + const OrtValue* out = nullptr; + auto status = Status(GetApi().ValueInfo_GetInitializerValue(this->p_, &out)); + if (!status.IsOK()) return status; + value = ConstValue{out}; + return status; +} + +template +inline Status ConstValueInfoImpl::GetExternalInitializerInfo(ExternalInitializerInfo& info) const { + OrtExternalInitializerInfo* out = nullptr; + auto status = Status(GetApi().ValueInfo_GetExternalInitializerInfo(this->p_, &out)); + if (!status.IsOK()) return status; + info = ExternalInitializerInfo{out}; + return status; +} + +template +inline bool ConstValueInfoImpl::IsRequiredGraphInput() const { + bool out = false; + ThrowOnError(GetApi().ValueInfo_IsRequiredGraphInput(this->p_, &out)); + return out; +} + +template +inline bool ConstValueInfoImpl::IsOptionalGraphInput() const { + bool out = false; + ThrowOnError(GetApi().ValueInfo_IsOptionalGraphInput(this->p_, &out)); + return out; +} + +template +inline bool ConstValueInfoImpl::IsGraphOutput() const { + bool out = false; + ThrowOnError(GetApi().ValueInfo_IsGraphOutput(this->p_, &out)); + return out; +} + +template +inline bool ConstValueInfoImpl::IsConstantInitializer() const { + bool out = false; + ThrowOnError(GetApi().ValueInfo_IsConstantInitializer(this->p_, &out)); + return out; +} + +template +inline bool ConstValueInfoImpl::IsFromOuterScope() const { + bool out = false; + ThrowOnError(GetApi().ValueInfo_IsFromOuterScope(this->p_, &out)); + return out; +} + +template +inline ModelMetadata ConstGraphImpl::GetModelMetadata() const { + OrtModelMetadata* out; + ThrowOnError(GetApi().Graph_GetModelMetadata(this->p_, &out)); + return ModelMetadata{out}; +} + +template +inline std::string ConstGraphImpl::GetName() const { + const char* name; + ThrowOnError(GetApi().Graph_GetName(this->p_, &name)); + return std::string(name); +} + +template +inline std::basic_string ConstGraphImpl::GetModelPath() const { + const ORTCHAR_T* path; + ThrowOnError(GetApi().Graph_GetModelPath(this->p_, &path)); + return std::basic_string(path); +} + +template +inline int64_t ConstGraphImpl::GetOnnxIRVersion() const { + int64_t version; + ThrowOnError(GetApi().Graph_GetOnnxIRVersion(this->p_, &version)); + return version; +} + +template +inline std::vector ConstGraphImpl::GetOperatorSets() const { + size_t num_opsets; + ThrowOnError(GetApi().Graph_GetNumOperatorSets(this->p_, &num_opsets)); + std::vector result; + if (num_opsets > 0) { + std::vector domains; + std::vector versions; + domains.resize(num_opsets); + versions.resize(num_opsets); + ThrowOnError(GetApi().Graph_GetOperatorSets(this->p_, domains.data(), versions.data(), num_opsets)); + result.reserve(num_opsets); + for (size_t i = 0; i < num_opsets; ++i) { + result.push_back({domains[i], versions[i]}); + } + } + return result; +} + +template +inline std::vector ConstGraphImpl::GetInputs() const { + static_assert(sizeof(const OrtValueInfo*) == sizeof(ConstValueInfo)); + size_t num_vi; + ThrowOnError(GetApi().Graph_GetNumInputs(this->p_, &num_vi)); + std::vector result; + if (num_vi > 0) { + result.resize(num_vi); + ThrowOnError(GetApi().Graph_GetInputs(this->p_, reinterpret_cast(result.data()), num_vi)); + } + return result; +} + +template +inline std::vector ConstGraphImpl::GetOutputs() const { + static_assert(sizeof(const OrtValueInfo*) == sizeof(ConstValueInfo)); + size_t num_vi; + ThrowOnError(GetApi().Graph_GetNumOutputs(this->p_, &num_vi)); + std::vector result; + if (num_vi > 0) { + result.resize(num_vi); + ThrowOnError(GetApi().Graph_GetOutputs(this->p_, reinterpret_cast(result.data()), num_vi)); + } + return result; +} + +template +inline std::vector ConstGraphImpl::GetInitializers() const { + static_assert(sizeof(const OrtValueInfo*) == sizeof(ConstValueInfo)); + size_t num_vi; + ThrowOnError(GetApi().Graph_GetNumInitializers(this->p_, &num_vi)); + std::vector result; + if (num_vi > 0) { + result.resize(num_vi); + ThrowOnError(GetApi().Graph_GetInitializers(this->p_, reinterpret_cast(result.data()), + num_vi)); + } + return result; +} + +template +inline std::vector ConstGraphImpl::GetNodes() const { + static_assert(sizeof(const OrtNode*) == sizeof(ConstNode)); + size_t num_nodes; + ThrowOnError(GetApi().Graph_GetNumNodes(this->p_, &num_nodes)); + std::vector result; + if (num_nodes > 0) { + result.resize(num_nodes); + ThrowOnError(GetApi().Graph_GetNodes(this->p_, reinterpret_cast(result.data()), num_nodes)); + } + return result; +} + +template +inline ConstNode ConstGraphImpl::GetParentNode() const { + const OrtNode* parent; + ThrowOnError(GetApi().Graph_GetParentNode(this->p_, &parent)); + return ConstNode{parent}; +} + +template +inline Graph ConstGraphImpl::GetGraphView(const std::vector& nodes) const { + OrtGraph* graph_viewer; + std::vector inputs_ptrs; + inputs_ptrs.reserve(nodes.size()); + std::transform(nodes.begin(), nodes.end(), std::back_inserter(inputs_ptrs), + [](ConstNode n) -> const OrtNode* { return n; }); + ThrowOnError(GetApi().Graph_GetGraphView(this->p_, inputs_ptrs.data(), + nodes.size(), &graph_viewer)); + return Graph{graph_viewer}; +} + +#if !defined(ORT_MINIMAL_BUILD) +template +inline void GraphImpl::SetInputs(std::vector& inputs) { + std::vector inputs_ptrs; + inputs_ptrs.reserve(inputs.size()); + std::transform(inputs.begin(), inputs.end(), std::back_inserter(inputs_ptrs), + [](ValueInfo& vi) -> OrtValueInfo* { return vi; }); + + ThrowOnError(GetModelEditorApi().SetGraphInputs(this->p_, inputs_ptrs.data(), inputs_ptrs.size())); + + // Graph now owns the inputs + std::for_each(inputs.begin(), inputs.end(), [](ValueInfo& vi) { vi.release(); }); +} + +template +inline void GraphImpl::SetOutputs(std::vector& outputs) { + std::vector outputs_ptrs; + outputs_ptrs.reserve(outputs.size()); + std::transform(outputs.begin(), outputs.end(), std::back_inserter(outputs_ptrs), + [](ValueInfo& vi) -> OrtValueInfo* { return vi; }); + + ThrowOnError(GetModelEditorApi().SetGraphOutputs(this->p_, outputs_ptrs.data(), outputs_ptrs.size())); + + // Graph now owns the outputs + std::for_each(outputs.begin(), outputs.end(), [](ValueInfo& vi) { vi.release(); }); +} + +template +inline void GraphImpl::AddInitializer(const std::string& name, Value& initializer, bool data_is_external) { + // Graph takes ownership of `initializer` + // On error the ownership is not transferred. + ThrowOnError(GetModelEditorApi().AddInitializerToGraph(this->p_, name.c_str(), initializer, data_is_external)); + initializer.release(); +} + +template +inline void GraphImpl::AddNode(Node& node) { + // Graph takes ownership of `node` + ThrowOnError(GetModelEditorApi().AddNodeToGraph(this->p_, node.release())); +} + +template +inline void ModelImpl::AddGraph(Graph& graph) { + // Model takes ownership of `graph` + ThrowOnError(GetModelEditorApi().AddGraphToModel(this->p_, graph.release())); +} +#endif // !defined(ORT_MINIMAL_BUILD) + +} // namespace detail + +#if !defined(ORT_MINIMAL_BUILD) +inline Graph::Graph() { + ThrowOnError(GetModelEditorApi().CreateGraph(&p_)); +} + +inline Model::Model(const std::vector& opsets) { + std::vector domains; + std::vector versions; + domains.reserve(opsets.size()); + versions.reserve(opsets.size()); + + for (const auto& pair : opsets) { + domains.push_back(pair.first.c_str()); + versions.push_back(pair.second); + } + + ThrowOnError(GetModelEditorApi().CreateModel(domains.data(), versions.data(), opsets.size(), &p_)); +} +#endif + +namespace detail { +template +inline const char* ConstKernelDefImpl::GetOperatorType() const { + return GetEpApi().KernelDef_GetOperatorType(this->p_); +} + +template +inline const char* ConstKernelDefImpl::GetDomain() const { + return GetEpApi().KernelDef_GetDomain(this->p_); +} + +template +inline std::pair ConstKernelDefImpl::GetSinceVersion() const { + int start = 0; + int end = 0; + + ThrowOnError(GetEpApi().KernelDef_GetSinceVersion(this->p_, &start, &end)); + return std::pair(start, end); +} + +template +inline const char* ConstKernelDefImpl::GetExecutionProvider() const { + return GetEpApi().KernelDef_GetExecutionProvider(this->p_); +} + +template +inline OrtMemType ConstKernelDefImpl::GetInputMemType(size_t input_index) const { + OrtMemType mem_type{}; + ThrowOnError(GetEpApi().KernelDef_GetInputMemType(this->p_, input_index, &mem_type)); + + return mem_type; +} + +template +inline OrtMemType ConstKernelDefImpl::GetOutputMemType(size_t output_index) const { + OrtMemType mem_type{}; + ThrowOnError(GetEpApi().KernelDef_GetOutputMemType(this->p_, output_index, &mem_type)); + + return mem_type; +} +} // namespace detail + +inline KernelDefBuilder::KernelDefBuilder() { + ThrowOnError(GetEpApi().CreateKernelDefBuilder(&p_)); +} + +inline KernelDefBuilder::KernelDefBuilder(OrtKernelDefBuilder* p) : detail::Base{p} { +} + +inline KernelDefBuilder& KernelDefBuilder::SetOperatorType(const char* op_type) { + ThrowOnError(GetEpApi().KernelDefBuilder_SetOperatorType(p_, op_type)); + return *this; +} + +inline KernelDefBuilder& KernelDefBuilder::SetDomain(const char* domain) { + ThrowOnError(GetEpApi().KernelDefBuilder_SetDomain(p_, domain)); + return *this; +} + +inline KernelDefBuilder& KernelDefBuilder::SetSinceVersion(int since_version_start, int since_version_end) { + ThrowOnError(GetEpApi().KernelDefBuilder_SetSinceVersion(p_, since_version_start, since_version_end)); + return *this; +} + +inline KernelDefBuilder& KernelDefBuilder::SetExecutionProvider(const char* ep_name) { + ThrowOnError(GetEpApi().KernelDefBuilder_SetExecutionProvider(p_, ep_name)); + return *this; +} + +inline KernelDefBuilder& KernelDefBuilder::SetInputMemType(size_t input_index, OrtMemType mem_type) { + ThrowOnError(GetEpApi().KernelDefBuilder_SetInputMemType(p_, input_index, mem_type)); + return *this; +} + +inline KernelDefBuilder& KernelDefBuilder::SetOutputMemType(size_t output_index, OrtMemType mem_type) { + ThrowOnError(GetEpApi().KernelDefBuilder_SetOutputMemType(p_, output_index, mem_type)); + return *this; +} + +inline KernelDefBuilder& KernelDefBuilder::AddTypeConstraint(const char* arg_name, + const OrtDataType* data_type) { + ThrowOnError(GetEpApi().KernelDefBuilder_AddTypeConstraint(p_, arg_name, &data_type, 1)); + return *this; +} + +inline KernelDefBuilder& KernelDefBuilder::AddTypeConstraint(const char* arg_name, + const std::vector& data_types) { + ThrowOnError(GetEpApi().KernelDefBuilder_AddTypeConstraint(p_, arg_name, data_types.data(), data_types.size())); + return *this; +} + +inline KernelDefBuilder& KernelDefBuilder::AddInputOutputAlias(int input_index, int output_index) { + ThrowOnError(GetEpApi().KernelDefBuilder_AddInputOutputAliases(p_, &input_index, &output_index, 1)); + return *this; +} + +inline KernelDefBuilder& KernelDefBuilder::AddInputOutputAliases(const std::vector& input_indices, + const std::vector& output_indices) { + if (input_indices.size() != output_indices.size()) { + ORT_CXX_API_THROW("Expecting input and output indices to have the same element count", ORT_INVALID_ARGUMENT); + } + + ThrowOnError(GetEpApi().KernelDefBuilder_AddInputOutputAliases(p_, input_indices.data(), output_indices.data(), + input_indices.size())); + return *this; +} + +inline KernelDefBuilder& KernelDefBuilder::AddInputOutputMutableAlias(int input_index, int output_index) { + ThrowOnError(GetEpApi().KernelDefBuilder_AddInputOutputMutableAliases(p_, &input_index, &output_index, 1)); + return *this; +} + +inline KernelDefBuilder& KernelDefBuilder::AddInputOutputMutableAliases(const std::vector& input_indices, + const std::vector& output_indices) { + if (input_indices.size() != output_indices.size()) { + ORT_CXX_API_THROW("Expecting input and output indices to have the same element count", ORT_INVALID_ARGUMENT); + } + + ThrowOnError(GetEpApi().KernelDefBuilder_AddInputOutputMutableAliases(p_, input_indices.data(), output_indices.data(), + input_indices.size())); + return *this; +} + +inline KernelDef KernelDefBuilder::Build() { + OrtKernelDef* kernel_def = nullptr; + ThrowOnError(GetEpApi().KernelDefBuilder_Build(p_, &kernel_def)); + return KernelDef(kernel_def); +} + +inline KernelRegistry::KernelRegistry() { + ThrowOnError(GetEpApi().CreateKernelRegistry(&p_)); +} + +inline KernelRegistry::KernelRegistry(OrtKernelRegistry* p) : detail::Base{p} { +} + +inline Status KernelRegistry::AddKernel(const OrtKernelDef* kernel_def, OrtKernelCreateFunc kernel_create_func, + void* kernel_create_func_state) { + return Status{GetEpApi().KernelRegistry_AddKernel(p_, kernel_def, kernel_create_func, kernel_create_func_state)}; +} + +namespace detail { +template +inline Status SharedPrePackedWeightCacheImpl::StoreWeightData(void** buffer_data_ptrs, size_t* buffer_sizes, + size_t num_buffers) { + return Status{GetEpApi().SharedPrePackedWeightCache_StoreWeightData(this->p_, buffer_data_ptrs, buffer_sizes, + num_buffers)}; +} +} // namespace detail + +inline Ort::KeyValuePairs GetEnvConfigEntries() { + OrtKeyValuePairs* entries = nullptr; + Ort::ThrowOnError(GetEpApi().GetEnvConfigEntries(&entries)); + + return Ort::KeyValuePairs{entries}; +} +} // namespace Ort diff --git a/dist/include/onnxruntime/onnxruntime_env_config_keys.h b/dist/include/onnxruntime/onnxruntime_env_config_keys.h new file mode 100644 index 0000000..b603765 --- /dev/null +++ b/dist/include/onnxruntime/onnxruntime_env_config_keys.h @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +// This file contains well-known keys for OrtEnv configuration entries, which may be used to configure EPs or +// other global settings. +// Refer to OrtEnvCreationOptions::config_entries and OrtApi::CreateEnvWithOptions. +// This file does NOT specify all available keys as EPs may accept custom entries with the prefix "ep..". + +// Key for a boolean option that, when enabled, allows EP factories to create virtual OrtHardwareDevice +// instances via OrtEpApi::CreateHardwareDevice(). +// +// This config entry is automatically set to "1" by ORT if an application registers an EP library with a registration +// name that ends in the suffix ".virtual". See OrtApi::RegisterExecutionProviderLibrary(). +// +// Note: A virtual OrtHardwareDevice does not represent actual hardware on the device, and is identified via the +// metadata entry "is_virtual" with a value of "1". +// +// Allowed values: +// - "0": Default. Creation of virtual devices is not allowed. +// This is the assumed default value if this key is not present in the environment's configuration entries. +// - "1": Creation of virtual devices is allowed. +static const char* const kOrtEnvAllowVirtualDevices = "allow_virtual_devices"; diff --git a/dist/include/onnxruntime/onnxruntime_ep_c_api.h b/dist/include/onnxruntime/onnxruntime_ep_c_api.h new file mode 100644 index 0000000..b888d0d --- /dev/null +++ b/dist/include/onnxruntime/onnxruntime_ep_c_api.h @@ -0,0 +1,2112 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// Do not include this file directly. Please include "onnxruntime_c_api.h" instead. + +#if defined(__DOXYGEN__) +// When running a Doxygen build, include onnxruntime_c_api.h. Doxygen expects header files to be self-contained. +#include "onnxruntime_c_api.h" +#else +// In normal usage, do not include onnxruntime_c_api.h. This file is explicitly included in onnxruntime_c_api.h. +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +ORT_RUNTIME_CLASS(Ep); +ORT_RUNTIME_CLASS(EpFactory); +ORT_RUNTIME_CLASS(EpGraphSupportInfo); +ORT_RUNTIME_CLASS(MemoryDevice); // opaque class to wrap onnxruntime::OrtDevice +ORT_RUNTIME_CLASS(NodeComputeContext); + +ORT_RUNTIME_CLASS(DataTransferImpl); +ORT_RUNTIME_CLASS(SyncNotificationImpl); +ORT_RUNTIME_CLASS(SyncStreamImpl); + +ORT_RUNTIME_CLASS(ExternalResourceImporterImpl); + +/** \brief Base struct for imported external memory handles. + * + * EPs derive from this struct to add EP-specific fields (e.g., CUdeviceptr for CUDA). + * EP is responsible for creating and releasing instances of the derived type. + * + * Example derived type for CUDA EP: + * \code + * struct MyCudaExternalMemoryHandle : OrtExternalMemoryHandle { + * CUexternalMemory ext_memory; + * CUdeviceptr mapped_ptr; + * bool is_dedicated; + * }; + * \endcode + * + * \since Version 1.24. + */ +struct OrtExternalMemoryHandle { + uint32_t version; ///< Must be ORT_API_VERSION + const OrtEpDevice* ep_device; ///< EP device that created this handle + OrtExternalMemoryDescriptor descriptor; ///< External memory descriptor + + /** \brief Release callback for this handle. EP sets this to its release function. + * + * ORT calls this when ReleaseExternalMemoryHandle is invoked. The EP's callback + * should cast the handle to its derived type and delete it. + */ + void(ORT_API_CALL* Release)(_In_ OrtExternalMemoryHandle* handle); +}; + +/** \brief Base struct for imported external semaphore handles. + * + * EPs derive from this struct to add EP-specific fields (e.g., CUexternalSemaphore for CUDA). + * EP is responsible for creating and releasing instances of the derived type. + * + * Example derived type for CUDA EP: + * \code + * struct MyCudaExternalSemaphoreHandle : OrtExternalSemaphoreHandle { + * CUexternalSemaphore ext_semaphore; + * }; + * \endcode + * + * \since Version 1.24. + */ +struct OrtExternalSemaphoreHandle { + uint32_t version; ///< Must be ORT_API_VERSION + const OrtEpDevice* ep_device; ///< EP device that created this handle + OrtExternalSemaphoreDescriptor descriptor; ///< External semaphore descriptor + + /** \brief Release callback for this handle. EP sets this to its release function. + * + * ORT calls this when ReleaseExternalSemaphoreHandle is invoked. The EP's callback + * should cast the handle to its derived type and delete it. + */ + void(ORT_API_CALL* Release)(_In_ OrtExternalSemaphoreHandle* handle); +}; + +// Opaque types for kernel-based EPs +ORT_RUNTIME_CLASS(KernelRegistry); +ORT_RUNTIME_CLASS(KernelDefBuilder); +ORT_RUNTIME_CLASS(KernelDef); +ORT_RUNTIME_CLASS(DataType); // combination of ONNXType (e.g., Tensor, Map, Sequence) and ONNXTensorElementDataType +ORT_RUNTIME_CLASS(SharedPrePackedWeightCache); + +/** \brief Struct that an EP implements for IDataTransfer to copy between devices it uses and CPU. + * + * \since Version 1.23. + */ +struct OrtDataTransferImpl { + uint32_t ort_version_supported; ///< Must be initialized to ORT_API_VERSION + + /** \brief Release the OrtDataTransferImpl instance. + * + * This is called by ORT when the OrtDataTransferImpl instance is no longer needed. + * The implementation should release any resources held by the instance. + * + * \param[in] this_ptr Pointer to the OrtDataTransferImpl instance. + * + * \since Version 1.23. + */ + ORT_API_T(void, Release, _In_ OrtDataTransferImpl* this_ptr); + + /** \brief Check if the implementation can copy between the source and destination memory devices. + * + * \param[in] this_ptr Pointer to the OrtDataTransferImpl instance. + * \param[in] src_memory_device Source OrtMemoryDevice to copy from. + * \param[in] dst_memory_device Destination OrtMemoryDevice to copy to. + * \return True if the implementation can copy between the devices. + * + * \since Version 1.23. + */ + ORT_API_T(bool, CanCopy, _In_ const OrtDataTransferImpl* this_ptr, + _In_ const OrtMemoryDevice* src_memory_device, _In_ const OrtMemoryDevice* dst_memory_device); + + /** \brief Copy tensors from src_tensors to dst_tensors using the provided streams. + * + * The implementation can use the provided streams to perform asynchronous copies if supported. + * If a stream is not available, the copy is performed synchronously. + * + * \param[in] this_ptr Pointer to the OrtDataTransferImpl instance. + * \param[in] src_tensors Array of source OrtValue pointers to copy from. + * \param[in] dst_tensors Array of destination OrtValue pointers to copy to. + * \param[in] streams Array of OrtSyncStream pointers for the copy operations, if the execution provider is stream + * aware. nullptr if it is not. + * \param[in] num_tensors Number of tensors to copy. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(CopyTensors, _In_ OrtDataTransferImpl* this_ptr, + _In_reads_(num_tensors) const OrtValue** src_tensors, + _In_reads_(num_tensors) OrtValue** dst_tensors, + _In_reads_(num_tensors) OrtSyncStream** streams, + _In_ size_t num_tensors); +}; + +/** \brief Struct that an EP implements for Stream Notifications. + * + * \since Version 1.23. + */ +struct OrtSyncNotificationImpl { + uint32_t ort_version_supported; ///< Must be initialized to ORT_API_VERSION + + /** \brief Release the OrtSyncNotificationImpl instance. + * + * This is called by ORT when the OrtSyncNotificationImpl instance is no longer needed. + * The implementation should release any resources held by the instance. + * + * \param[in] this_ptr Pointer to the OrtSyncNotificationImpl instance. + * + * \since Version 1.23. + */ + ORT_API_T(void, Release, _In_ OrtSyncNotificationImpl* this_ptr); + + /** \brief Called by ORT to activate the notification. + * + * \param[in] this_ptr Pointer to the OrtSyncNotificationImpl instance. + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Activate, _In_ OrtSyncNotificationImpl* this_ptr); + + /** \brief Wait for a device to device operation to complete. + * + * \param[in] this_ptr Pointer to the OrtSyncNotificationImpl instance. + * \param[in] consumer_stream The OrtSyncStream instance that will wait on this notification to be activated. + * + * \since Version 1.23. + */ + ORT_API2_STATUS(WaitOnDevice, _In_ OrtSyncNotificationImpl* this_ptr, _In_ OrtSyncStream* consumer_stream); + + /** \brief Wait for a device to host operation to complete. + * + * \param[in] this_ptr Pointer to the OrtSyncNotificationImpl instance. + * + * \since Version 1.23. + */ + ORT_API2_STATUS(WaitOnHost, _In_ OrtSyncNotificationImpl* this_ptr); +}; + +/** \brief Struct that an EP implements if it wishes to implement Stream support. + * + * This struct provides the overrides for onnxruntime::Stream's virtual methods. + * + * \since Version 1.23. + */ +struct OrtSyncStreamImpl { + uint32_t ort_version_supported; ///< Must be initialized to ORT_API_VERSION + + /** \brief Release the OrtSyncStreamImpl instance. + * + * This is called by ORT when the OrtSyncStreamImpl instance is no longer needed. + * The implementation should release any resources held by the instance. + * + * \param[in] this_ptr Pointer to the OrtSyncStreamImpl instance. + * + * \since Version 1.23. + */ + ORT_API_T(void, Release, _In_ OrtSyncStreamImpl* this_ptr); + + /** \brief Get the handle of the stream. + * + * This returns the native handle for the stream. e.g. cudaStream_t for CUDA streams. + * + * \param[in] this_ptr Pointer to the OrtSyncStreamImpl instance. + * \return The handle of the stream. + * + * \since Version 1.23. + */ + ORT_API_T(void*, GetHandle, _In_ OrtSyncStreamImpl* this_ptr); + + /** \brief Create an OrtSyncNotificationImpl for the OrtSyncStreamImpl instance. + * + * \param[in] this_ptr Pointer to the OrtSyncStreamImpl instance + * \param[out] notification The new OrtSyncNotificationImpl instance. + * + * \since Version 1.23. + */ + ORT_API2_STATUS(CreateNotification, _In_ OrtSyncStreamImpl* this_ptr, + _Outptr_ OrtSyncNotificationImpl** notification); + + /** \brief Flush the stream. + * + * This is called by ORT to flush the stream, ensuring that all operations submitted to the stream are completed. + * + * \param[in] this_ptr Pointer to the OrtSyncStreamImpl instance. + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Flush, _In_ OrtSyncStreamImpl* this_ptr); + + /** \brief Notify the stream that a session run has ended. + * + * This is called by ORT to notify the stream that a session run has ended, allowing the stream to perform any + * necessary cleanup or finalization. + * + * \param[in] this_ptr Pointer to the OrtSyncStreamImpl instance. + * + * \since Version 1.23. + */ + ORT_API2_STATUS(OnSessionRunEnd, _In_ OrtSyncStreamImpl* this_ptr); +}; + +/** \brief Struct that an EP implements for external resource import (memory + semaphore import). + * + * This capability object provides methods for importing external GPU memory and semaphores + * for zero-copy import. EPs that support D3D12, CUDA, HIP, or Vulkan external resource APIs + * can implement this interface. + * + * \since Version 1.24. + */ +struct OrtExternalResourceImporterImpl { + uint32_t ort_version_supported; ///< Must be initialized to ORT_API_VERSION + + // Memory operations (stream-independent) + + /** \brief Check if the implementation can import external memory of the given handle type. + * + * \param[in] this_ptr Pointer to the OrtExternalResourceImporterImpl instance. + * \param[in] handle_type The type of external memory handle to check. + * \return True if the handle type is supported. + * + * \since Version 1.24. + */ + ORT_API_T(bool, CanImportMemory, + _In_ const OrtExternalResourceImporterImpl* this_ptr, + _In_ OrtExternalMemoryHandleType handle_type); + + /** \brief Import external memory. + * + * The EP creates a derived type of OrtExternalMemoryHandle and returns a pointer to the base. + * EP is responsible for the lifetime of the handle (release via ReleaseMemory). + * + * \param[in] this_ptr Pointer to the OrtExternalResourceImporterImpl instance. + * \param[in] desc Descriptor containing the external memory handle and properties. + * \param[out] out_handle Output parameter set to the created OrtExternalMemoryHandle (EP's derived type). + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(ImportMemory, + _In_ OrtExternalResourceImporterImpl* this_ptr, + _In_ const OrtExternalMemoryDescriptor* desc, + _Outptr_ OrtExternalMemoryHandle** out_handle); + + /** \brief Release an imported external memory handle. + * + * The EP deletes its derived type instance. + * + * \param[in] this_ptr Pointer to the OrtExternalResourceImporterImpl instance. + * \param[in] handle The OrtExternalMemoryHandle to release (EP casts to its derived type). + * + * \since Version 1.24. + */ + ORT_API_T(void, ReleaseMemory, + _In_ OrtExternalResourceImporterImpl* this_ptr, + _In_ OrtExternalMemoryHandle* handle); + + /** \brief Create a tensor backed by imported external memory. + * + * The created tensor is a view over the imported memory and does not copy data. + * + * \param[in] this_ptr Pointer to the OrtExternalResourceImporterImpl instance. + * \param[in] mem_handle The imported external memory handle (EP casts to its derived type). + * \param[in] tensor_desc Descriptor specifying tensor element type, shape, and optional offset. + * \param[out] out_tensor Output parameter set to the created OrtValue containing the tensor. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(CreateTensorFromMemory, + _In_ OrtExternalResourceImporterImpl* this_ptr, + _In_ const OrtExternalMemoryHandle* mem_handle, + _In_ const OrtExternalTensorDescriptor* tensor_desc, + _Outptr_ OrtValue** out_tensor); + + // Semaphore operations (require stream) + + /** \brief Check if the implementation can import external semaphores of the given type. + * + * \param[in] this_ptr Pointer to the OrtExternalResourceImporterImpl instance. + * \param[in] type The type of external semaphore to check. + * \return True if the semaphore type is supported. + * + * \since Version 1.24. + */ + ORT_API_T(bool, CanImportSemaphore, + _In_ const OrtExternalResourceImporterImpl* this_ptr, + _In_ OrtExternalSemaphoreType type); + + /** \brief Import an external semaphore. + * + * The EP creates a derived type of OrtExternalSemaphoreHandle and returns a pointer to the base. + * EP is responsible for the lifetime of the handle (release via ReleaseSemaphore). + * + * \param[in] this_ptr Pointer to the OrtExternalResourceImporterImpl instance. + * \param[in] desc Descriptor containing the external semaphore handle and type. + * \param[out] out_handle Output parameter set to the created OrtExternalSemaphoreHandle (EP's derived type). + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(ImportSemaphore, + _In_ OrtExternalResourceImporterImpl* this_ptr, + _In_ const OrtExternalSemaphoreDescriptor* desc, + _Outptr_ OrtExternalSemaphoreHandle** out_handle); + + /** \brief Release an imported external semaphore handle. + * + * The EP deletes its derived type instance. + * + * \param[in] this_ptr Pointer to the OrtExternalResourceImporterImpl instance. + * \param[in] handle The OrtExternalSemaphoreHandle to release (EP casts to its derived type). + * + * \since Version 1.24. + */ + ORT_API_T(void, ReleaseSemaphore, + _In_ OrtExternalResourceImporterImpl* this_ptr, + _In_ OrtExternalSemaphoreHandle* handle); + + /** \brief Wait on an external semaphore on the EP's stream. + * + * Inserts a wait operation into the EP's stream that blocks until the semaphore + * reaches the specified value. + * + * \param[in] this_ptr Pointer to the OrtExternalResourceImporterImpl instance. + * \param[in] handle The imported external semaphore (EP casts to its derived type). + * \param[in] stream The OrtSyncStream to wait on. + * \param[in] value The fence/semaphore value to wait for. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(WaitSemaphore, + _In_ OrtExternalResourceImporterImpl* this_ptr, + _In_ OrtExternalSemaphoreHandle* handle, + _In_ OrtSyncStream* stream, + _In_ uint64_t value); + + /** \brief Signal an external semaphore from the EP's stream. + * + * Inserts a signal operation into the EP's stream that sets the semaphore + * to the specified value when reached. + * + * \param[in] this_ptr Pointer to the OrtExternalResourceImporterImpl instance. + * \param[in] handle The imported external semaphore (EP casts to its derived type). + * \param[in] stream The OrtSyncStream to signal from. + * \param[in] value The fence/semaphore value to signal. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(SignalSemaphore, + _In_ OrtExternalResourceImporterImpl* this_ptr, + _In_ OrtExternalSemaphoreHandle* handle, + _In_ OrtSyncStream* stream, + _In_ uint64_t value); + + // Release the capability object itself + + /** \brief Release the OrtExternalResourceImporterImpl instance. + * + * This is called by ORT when the OrtExternalResourceImporterImpl instance is no longer needed. + * The implementation should release any resources held by the instance. + * + * \param[in] this_ptr Pointer to the OrtExternalResourceImporterImpl instance. + * + * \since Version 1.24. + */ + ORT_API_T(void, Release, _In_ OrtExternalResourceImporterImpl* this_ptr); +}; + +struct OrtNodeFusionOptions; +typedef struct OrtNodeFusionOptions OrtNodeFusionOptions; + +struct OrtNodeComputeInfo; +typedef struct OrtNodeComputeInfo OrtNodeComputeInfo; + +/** + * \brief The OrtNodeFusionOptions struct specifies options for fusing nodes supported by an execution provider. + * + * Refer to OrtEpApi::EpGraphSupportInfo_AddNodesToFuse. + * + * \since Version 1.23. + */ +struct OrtNodeFusionOptions { + /** \brief The ONNX Runtime version the OrtNodeFusionOptions was compiled with. + * + * Implementation should set to ORT_API_VERSION. + * ORT will use this to ensure it does not use members that were not available when the EP library was compiled. + * + * \since Version 1.23. + */ + uint32_t ort_version_supported; + + /** \brief If set to true, specify that the execution provider does not require ONNX Runtime to provide constant + * initializers as inputs to the fused node during model inference. This is used when the execution + * provider saves a copy of constant initializers, and allows ONNX Runtime to release constant initializers that + * are not used by any execution provider. + * + * If not specified, defaults to false. That is, ONNX Runtime provides constant initializers as inputs to + * the fused node by default. + * + * \since Version 1.23. + */ + bool drop_constant_initializers; + + // const OrtNode* fused_node_schema; +}; + +/** + * \brief The OrtNodeComputeInfo struct provides functions that an OrtEp implements to specify the compute + * function for a compiled OrtGraph instance. + * \since Version 1.23. + */ +struct OrtNodeComputeInfo { + /** \brief The ONNX Runtime version the OrtNodeComputeInfo was compiled with. + * + * Implementation should set to ORT_API_VERSION. + * ORT will use this to ensure it does not call functions that were not available when the EP library was compiled. + * + * \since Version 1.23. + */ + uint32_t ort_version_supported; + + /** \brief Creates an opaque compute state object that is then passed to the Compute() function during inference. + * \param[in] this_ptr The OrtNodeComputeInfo instance. + * \param[in] compute_context OrtNodeComputeContext instance that contains compiled/fused node's name and host + * memory allocation functions. Can optionally be used to build the compute state. + * \param[out] compute_state Output parameter that is assigned the opaque computation state. ONNX Runtime calls + * ReleaseState() (after calling Compute()) to allow the implementer to release the + * compute state. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + OrtStatus*(ORT_API_CALL* CreateState)(_In_ OrtNodeComputeInfo* this_ptr, + _In_ OrtNodeComputeContext* compute_context, + _Outptr_ void** compute_state); + + /** \brief Computation function called to execute the fused node compiled by an OrtEp instance. + * \param[in] this_ptr The OrtNodeComputeInfo instance. + * \param[in] compute_state The opaque computation state returned by CreateState(). + * \param[in] kernel_context The OrtKernelContext instance used to access inputs/outputs. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + OrtStatus*(ORT_API_CALL* Compute)(_In_ OrtNodeComputeInfo* this_ptr, _In_ void* compute_state, + _In_ OrtKernelContext* kernel_context); + + /** \brief Releases the compute state returned by CreateState(). + * \param[in] this_ptr The OrtNodeComputeInfo instance. + * \param[inout] compute_state The opaque compute state returned by CreateState(). + * + * \since Version 1.23. + */ + void(ORT_API_CALL* ReleaseState)(_In_ OrtNodeComputeInfo* this_ptr, _Frees_ptr_opt_ void* compute_state); +}; + +struct OrtKernelImpl; +typedef struct OrtKernelImpl OrtKernelImpl; + +/** + * \brief Contains functions that an OrtEp implements to specify the computation for an operator kernel. + * \since Version 1.24. + */ +struct OrtKernelImpl { + uint32_t ort_version_supported; ///< Must be initialized to ORT_API_VERSION + uint32_t flags; ///< EP must initialize to 0. Used internally by ORT. + + /** \brief Computation function called to execute the kernel on an EP. + * + * \note Implementation of this function is required. + * + * \param[in] this_ptr The OrtKernelImpl instance. + * \param[in] context The OrtKernelContext instance that provides access to the inputs and outputs. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(Compute, _In_ OrtKernelImpl* this_ptr, _In_ OrtKernelContext* context); + + /** \brief Called by ORT to release the OrtKernelImpl instance and its resources. + * + * \note Implementation of this function is required. + * + * \param[in] this_ptr The OrtKernelImpl instance. + * + * \since Version 1.24. + */ + ORT_API_T(void, Release, _In_ OrtKernelImpl* this_ptr); + + /** \brief Optional function to pre-pack a constant tensor (i.e., a weight) to the kernel's preferred data layout. + * + * For example, a Conv kernel can define this function to pack input W to the channel-last data layout + * before inference. + * + * Pre-packing can operate in three different modes: no pre-packing mode, sharing mode, and non-sharing mode. + * 1) No pre-packing mode: The kernel can forgo any weight pre-packing for the given `input_index` by setting + * `is_packed` to false and returning a successful OrtStatus. In this mode, the kernel's + * OrtKernelImpl::SetSharedPrePackedWeight() function is not called for that specific + * `input_index`. + * 2) Sharing mode: Sharing is allowed if the `prepacked_weight_cache` argument is not NULL and the EP stores + * weight data in CPU-accessible memory. In this case, the kernel can optionally choose + * to share the packed weight with other kernels that use the same weight + * (compared by content hash). To do so, the kernel must allocate the packed weight with the + * provided `allocator`, then it stores the packed weight data into `prepacked_weight_cache` + * via SharedPrePackedWeightCache_StoreWeightData(), sets `is_packed` to true, and returns a + * successful OrtStatus. ORT will subsequently call OrtKernelImpl::SetSharedPrePackedWeight() + * to provide this kernel with the actual shared weight data, whose memory location could + * differ (i.e., if shared data was allocated by a previously processed kernel). + * 3) Non-sharing mode: In non-sharing mode, the `prepacked_weight_cache` argument is ignored. In this mode, + * the implementation allocates the packed data with the provided `allocator`, sets + * `is_packed` to true, and returns a successful OrtStatus. The kernel is ultimately + * responsible for releasing the packed data for the weight with `allocator`. + * ORT may release the original (unpacked) weight, which must not be accessed in + * OrtKernelImpl::Compute(). Note that in this mode, the kernel's + * OrtKernelImpl::SetSharedPrePackedWeight() function is not called by ORT for that specific + * `input_index`. + * + * \note This function is based on the internal OpKernel::PrePack() virtual function used within ORT. + * + * \param[in] this_ptr The OrtKernelImpl instance. + * \param[in] tensor The OrtValue instance representing the constant tensor (weight). Do not cache in the kernel. + * \param[in] input_index The input index of the tensor in this kernel. + * \param[in] allocator Allocator for allocating the pre-packed data. Its use is required in sharing mode and + * recommended, but not required, in the non-sharing mode. This will be an allocator set by + * the application for the session/environment (e.g., via CreateAndRegisterAllocator[V2] + * or RegisterAllocator), or an allocator on the OrtEpDevice (read-only or default) otherwise. + * The allocator remains valid throughout the lifetime of the OrtKernelImpl instance. + * \param[in] prepacked_weight_cache May be NULL. If not NULL, the kernel may choose to share a packed weight by + * first storing it in the OrtSharedPrePackedWeightCache instance and then + * receiving the actual shared weight data in the call to + * OrtKernelImpl::SetSharedPrePackedWeight(). See the above description for + * "sharing mode". + * \param[out] is_packed Output parameter that the implementation sets to true if the kernel packed the tensor data. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \note Implementation of this function is optional. If not implemented (set to NULL), ORT assumes the kernel + * does not pre-pack weight data (i.e., `is_packed` defaults to false). + * + * \since Version 1.24. + */ + ORT_API2_STATUS(PrePackWeight, _In_ OrtKernelImpl* this_ptr, _In_ const OrtValue* tensor, + _In_ int input_index, _Inout_ OrtAllocator* allocator, + _In_opt_ OrtSharedPrePackedWeightCache* prepacked_weight_cache, _Out_ bool* is_packed); + + /** \brief Optional function that receives data for a shared pre-packed weight from ORT. + * + * ORT calls this function after calling OrtKernelImpl::PrePackWeight for a specific `input_index` if: + * - OrtKernelImpl::PrePackWeight set the output parameter `is_packed` to true. + * - OrtKernelImpl::PrePackWeight stored weight data to share into the provided OrtSharedPrePackedWeightCache + * parameter (`prepacked_weight_cache`) via the API SharedPrePackedWeightCache_StoreWeightData. + * + * Refer to the description of the "sharing-mode" in the documentation for OrtKernelImpl::PrePackWeight(). + * + * \note ORT will not call this function for an `input_index` that a previous call to + * OrtKernelImpl::PrePackWeight() did not elect to pre-pack and share. + * + * \note This function is based on the internal OpKernel::UseSharedPrePackedBuffers() virtual function used + * within ORT. + * + * \param[in] this_ptr The OrtKernelImpl instance. + * \param[in] buffer_data_ptrs An array of buffer data pointers that collectively hold the pre-packed data for a + * single shared weight. The buffers are provided in the same order and with the same + * contents (in a potentially different memory location) as the buffers + * passed into SharedPrePackedWeightCache_StoreWeightData() within the + * OrtKernelImpl::PrePackWeight() call for the same `input_index`. + * \param[in] buffer_data_sizes An array of buffer byte sizes, one per element in `buffer_data_ptrs`. + * \param[in] num_buffers The number of buffers used to store the data for the shared pre-packed weight. + * Specifies the number of elements in the `buffer_data_ptrs` and `buffer_data_sizes` arrays. + * \param[in] input_index The input index of the tensor in this kernel. This index identifies the identity of + * the weight. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \note Implementation of this function is generally optional. It is only required if OrtKernelImpl::PrePack() + * elects to share pre-packed weights. + * + * \since Version 1.24. + */ + ORT_API2_STATUS(SetSharedPrePackedWeight, _In_ OrtKernelImpl* this_ptr, + _In_reads_(num_buffers) const void* const* buffer_data_ptrs, + _In_reads_(num_buffers) const size_t* buffer_data_sizes, + _In_ size_t num_buffers, _In_ int input_index); +}; + +/** \brief Type definition for a function that creates an OrtKernelImpl instance for an operator kernel. + * + * \param[in] kernel_create_func_state Opaque state initially provided by the EP that registered the kernel. + * Refer to OrtEpApi::KernelRegistry_AddKernel(). May be null. + * \param[in] info The OrtKernelInfo instance that provides access to the kernel's input and output characteristics. + * \param[out] kernel_out Output parameter set to the new OrtKernelImpl instance. On success, ownership of this + * OrtKernelImpl instance transfers to ORT, which will call OrtKernelImpl::Release() to + * release the instance when it is no longer used. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ +typedef OrtStatus*(ORT_API_CALL* OrtKernelCreateFunc)(_In_ void* kernel_create_func_state, + _In_ const OrtKernelInfo* info, + _Outptr_result_maybenull_ OrtKernelImpl** kernel_out); + +struct OrtLoopKernelHelper; +typedef struct OrtLoopKernelHelper OrtLoopKernelHelper; + +/** + * \brief Contains helper functions for a Loop OrtKernelImpl created via OrtEpApi::CreateLoopKernel. + * \since Version 1.24. + */ +struct OrtLoopKernelHelper { + uint32_t ort_version_supported; ///< Must be initialized to ORT_API_VERSION + + /** \brief Called by ORT to release the OrtLoopKernelHelper instance and its resources. + * + * \param[in] this_ptr The OrtLoopKernelHelper instance. + * + * \since Version 1.24. + */ + ORT_API_T(void, Release, _In_ OrtLoopKernelHelper* this_ptr); + + /** \brief Helper function that concatenates OrtValue instances from each loop iteration into a single + * pre-allocated output buffer. + * + * \note Implementing this function is required for all Loop opset versions. + * + * \param[in] this_ptr The OrtLoopKernelHelper instance. + * \param[in] stream_handle Optional native stream handle that enables asynchronous operations. May be NULL. + * \param[in] per_iteration_outputs Array of OrtValue instances from each iteration. All OrtValue elements have the + * same shape. + * \param[in] num_per_iteration_outputs The number of OrtValue* elements in the `per_iteration_outputs` array. + * \param[out] output The pre-allocated output buffer. Memory is allocated on the device for the EP running the + * Loop node. + * \param[in] output_size_in_bytes The size in bytes of the `output` buffer. It is guaranteed to be large enough + * to hold the concatenated data of each element in `per_iteration_outputs`. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(ConcatOutput, _In_ OrtLoopKernelHelper* this_ptr, _In_opt_ void* stream_handle, + _In_reads_(num_per_iteration_outputs) const OrtValue* const* per_iteration_outputs, + _In_ size_t num_per_iteration_outputs, _Out_writes_bytes_all_(output_size_in_bytes) void* output, + _In_ size_t output_size_in_bytes); +}; + +struct OrtScanKernelHelper; +typedef struct OrtScanKernelHelper OrtScanKernelHelper; + +/** + * \brief Contains helper functions for a Scan OrtKernelImpl created via OrtEpApi::CreateScanKernel. + * \since Version 1.24. + */ +struct OrtScanKernelHelper { + uint32_t ort_version_supported; ///< Must be initialized to ORT_API_VERSION + + /** \brief Called by ORT to release the OrtScanKernelHelper instance and its resources. + * + * \param[in] this_ptr The OrtScanKernelHelper instance. + * + * \since Version 1.24. + */ + ORT_API_T(void, Release, _In_ OrtScanKernelHelper* this_ptr); + + /** \brief Helper function that transposes an OrtValue instance during execution of a Scan kernel. + * + * \note Called for Scan (opset >= 9) when the 'scan_input_axes' or 'scan_output_axes' attributes contain + * non-zero values. Implementing this function is required for Scan opset versions >= 9. + * + * \param[in] this_ptr The OrtScanKernelHelper instance. + * \param[in] permutation An array of integers that defines how the input tensor's axes should be permuted. + * \param[in] num_permutation_elems The number of integer elements in the `permutation` array. + * \param[in] input The input OrtValue tensor to transpose. + * \param[in] stream An optional OrtSyncStream instance to be used for asynchronous operations. May be NULL. + * \param[out] output The pre-allocated output OrtValue instance into which to store the results of the + * transpose operation. Must not be released as it is owned by ORT. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(Transpose, _In_ OrtScanKernelHelper* this_ptr, + _In_reads_(num_permutation_elems) const size_t* permutation, _In_ size_t num_permutation_elems, + _In_ const OrtValue* input, _In_opt_ OrtSyncStream* stream, _Inout_ OrtValue* output); +}; + +/** + * \brief The OrtEpApi struct provides functions that are relevant to the implementation of an execution provider. + * + * \since Version 1.22. + */ +struct OrtEpApi { + /** \brief Create an OrtEpDevice for the EP and an OrtHardwareDevice. + * \param[in] ep_factory Execution provider factory that is creating the instance. + * \param[in] hardware_device Hardware device that the EP can utilize. + * \param[in] ep_metadata Optional OrtKeyValuePairs instance for execution provider metadata that may be used + * during execution provider selection and passed to CreateEp. + * ep_device will copy this instance and the user should call ReleaseKeyValuePairs. + * \param[in] ep_options Optional OrtKeyValuePairs instance for execution provider options that will be added + * to the Session configuration options if the execution provider is selected. + * ep_device will copy this instance and the user should call ReleaseKeyValuePairs. + * \param ep_device OrtExecutionDevice that is created. + * + * \since Version 1.22. + */ + ORT_API2_STATUS(CreateEpDevice, _In_ OrtEpFactory* ep_factory, + _In_ const OrtHardwareDevice* hardware_device, + _In_opt_ const OrtKeyValuePairs* ep_metadata, + _In_opt_ const OrtKeyValuePairs* ep_options, + _Out_ OrtEpDevice** ep_device); + + ORT_CLASS_RELEASE(EpDevice); + + /** \brief Specify nodes that are supported by an OrtEp and should be fused into one node. + * + * Because the nodes will be fused into one "fused node", there must not exist an unsupported node in + * a path between two of the provided nodes. Otherwise, the graph will become invalid. + * + * This function can be called multiple times. A subsequent call to this function will force the next set of + * nodes to be fused into a different node. + * + * \param[in] graph_support_info OrtEpGraphSupportInfo instance to which to add the supported nodes. + * \param[in] nodes Array of nodes supported by the EP that should be fused/compiled. + * \param[in] num_nodes The number of supported nodes. + * \param[in] node_fusion_options Optional node fusion options. Ignored if set to NULL. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(EpGraphSupportInfo_AddNodesToFuse, _In_ OrtEpGraphSupportInfo* graph_support_info, + _In_reads_(num_nodes) const OrtNode* const* nodes, _In_ size_t num_nodes, + _In_opt_ const OrtNodeFusionOptions* node_fusion_options); + + /** \brief Specify a node that is supported by an OrtEp and should be run with a registered EP kernel. + * + * \param[in] graph_support_info OrtEpGraphSupportInfo instance to which to add the supported node. + * \param[in] node The supported OrtNode instance. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(EpGraphSupportInfo_AddSingleNode, _In_ OrtEpGraphSupportInfo* graph_support_info, + _In_ const OrtNode* node); + + /** \brief Query a OrtNodeComputeContext for the name of the node that encapsulates the compiled/fused node. + * + * Used in OrtNodeComputeInfo::CreateComputeState(). + * + * \param[in] context The OrtNodeComputeContext instance to query. + * \return The node's name. + * + * \note Returned string is owned by ORT and valid only while OrtNodeComputeInfo::CreateComputeState() is called. + * + * \since Version 1.23. + */ + ORT_API_T(const char*, NodeComputeContext_NodeName, _In_ const OrtNodeComputeContext* context); + + /** \brief Register an allocator with the OrtEpDevice. + * + * This allows an EP to provide OrtMemoryInfo for DEFAULT and HOST_ACCESSIBLE memory type as needed. + * The registered values will be used in calls to OrtEpFactory::CreateAllocator to ensure the required allocator/s + * are available for EP usage. + * + * Multiple calls for the same entry type will replace a previous entry. + * + * Available entries: + * - OrtDeviceAllocator with type of OrtDeviceMemoryType_DEFAULT + * - OrtDeviceAllocator with type of OrtDeviceMemoryType_HOST_ACCESSIBLE + * - OrtReadOnlyAllocator with type of OrtDeviceMemoryType_DEFAULT + * - if provided this allocator will only be used to copy initializers to the device the EP uses. + * ORT will use the OrtDeviceAllocator if not provided. + * + * \param[in] ep_device The OrtEpDevice instance to register the OrtMemoryInfo with. + * \param[in] allocator_memory_info The OrtMemoryInfo information for the allocator. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(EpDevice_AddAllocatorInfo, _In_ OrtEpDevice* ep_device, + _In_ const OrtMemoryInfo* allocator_memory_info); + + /** \brief Get the OrtMemoryDevice from an OrtMemoryInfo instance. + * + * This is required for OrtDataTransferImpl (which implements onnxruntime::IDataTransfer) where the OrtMemoryDevice + * is used in the CanCopy and CopyTensors functions. + * + * \param[in] memory_info The OrtMemoryInfo instance to get the memory device from. + * \return The OrtMemoryDevice associated with the OrtMemoryInfo instance. + * + * \since Version 1.23. + */ + ORT_API_T(const OrtMemoryDevice*, MemoryInfo_GetMemoryDevice, _In_ const OrtMemoryInfo* memory_info); + + /** \brief Get the OrtMemoryDevice from an OrtValue instance if it contains a Tensor. + * + * \param[in] value The OrtValue instance to get the memory device from. + * \return Memory device if OrtValue contains a Tensor, nullptr otherwise. + * + * \since Version 1.23. + */ + ORT_API_T(const OrtMemoryDevice*, Value_GetMemoryDevice, _In_ const OrtValue* value); + + /** \brief Compare two OrtMemoryDevice instances for equality. + * + * This is used to check if two memory devices are the same. + * Used to implement DataTransferImpl::CanCopy. + * + * \param[in] a The first OrtMemoryDevice instance to compare. + * \param[in] b The second OrtMemoryDevice instance to compare. + * \return True if the two OrtMemoryDevice instances are equal, false otherwise. + * + * \since Version 1.23. + */ + ORT_API_T(bool, MemoryDevice_AreEqual, _In_ const OrtMemoryDevice* a, _In_ const OrtMemoryDevice* b); + + /** \brief Get the OrtMemoryInfoDeviceType value from an OrtMemoryDevice instance. + * + * \param[in] memory_device OrtMemoryDevice instance. + * \return The OrtMemoryInfoDeviceType value. + * + * \since Version 1.23. + */ + ORT_API_T(OrtMemoryInfoDeviceType, MemoryDevice_GetDeviceType, _In_ const OrtMemoryDevice* memory_device); + + /** \brief Get the OrtDeviceMemoryType value from an OrtMemoryDevice instance. + * + * \param[in] memory_device OrtMemoryDevice instance. + * \return The OrtDeviceMemoryType value. + * + * \since Version 1.23. + */ + ORT_API_T(OrtDeviceMemoryType, MemoryDevice_GetMemoryType, _In_ const OrtMemoryDevice* memory_device); + + /** \brief Get the vendor ID from an OrtMemoryDevice instance. + * + * The vendor ID is used to identify the vendor of the device, and is typically set to the PCI vendor ID. + * + * If the device is not vendor specific (e.g. CPU memory) the vendor ID is set to 0. + * + * \param[in] memory_device OrtMemoryDevice instance. + * \return The vendor ID value. + * + * \since Version 1.23. + */ + ORT_API_T(uint32_t, MemoryDevice_GetVendorId, _In_ const OrtMemoryDevice* memory_device); + + /** \brief Get the device ID from an OrtMemoryDevice instance. + * + * \param[in] memory_device OrtMemoryDevice instance. + * \return The device ID. + * + * \since Version 1.23. + */ + ORT_API_T(uint32_t, MemoryDevice_GetDeviceId, _In_ const OrtMemoryDevice* memory_device); + + /** \brief Get the OrtSyncStreamImpl associated with an OrtSyncStream instance. + * + * This allows an the plugin library to connect its OrtSyncStreamImpl instance with an OrtSyncStream if needed. + * + * \param[in] stream The OrtSyncStream instance to find an OrtSyncStreamImpl for. + * \return The associated OrtSyncStreamImpl if found. nullptr otherwise. + * + * \since Version 1.23. + * + * \remarks There should always be an OrtSyncStreamImpl associated with an OrtSyncStream instance that the EP gets. + */ + ORT_API_T(const OrtSyncStreamImpl*, SyncStream_GetImpl, _In_ const OrtSyncStream* stream); + + /** \brief Get the current sync ID for a stream. + * + * \param[in] stream The OrtSyncStream to get the sync ID for. + * \return Current sync ID. + * + * \since Version 1.23. + */ + ORT_API_T(uint64_t, SyncStream_GetSyncId, _In_ const OrtSyncStream* stream); + + /** \brief Get the sync ID for the last time the consumer_stream waited on the producer_stream. + * + * When two streams are synchronized, the sync id represents the event used in that synchronization. + * + * \param[in] producer_stream The OrtSyncStream that produced the data. + * \param[in] consumer_stream The OrtSyncStream that waited on the producer_stream. + * \return ID for last sync. 0 if no sync has occurred between the two streams. + * + * \since Version 1.23. + */ + ORT_API_T(uint64_t, GetSyncIdForLastWaitOnSyncStream, + _In_ const OrtSyncStream* producer_stream, _In_ const OrtSyncStream* consumer_stream); + + /** \brief Create an OrtHardwareDevice. + * + * \note Called within OrtEpFactory::GetSupportedDevices to create a new hardware device (e.g., virtual). + * + * \param[in] type The hardware device type. + * \param[in] vendor_id The hardware device's vendor identifier. + * \param[in] device_id The hardware device's identifier. + * \param[in] vendor_name The hardware device's vendor name as a null-terminated string. Copied by ORT. + * \param[in] metadata Optional OrtKeyValuePairs instance for hardware device metadata that may be queried by + * applications via OrtApi::GetEpDevices(). + * Refer to onnxruntime_ep_device_ep_metadata_keys.h for common OrtHardwareDevice metadata keys. + * \param[out] hardware_device Output parameter set to the new OrtHardwareDevice instance that is created. + * Must be release with ReleaseHardwareDevice(). + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(CreateHardwareDevice, _In_ OrtHardwareDeviceType type, + _In_ uint32_t vendor_id, + _In_ uint32_t device_id, + _In_ const char* vendor_name, + _In_opt_ const OrtKeyValuePairs* metadata, + _Out_ OrtHardwareDevice** hardware_device); + + ORT_CLASS_RELEASE(HardwareDevice); + + /** \brief Creates an empty kernel registry. A kernel registry contains kernel creation information for + * every operator kernel supported by an EP. + * + * \remarks Refer to OrtEp::GetKernelRegistry, which returns an EP's kernel registry to ORT. + * + * \param[out] kernel_registry Output parameter set to the new OrtKernelRegistry instance. + * Must be released with OrtEpApi::ReleaseKernelRegistry. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(CreateKernelRegistry, _Outptr_ OrtKernelRegistry** kernel_registry); + + ORT_CLASS_RELEASE(KernelRegistry); + + /** \brief Adds kernel creation information for a supported operator kernel to the given kernel registry. + * + * \remarks Refer to OrtEp::GetKernelRegistry, which returns an EP's kernel registry to ORT. + * + * \param[in] kernel_registry The OrtKernelRegistry instance. + * \param[in] kernel_def The kernel definition, which includes operator type, version, EP name, type constraints, etc. + * \param[in] kernel_create_func Function that creates an instance of the operator kernel as a OrtKernelImpl instance. + * \param[in] kernel_create_func_state Custom state passed to the kernel creation function. Can be null. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(KernelRegistry_AddKernel, _In_ OrtKernelRegistry* kernel_registry, + _In_ const OrtKernelDef* kernel_def, _In_ OrtKernelCreateFunc kernel_create_func, + _In_ void* kernel_create_func_state); + + /** \brief Creates a kernel definition builder used to create instances of OrtKernelDef. + * + * \param[out] kernel_def_builder_out Output parameter set to the new OrtKernelDefBuilder instance. + * Must be released with OrtEpApi::ReleaseKernelDefBuilder(). + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(CreateKernelDefBuilder, _Outptr_ OrtKernelDefBuilder** kernel_def_builder_out); + + ORT_CLASS_RELEASE(KernelDefBuilder); + + /** \brief Sets the kernel's operator type. + * + * \param[in] kernel_def_builder The OrtKernelDefBuilder instance. + * \param[in] op_type A null-terminated string representing the operator type. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(KernelDefBuilder_SetOperatorType, _In_ OrtKernelDefBuilder* kernel_def_builder, + _In_ const char* op_type); + + /** \brief Sets the kernel's domain. + * + * \param[in] kernel_def_builder The OrtKernelDefBuilder instance. + * \param[in] domain A null-terminated string representing the operator's domain. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(KernelDefBuilder_SetDomain, _In_ OrtKernelDefBuilder* kernel_def_builder, _In_ const char* domain); + + /** \brief Sets the kernel's opset version range that is supported. + * + * \param[in] kernel_def_builder The OrtKernelDefBuilder instance. + * \param[in] since_version_start The starting opset version that is supported. + * \param[in] since_version_end The ending opset version (inclusive) that is supported. + * Can be set equal to the starting version to indicate that only one + * version is supported. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(KernelDefBuilder_SetSinceVersion, _In_ OrtKernelDefBuilder* kernel_def_builder, + _In_ int since_version_start, _In_ int since_version_end); + + /** \brief Sets the name of the kernel's intended execution provider. + * + * \param[in] kernel_def_builder The OrtKernelDefBuilder instance. + * \param[in] ep_name A null-terminated string representing the execution provider's name. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(KernelDefBuilder_SetExecutionProvider, _In_ OrtKernelDefBuilder* kernel_def_builder, + _In_ const char* ep_name); + + /** \brief Sets the memory type for a kernel input. + * + * \param[in] kernel_def_builder The OrtKernelDefBuilder instance. + * \param[in] input_index The index of the input. + * \param[in] mem_type The input's memory type. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(KernelDefBuilder_SetInputMemType, _In_ OrtKernelDefBuilder* kernel_def_builder, + _In_ size_t input_index, _In_ OrtMemType mem_type); + + /** \brief Sets the memory type for a kernel output. + * + * \param[in] kernel_def_builder The OrtKernelDefBuilder instance. + * \param[in] output_index The index of the output. + * \param[in] mem_type The output's memory type. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(KernelDefBuilder_SetOutputMemType, _In_ OrtKernelDefBuilder* kernel_def_builder, + _In_ size_t output_index, _In_ OrtMemType mem_type); + + /** \brief Adds type constraints for a kernel argument represented as a string (e.g., "T"). + * + * \param[in] kernel_def_builder The OrtKernelDefBuilder instance. + * \param[in] arg_name A null-terminated string representing the argument to constrain (e.g., "T"). + * \param[in] types Array of OrtDataType instances representing allowed types for the argument. + * Must contain `num_types` elements. + * \param[in] num_types The number of OrtDataType elements in the `types` array. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(KernelDefBuilder_AddTypeConstraint, _In_ OrtKernelDefBuilder* kernel_def_builder, + _In_ const char* arg_name, _In_reads_(num_types) const OrtDataType* const* types, + _In_ size_t num_types); + + /** \brief Adds aliases for the given input and output pairs. + * + * \note Used for operators like Identity and Reshape to allow ORT to reuse the input buffer for the output + * without modification. + * + * \param[in] kernel_def_builder The OrtKernelDefBuilder instance. + * \param[in] input_indices Array of input indices. Array must contain `num_io_indices` elements. + * \param[in] output_indices Array of output indices. Each output index is aliased with a corresponding + * input index in `input_indices`. Array must contain `num_io_indices` elements. + * \param[in] num_io_indices The number of input/output index pairs to alias. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(KernelDefBuilder_AddInputOutputAliases, _In_ OrtKernelDefBuilder* kernel_def_builder, + _In_reads_(num_io_indices) int const* input_indices, + _In_reads_(num_io_indices) int const* output_indices, + _In_ size_t num_io_indices); + + /** \brief Adds mutable aliases for the given input and output pairs. + * + * \note Allows ORT to reuse and *modify* an input buffer (in-place) for the output buffer. + * This is also known as "MayInplace" within the ORT codebase. + * + * \param[in] kernel_def_builder The OrtKernelDefBuilder instance. + * \param[in] input_indices Array of input indices. Array must contain `num_io_indices` elements. + * \param[in] output_indices Array of output indices. Each output index is aliased with a corresponding + * input index in `input_indices`. Array must contain `num_io_indices` elements. + * \param[in] num_io_indices The number of input/output index pairs to alias. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(KernelDefBuilder_AddInputOutputMutableAliases, _In_ OrtKernelDefBuilder* kernel_def_builder, + _In_reads_(num_io_indices) int const* input_indices, + _In_reads_(num_io_indices) int const* output_indices, + _In_ size_t num_io_indices); + + /** \brief Creates a OrtKernelDef instance from the given kernel definition builder. + * + * \param[in] kernel_def_builder The OrtKernelDefBuilder instance. + * \param[out] kernel_def_out The new OrtKernelDef instance. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(KernelDefBuilder_Build, _In_ OrtKernelDefBuilder* kernel_def_builder, + _Outptr_ OrtKernelDef** kernel_def_out); + + ORT_CLASS_RELEASE(KernelDef); + + /** \brief Returns the operator type from the kernel definition. + * + * \param[in] kernel_def The OrtKernelDef instance. + * \return A null-terminated string representing the operator type. + * + * \since Version 1.24. + */ + ORT_API_T(const char*, KernelDef_GetOperatorType, _In_ const OrtKernelDef* kernel_def); + + /** \brief Returns the operator's domain from the kernel definition. + * + * \param[in] kernel_def The OrtKernelDef instance. + * \return A null-terminated string representing the operator's domain. + * + * \since Version 1.24. + */ + ORT_API_T(const char*, KernelDef_GetDomain, _In_ const OrtKernelDef* kernel_def); + + /** \brief Gets the kernel's opset version range that is supported. + * + * \param[in] kernel_def The OrtKernelDef instance. + * \param[out] start_version Output parameter set to the starting opset version that is supported. + * \param[out] end_version Output parameter set to the ending opset version (inclusive) that is supported. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(KernelDef_GetSinceVersion, _In_ const OrtKernelDef* kernel_def, + _Out_ int* start_version, _Out_ int* end_version); + + /** \brief Returns the name of the kernel's intended execution provider. + * + * \param[in] kernel_def The OrtKernelDef instance. + * \return A null-terminated string representing the name of the execution provider. + * + * \since Version 1.24. + */ + ORT_API_T(const char*, KernelDef_GetExecutionProvider, _In_ const OrtKernelDef* kernel_def); + + /** \brief Gets the memory type for a kernel input. + * + * \param[in] kernel_def The OrtKernelDef instance. + * \param[in] input_index The index of the input. + * \param[out] mem_type Output parameter set to the input's memory type. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(KernelDef_GetInputMemType, _In_ const OrtKernelDef* kernel_def, + _In_ size_t input_index, _Out_ OrtMemType* mem_type); + + /** \brief Gets the memory type for a kernel output. + * + * \param[in] kernel_def The OrtKernelDef instance. + * \param[in] output_index The index of the output. + * \param[out] mem_type Output parameter set to the output's memory type. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(KernelDef_GetOutputMemType, _In_ const OrtKernelDef* kernel_def, + _In_ size_t output_index, _Out_ OrtMemType* mem_type); + + /** \brief Gets the OrtDataType that represents the data type for a tensor of the given element type. + * + * \param[in] elem_type The tensor's element type. + * \param[out] out Output parameter set to the OrtDataType. Owned by ORT and must not be released. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(GetTensorDataType, _In_ ONNXTensorElementDataType elem_type, + _Outptr_ const OrtDataType** out); + + /** \brief Gets the kernel definition for a given node, if any exists for the calling execution provider. + * + * Used within OrtEp::GetCapability() to get the registered kernel definition for the given node. + * The kernel definition is set to NULL if there is no registered kernel definition for the node + * and execution provider. + * + * \param[in] graph_support_info The OrtEpGraphSupportInfo instance to query. + * \param[in] node The node for which to look up a kernel definition. + * \param[out] out_kernel_def Output parameter set to the OrtKernelDef or NULL. + * Owned by ORT and must not be released. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(EpGraphSupportInfo_LookUpKernel, _In_ OrtEpGraphSupportInfo* graph_support_info, + _In_ const OrtNode* node, _Outptr_result_maybenull_ const OrtKernelDef** out_kernel_def); + + /** \brief Sets one or more data buffers that collectively hold the pre-packed data for a single shared weight. + * + * \note Used within the implementation of OrtKernelImpl::PrePackWeight() when the kernel wants to share pre-packed + * weight data with other kernels. The buffer data MUST be allocated with the OrtAllocator provided to + * OrtKernelImpl::PrePack. + * + * \note Ownership of weight data transfers to the OrtSharedPrePackedWeightCache instance on success. + * If this function returns an error status, the caller retains ownership of the weight data. + * + * \note Subsequent calls with the same OrtSharedPrePackedWeightCache instance release and replace the old data. + * + * \param[in] prepacked_weight_cache The OrtSharedPrePackedWeightCache instance. + * \param[in] buffer_data_ptrs An array of buffer data pointers that collectively hold the pre-packed data for a + * single shared weight. Note that sometimes a single weight may have multiple pre-packed + * buffers and it is up to the kernel implementation to determine how to split the data + * into multiple buffers (if desired). + * \param[in] buffer_data_sizes An array of buffer byte sizes, one per element in `buffer_data_ptrs`. + * \param[in] num_buffers The number of buffers used to store the data for the shared pre-packed weight. + * Specifies the number of elements in the `buffer_data_ptrs` and `buffer_data_sizes` arrays. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(SharedPrePackedWeightCache_StoreWeightData, + _In_ OrtSharedPrePackedWeightCache* prepacked_weight_cache, + _In_reads_(num_buffers) void** buffer_data_ptrs, _In_reads_(num_buffers) size_t* buffer_data_sizes, + _In_ size_t num_buffers); + + /** \brief Get the OrtEp instance to which the node is assigned from the OrtKernelInfo. + * + * \note Used within OrtKernelImpl implementations to obtain a reference to the OrtEp. + * + * \param[in] info The ::OrtKernelInfo instance. + * \param[out] ep Output parameter set to the OrtEp instance associated with the OrtKernelInfo. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.24 + */ + ORT_API2_STATUS(KernelInfo_GetEp, _In_ const OrtKernelInfo* info, _Outptr_ const OrtEp** ep); + + /** \brief Set the details of an OrtDeviceEpIncompatibilityDetails instance. + * + * Used by execution provider factories to set incompatibility details in their + * GetHardwareDeviceIncompatibilityDetails implementation. ORT creates and initializes the object + * before passing it to the EP, so calling this function is optional. The EP uses this function + * to set incompatibility information when the device is not compatible. + * + * \param[in,out] details The OrtDeviceEpIncompatibilityDetails instance to update. + * \param[in] reasons_bitmask Bitmask of OrtDeviceEpIncompatibilityReason values. (0 = no incompatibility). + * \param[in] error_code Optional EP-specific error code (0 = no error). + * \param[in] notes Optional human-readable notes. Can be null. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(DeviceEpIncompatibilityDetails_SetDetails, _Inout_ OrtDeviceEpIncompatibilityDetails* details, + _In_ uint32_t reasons_bitmask, + _In_ int32_t error_code, + _In_opt_z_ const char* notes); + + /** \brief Creates an OrtKernelImpl instance for an If operator. + * + * Control flow operators require access to ORT session internals to orchestrate subgraph operations. + * This function allows an EP to create a properly configured OrtKernelImpl with access to ORT internals that + * the EP can add to its kernel registry. + * + * An EP is required to create an OrtKernelDef that keeps input[0] ('cond') on the CPU (i.e., OrtMemTypeCPUInput) + * as this input is used by CPU logic. The output should remain on the device (i.e., OrtMemTypeDefault), which is + * the default setting, to avoid copying to/from CPU. + * + * Example kernel definition (CXX API): + * Ort::KernelDef kernel_def = Ort::KernelDefBuilder() + * .SetDomain("").SetOperatorType("If").SetSinceVersion(21, 22) + * .SetExecutionProvider("MyEp") + * .SetInputMemType(0, OrtMemTypeCPUInput) // 'cond' on CPU + * .SetOutputMemType(0, OrtMemTypeDefault) // output on EP device + * .AddTypeConstraint("B", ...) + * .AddTypeConstraint("V", ...).Build(); + * + * \param[in] kernel_info The ::OrtKernelInfo instance for an If node. This function returns error ORT_FAIL + * if the opset version specified by `kernel_info` is unsupported. + * \param[out] kernel_out Output parameter set to the OrtKernelImpl instance for the If node. + * Must be released via ::ReleaseKernelImpl, unless ownership is transferred + * to ORT (see OrtKernelCreateFunc and ::KernelRegistry_AddKernel()). + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.24 + */ + ORT_API2_STATUS(CreateIfKernel, _In_ const OrtKernelInfo* kernel_info, _Outptr_ OrtKernelImpl** kernel_out); + + /** \brief Creates an OrtKernelImpl instance for a Loop operator. + * + * Control flow operators require access to ORT session internals to orchestrate subgraph operations. + * This function allows an EP to create a properly configured OrtKernelImpl with access to ORT internals that + * the EP can add to its kernel registry. + * + * An EP is required to create an OrtKernelDef that keeps input[0] ('M') and input[1] ('cond') on the CPU + * (i.e., OrtMemTypeCPUInput) as these inputs are used by CPU logic. Input[2] ('v_initial') and the output should + * remain on the device (i.e., OrtMemTypeDefault), which is the default setting, to avoid copying to/from CPU. + * + * Example kernel definition (CXX API): + * Ort::KernelDef kernel_def = Ort::KernelDefBuilder() + * .SetDomain("").SetOperatorType("Loop").SetSinceVersion(21, 22) + * .SetExecutionProvider("MyEp") + * .SetInputMemType(0, OrtMemTypeCPUInput) // 'M' on CPU + * .SetInputMemType(1, OrtMemTypeCPUInput) // 'cond' on CPU + * .SetInputMemType(2, OrtMemTypeDefault) // 'v_initial' on EP device + * .SetOutputMemType(0, OrtMemTypeDefault) // output on EP device + * .AddTypeConstraint("I", ...) + * .AddTypeConstraint("B", ...) + * .AddTypeConstraint("V", ...).Build(); + * + * \param[in] kernel_info The ::OrtKernelInfo instance for a Loop node. This function returns error ORT_FAIL + * if the opset version specified by `kernel_info` is unsupported. + * \param[in] helper A OrtLoopKernelHelper instance that contains helper functions that ORT calls during kernel + * execution to operate on tensors allocated with the EP's device memory. + * ORT will call OrtLoopKernelHelper::Release() to release the helper and its resources. + * \param[out] kernel_out Output parameter set to the OrtKernelImpl instance for the Loop node. + * Must be released via ::ReleaseKernelImpl, unless ownership is transferred + * to ORT (see OrtKernelCreateFunc and ::KernelRegistry_AddKernel()). + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.24 + */ + ORT_API2_STATUS(CreateLoopKernel, _In_ const OrtKernelInfo* kernel_info, _In_ OrtLoopKernelHelper* helper, + _Outptr_ OrtKernelImpl** kernel_out); + + /** \brief Creates an OrtKernelImpl instance for a Scan operator. Does not support opset versions older than 9. + * + * Control flow operators require access to ORT session internals to orchestrate subgraph operations. + * This function allows an EP to create a properly configured OrtKernelImpl with access to ORT internals that + * the EP can add to its kernel registry. + * + * It is recommended that an EP create an OrtKernelDef that keeps the inputs and outputs on the EP's + * device (i.e., OrtMemTypeDefault), which is the default setting, to avoid copying to/from CPU. + * + * Example kernel definition (CXX API): + * Ort::KernelDef kernel_def = Ort::KernelDefBuilder() + * .SetDomain("").SetOperatorType("Scan").SetSinceVersion(21, 22) + * .SetExecutionProvider("MyEp") + * .SetInputMemType(0, OrtMemTypeDefault) // input[0] on EP device + * .SetOutputMemType(0, OrtMemTypeDefault) // output[0] on EP device + * .AddTypeConstraint("V", ...).Build(); + * + * \param[in] kernel_info The ::OrtKernelInfo instance for a Scan node. This function returns error ORT_FAIL + * if the opset version specified by `kernel_info` is unsupported. + * \param[in] helper A OrtScanKernelHelper instance that contains helper functions that ORT calls during kernel + * execution to operate on tensors allocated with the EP's device memory. + * ORT will call OrtScanKernelHelper::Release() to release the helper and its resources. + * \param[out] kernel_out Output parameter set to the OrtKernelImpl instance for the Scan node. + * Must be released via ::ReleaseKernelImpl, unless ownership is transferred + * to ORT (see OrtKernelCreateFunc and ::KernelRegistry_AddKernel()). + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.24 + */ + ORT_API2_STATUS(CreateScanKernel, _In_ const OrtKernelInfo* kernel_info, _In_ OrtScanKernelHelper* helper, + _Outptr_ OrtKernelImpl** kernel_out); + + ORT_CLASS_RELEASE(KernelImpl); + + /** \brief Gets a new OrtKeyValuePairs instance containing a copy of all configuration entries set on the environment. + * + * \note An application provides environment-level configuration options for execution provider libraries by + * using keys with the prefix 'ep_factory.\\.'. Ex: the key 'ep_factory.my_ep.some_ep_key' represents + * a key named 'some_ep_key' that is meant to be consumed by an execution provider named 'my_ep'. Refer to + * the specific execution provider's documentation for valid keys and values. + * + * \note Refer to onnxruntime_env_config_keys.h for common configuration entry keys and their supported values. + * + * \param[out] config_entries Output parameter set to the OrtKeyValuePairs instance containing all configuration entries. + * Must be released via OrtApi::ReleaseKeyValuePairs. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.24 + */ + ORT_API2_STATUS(GetEnvConfigEntries, _Outptr_ OrtKeyValuePairs** config_entries); +}; + +/** + * \brief The data layout type. + * + * EPs may specify a preferred data layout type. ORT's default layout type is OrtEpDataLayout_NCHW, or + * OrtEpDataLayout_Default. + * + * \since Version 1.23. + */ +typedef enum OrtEpDataLayout { + OrtEpDataLayout_NCHW = 0, + OrtEpDataLayout_NHWC, + + OrtEpDataLayout_Default = OrtEpDataLayout_NCHW, +} OrtEpDataLayout; + +/** + * \brief The OrtEp struct provides functions to implement for an execution provider. + * \since Version 1.22. + */ +struct OrtEp { + /** \brief The ONNX Runtime version the execution provider was compiled with. + * + * Implementation should set to ORT_API_VERSION. + * ORT will use this to ensure it does not call functions that were not available when the library was compiled. + * + * \since Version 1.22. + */ + uint32_t ort_version_supported; + + /** \brief Get the execution provider name. + * + * The returned string should be a null-terminated, UTF-8 encoded string. ORT will copy it. + * + * \param[in] this_ptr The OrtEp instance. + * \return The execution provider name. + * + * \since Version 1.22. + */ + ORT_API_T(const char*, GetName, _In_ const OrtEp* this_ptr); + + /** \brief Get information about the nodes supported by the OrtEp instance. + * + * IMPORTANT: This is not the final version of this API function. This is currently experimental but will + * be stabilized by the ONNX Runtime 1.23 release. + * + * \param[in] this_ptr The OrtEp instance. + * \param[in] graph The OrtGraph instance for which to populate node support. The OrtGraph could be a nested subgraph + * contained by a node (e.g., an If or Loop node). ONNX Runtime calls this function separately + * for each nested subgraph. + * \param[inout] graph_support_info OrtEpGraphSupportInfo instance that the implementer must fill out in order to + * specify the supported nodes. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(GetCapability, _In_ OrtEp* this_ptr, _In_ const OrtGraph* graph, + _Inout_ OrtEpGraphSupportInfo* graph_support_info); + + /** \brief Compile OrtGraph instances assigned to the OrtEp. Implementer must set a OrtNodeComputeInfo instance + * for each OrtGraph in order to define its computation function. + * + * If the session is configured to generate a pre-compiled model, the execution provider must return EPContext nodes, + * as OrtNode instances, that ONNX Runtime uses to create a pre-compiled model, known as an "EPContext model". + * An EPContext model contains EPContext nodes. Each EPContext node encapsulates the pre-compiled binary data for a + * OrtGraph compiled for a specific execution provider. For more details about the EPContext design, refer to: + * \htmlonly + * EPContext design document. + * \endhtmlonly + * + * \param[in] this_ptr The OrtEp instance. + * \param[in] graphs Array of `count` OrtGraph instances to compile. Each graph contains only the nodes for + * which the execution provider indicated support. Nested subgraphs contained by a + * node, such as an If or Loop, have separate OrtGraph instances. + * \param[in] fused_nodes Array of `count` fused nodes that will replace the compiled graphs. + * Each fused node is an OrtNode initialized with the intended fused node name and + * input/output information. + * \param[in] count The number of OrtGraph instances to compile. + * \param[out] node_compute_infos Array of `count` OrtNodeComputeInfo instances that define each OrtGraph instance's + * computation function. The implementer allocates the OrtNodeComputeInfo instances. + * ORT calls ReleaseNodeComputeInfos() to release multiple instances in a batch. + * \param[out] ep_context_nodes Output array of `count` OrtNode instances, each representing an EPContext + * node for a compiled OrtGraph. The execution provider must use + * OrtModelEditorApi::CreateNode to create the OrtNode instances. ONNX Runtime takes + * ownership of the OrtNode instances, so the execution provider must NOT call + * OrtApi::ReleaseNode. Should be ignored if the session is not configured to generate an + * EPContext model. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \note Do NOT cache the provided OrtGraph instances in any of the OrtNodeComputeInfo functions because the + * graphs are only valid for the duration of the call to Compile. Any graph/node/input/output + * names that are needed by the OrtNodeComputeInfo functions must be copied and stored by the OrtEp. + * + * \note As of version 1.24, implementation of this function is optional if the EP does not compile nodes and + * uses a kernel registry instead. + * + * \since Version 1.23. + */ + ORT_API2_STATUS(Compile, _In_ OrtEp* this_ptr, _In_ const OrtGraph** graphs, + _In_ const OrtNode** fused_nodes, _In_ size_t count, + _Out_writes_all_(count) OrtNodeComputeInfo** node_compute_infos, + _Out_writes_(count) OrtNode** ep_context_nodes); + + /** \brief Release OrtNodeComputeInfo instances. + * + * \param[in] this_ptr The OrtEp instance. + * \param[inout] node_compute_infos The OrtNodeComputeInfo instances to release. + * \param[in] num_node_compute_infos The number of OrtNodeComputeInfo instances. + * + * \note As of version 1.24, implementation of this function is optional if the EP does not compile nodes and + * uses a kernel registry instead. + * + * \since Version 1.23. + */ + ORT_API_T(void, ReleaseNodeComputeInfos, _In_ OrtEp* this_ptr, + OrtNodeComputeInfo** node_compute_infos, + _In_ size_t num_node_compute_infos); + + /** \brief Get the EP's preferred data layout. + * + * \note Implementation of this function is optional. + * If not implemented, ORT will assume that this EP prefers the data layout `OrtEpDataLayout::NCHW`. + * + * \param[in] this_ptr The OrtEp instance. + * \param[out] preferred_data_layout The EP's preferred data layout. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(GetPreferredDataLayout, _In_ OrtEp* this_ptr, _Out_ OrtEpDataLayout* preferred_data_layout); + + /** \brief Given an op with domain `domain` and type `op_type`, determine whether an associated node's data layout + * should be converted to `target_data_layout`. + * If the EP prefers a non-default data layout (see `GetPreferredDataLayout()`), this function will be called + * during layout transformation with `target_data_layout` set to the EP's preferred data layout. + * + * \note Implementation of this function is optional. + * If an EP prefers a non-default data layout, it may implement this to customize the specific op data layout + * preferences at a finer granularity. + * + * \param[in] this_ptr The OrtEp instance. + * \param[in] domain The op domain. An empty string means the ONNX domain. + * \param[in] op_type The op type. + * \param[in] target_data_layout The target data layout. + * \param[out] should_convert Whether the associated node's data layout should be converted to `target_data_layout`. + * If greater than 0, convert. + * If 0, don't convert. + * Otherwise, if less than 0, leave the decision to ORT. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(ShouldConvertDataLayoutForOp, _In_ OrtEp* this_ptr, + _In_z_ const char* domain, _In_z_ const char* op_type, + _In_ OrtEpDataLayout target_data_layout, + _Outptr_ int* should_convert); + + /** \brief Set dynamic options on this EP. + * + * Dynamic options can be set by the user at any time after session creation with `OrtApi::SetEpDynamicOptions()`. + * + * \param[in] this_ptr The OrtEp instance. + * \param[in] option_keys The dynamic option keys. + * \param[in] option_values The dynamic option values. + * \param[in] num_options The number of dynamic options. + * + * \note Implementation of this function is optional. + * An EP should only implement this if it needs to handle any dynamic options. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(SetDynamicOptions, _In_ OrtEp* this_ptr, + _In_reads_(num_options) const char* const* option_keys, + _In_reads_(num_options) const char* const* option_values, + _In_ size_t num_options); + + /** \brief Called by ORT to notify the EP of the start of a run. + * + * \param[in] this_ptr The OrtEp instance. + * \param[in] run_options The run options for this run. + * + * \note Implementation of this function is optional. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(OnRunStart, _In_ OrtEp* this_ptr, _In_ const OrtRunOptions* run_options); + + /** \brief Called by ORT to notify the EP of the end of a run. + * + * \param[in] this_ptr The OrtEp instance. + * \param[in] run_options The run options for this run. + * \param[in] sync_stream Whether any associated stream should be synchronized during this call. + * Only applicable if there is such a stream. + * + * \note Implementation of this function is optional. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(OnRunEnd, _In_ OrtEp* this_ptr, _In_ const OrtRunOptions* run_options, _In_ bool sync_stream); + + /** \brief Create an OrtAllocator for the given OrtMemoryInfo for an OrtSession. + * + * The OrtMemoryInfo instance will match one of the values set in the OrtEpDevice using EpDevice_AddAllocatorInfo. + * Any allocator specific options should be read from the session options. + * + * If nullptr OrtEpFactory::CreateAllocator will be used. + * + * \param[in] this_ptr The OrtEpFactory instance. + * \param[in] memory_info The OrtMemoryInfo to create the allocator for. May be nullptr. + * \param[out] allocator The created OrtAllocator instance. Set to nullptr if the default CPU allocator is used. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(CreateAllocator, _In_ OrtEp* this_ptr, + _In_ const OrtMemoryInfo* memory_info, + _Outptr_result_maybenull_ OrtAllocator** allocator); + + /** \brief Create a synchronization stream for the given memory device for an OrtSession. + * + * This is used to create a synchronization stream for the execution provider and is used to synchronize + * operations on the device during model execution. + * Any stream specific options should be read from the session options. + * + * If nullptr OrtEpFactory::CreateSyncStreamForDevice will be used. + * + * \param[in] this_ptr The OrtEpFactory instance. + * \param[in] memory_device The OrtMemoryDevice to create the synchronization stream for. + * \param[out] stream The created OrtSyncStreamImpl instance. nullptr if the execution provider is not stream aware. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(CreateSyncStreamForDevice, _In_ OrtEp* this_ptr, + _In_ const OrtMemoryDevice* memory_device, + _Outptr_ OrtSyncStreamImpl** stream); + + /** \brief Get a string with details about the EP stack used to produce a compiled model. + * + * This function gets a compatibility information string that contains details about the execution provider + * used to compile a given model. This string can later be used with ValidateCompiledModelCompatibilityInfo + * to determine if a compiled model is compatible with the EP. + * + * The returned string should be a null-terminated, UTF-8 encoded string. ORT will copy it. + * + * \param[in] this_ptr The OrtEp instance. + * \param[in] graph The OrtGraph instance for which to generate compatibility information. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API_T(const char*, GetCompiledModelCompatibilityInfo, _In_ OrtEp* this_ptr, + _In_ const OrtGraph* graph); + + /** \brief Gets the execution provider's kernel registry, if any. + * + * A kernel registry contains kernel creation information for operator kernels supported by an EP. + * + * \param[in] this_ptr The OrtEp instance. + * \param[out] kernel_registry Output parameter set to the EP's kernel registry, which must remain valid throughout + * the lifetime of the EP. Can be NULL if the EP doesn't use a kernel registry. + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \note Implementation of this function is optional. If set to NULL, ORT assumes the EP compiles nodes. + * + * \since Version 1.24. + */ + ORT_API2_STATUS(GetKernelRegistry, _In_ OrtEp* this_ptr, + _Outptr_result_maybenull_ const OrtKernelRegistry** kernel_registry); + + /** \brief Gets whether the execution provider supports concurrent run calls made on the session. + * + * \param[in] this_ptr The OrtEp instance. + * \param[out] is_supported Whether concurrent runs are supported. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \note Implementation of this function is optional and it may be set to NULL. + * If not implemented, ORT assumes that concurrent runs are supported. + * + * \since Version 1.24. + */ + ORT_API2_STATUS(IsConcurrentRunSupported, _In_ OrtEp* this_ptr, _Outptr_ bool* is_supported); +}; + +/** \brief The function signature that ORT will call to create OrtEpFactory instances. + * + * This must be available in a function called 'CreateEpFactories' in the execution provider library. + * + * \param[in] registered_name The name the execution library is registered with by RegisterExecutionProviderLibrary + * \param[in] ort_api_base The OrtApiBase instance that is used by the factory to get the OrtApi instance for the + * version of ORT that the library was compiled against. + * \param[in] default_logger The default ORT logger that can be used for logging outside of an inference session. + * \param[in,out] factories The implementation should create and add OrtEpFactory instances to this + * pre-allocated array. + * i.e. usage is `factories[0] = new MyEpFactory();` + * \param[in] max_factories The maximum number of OrtEpFactory instances that can be added to `factories`. + * Current default is to allow 4 factories. This can be increased in the future if needed. + * \param[out] num_factories The number of OrtEpFactory instances created by the factory and added to `factories`. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ +typedef OrtStatus* (*CreateEpApiFactoriesFn)(_In_ const char* registered_name, _In_ const OrtApiBase* ort_api_base, + _In_ const OrtLogger* default_logger, + _Inout_ OrtEpFactory** factories, _In_ size_t max_factories, + _Out_ size_t* num_factories); + +/** \brief The function signature that ORT will call to release an OrtEpFactory instance. + * + * This must be available in a function called 'ReleaseEpFactory' in the execution provider library. + * + * \param[in] factory The OrtEpFactory instance to release. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ +typedef OrtStatus* (*ReleaseEpApiFactoryFn)(_In_ OrtEpFactory* factory); + +/** + * \brief The OrtEpFactory provides functions to create and manage execution providers. + * \since Version 1.22. + */ +struct OrtEpFactory { + /** \brief The ONNX Runtime version the execution provider was compiled with. + * + * Implementation should set to ORT_API_VERSION. + * ORT will use this to ensure it does not call functions that were not available when the library was compiled. + * + * \since Version 1.22. + */ + uint32_t ort_version_supported; + + /** \brief Get the name of the execution provider that the factory creates. + * + * The returned string should be a null-terminated, UTF-8 encoded string. ORT will copy it. + * + * \param[in] this_ptr The OrtEpFactory instance. + * \return The name of the execution provider the factory creates. + * + * \since Version 1.22. + */ + ORT_API_T(const char*, GetName, const OrtEpFactory* this_ptr); + + /** \brief Get the name of vendor who owns the execution provider that the factory creates. + * + * The returned string should be a null-terminated, UTF-8 encoded string. ORT will copy it. + * + * \param[in] this_ptr The OrtEpFactory instance. + * \return vendor The vendor name of the execution provider the factory creates. + * + * \since Version 1.22. + */ + ORT_API_T(const char*, GetVendor, const OrtEpFactory* this_ptr); // return EP vendor + + /** \brief Get information from the execution provider about OrtHardwareDevice support. + * + * \param[in] this_ptr The OrtEpFactory instance. + * Non-const as the factory is passed through to the CreateEp call via the OrtEpDevice. + * \param[in] devices The OrtHardwareDevice instances that are available. + * \param[in] num_devices The number of OrtHardwareDevice instances. + * \param[out] ep_devices OrtEpDevice instances for each OrtHardwareDevice that the EP can use. + * The implementation should call OrtEpApi::CreateEpDevice to create, and add the OrtEpDevice + * instances to this pre-allocated array. ORT will take ownership of the values returned. + * i.e. usage is `ep_devices[0] = ;` + * \param[in] max_ep_devices The maximum number of OrtEpDevices that can be added to ep_devices. + * Current default is 8. This can be increased if needed. + * \param[out] num_ep_devices The number of EP devices added to ep_devices. + * \return true if the factory can create an execution provider that uses `device`. + * + * \since Version 1.22. + */ + ORT_API2_STATUS(GetSupportedDevices, _In_ OrtEpFactory* this_ptr, + _In_reads_(num_devices) const OrtHardwareDevice* const* devices, + _In_ size_t num_devices, + _Inout_ OrtEpDevice** ep_devices, + _In_ size_t max_ep_devices, + _Out_ size_t* num_ep_devices); + + /** \brief Function to create an OrtEp instance for use in a Session. + * + * ORT will call ReleaseEp to release the instance when it is no longer needed. + * + * \param[in] this_ptr The OrtEpFactory instance. + * \param[in] devices The OrtHardwareDevice instances that the execution provider was selected to use. + * May be a subset of the OrtHardwareDevice instances that the execution provider's factory + * set as supported in the call to OrtEpFactory::GetSupportedDevices. + * \param[in] ep_metadata_pairs Execution provider metadata that was provided to OrtEpApi::CreateEpDevice, for each + * device. + * \param[in] num_devices The number of devices the execution provider was selected for. + * \param[in] session_options The OrtSessionOptions instance that contains the configuration options for the + * session. This will include ep_options from GetSupportedDevices as well as any + * user provided overrides. + * Execution provider options will have been added with a prefix of 'ep.[ep name].'. + * The OrtSessionOptions instance will NOT be valid after this call and should not be + * stored for later use. + * \param[in] logger The OrtLogger instance for the session that the execution provider should use for logging. + * \param[out] ep The OrtEp instance created by the factory. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.22. + */ + ORT_API2_STATUS(CreateEp, _In_ OrtEpFactory* this_ptr, + _In_reads_(num_devices) const OrtHardwareDevice* const* devices, + _In_reads_(num_devices) const OrtKeyValuePairs* const* ep_metadata_pairs, + _In_ size_t num_devices, + _In_ const OrtSessionOptions* session_options, + _In_ const OrtLogger* logger, _Outptr_ OrtEp** ep); + + /** \brief Release the OrtEp instance. + * + * \param[in] this_ptr The OrtEpFactory instance. + * \param[in] ep The OrtEp instance to release. + * + * \since Version 1.22. + */ + ORT_API_T(void, ReleaseEp, OrtEpFactory* this_ptr, struct OrtEp* ep); + + /** \brief Get the vendor id who owns the execution provider that the factory creates. + * + * This is typically the PCI vendor ID. See https://pcisig.com/membership/member-companies + * + * \param[in] this_ptr The OrtEpFactory instance. + * \return vendor_id The vendor ID of the execution provider the factory creates. + * + * \since Version 1.23. + */ + ORT_API_T(uint32_t, GetVendorId, const OrtEpFactory* this_ptr); + + /** \brief Get the version of the execution provider that the factory creates. + * + * The version string should adhere to the Semantic Versioning 2.0 specification + * (https://github.com/semver/semver/blob/v2.0.0/semver.md). + * + * The returned string should be a null-terminated, UTF-8 encoded string. ORT will copy it. + * + * \param[in] this_ptr The OrtEpFactory instance. + * \return The execution provider version string. + * + * \since Version 1.23. + */ + ORT_API_T(const char*, GetVersion, _In_ const OrtEpFactory* this_ptr); + + /** \brief Validate the compatibility of a compiled model with the execution provider factory for one or more devices. + * + * Given a compatibility info string produced during model compilation, the EP factory should determine whether the + * compiled model is compatible with the EP factory when targeting the provided hardware devices. All devices provided + * must belong to the same execution provider instance that this factory creates. + * + * The EP factory implementation should consider the set of devices (e.g., multi-adapter or multi-GPU scenarios) when + * evaluating compatibility and set `model_compatibility` accordingly. + * + * \param[in] this_ptr The OrtEpFactory instance. + * \param[in] devices Array of OrtHardwareDevice pointers that the EP would run on. All must map to this EP. + * \param[in] num_devices Number of entries in `devices`. + * \param[in] compatibility_info The compatibility information string produced when the model was compiled. + * \param[out] model_compatibility OrtCompiledModelCompatibility value describing the compatibility of the model with the EP. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(ValidateCompiledModelCompatibilityInfo, _In_ OrtEpFactory* this_ptr, + _In_reads_(num_devices) const OrtHardwareDevice* const* devices, + _In_ size_t num_devices, + _In_ const char* compatibility_info, + _Out_ OrtCompiledModelCompatibility* model_compatibility); + + /** \brief Create an OrtAllocator that can be shared across sessions for the given OrtMemoryInfo. + * + * The factory that creates the EP is responsible for providing the allocators required by the EP. + * The OrtMemoryInfo instance will match one of the values set in the OrtEpDevice using EpDevice_AddAllocatorInfo. + * + * \param[in] this_ptr The OrtEpFactory instance. + * \param[in] memory_info The OrtMemoryInfo to create the allocator for. May be nullptr. + * \param[in] allocator_options Optional key-value pairs for allocator options, can be nullptr. + * \param[out] allocator The created OrtAllocator instance. Set to nullptr if the default CPU allocator is used. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(CreateAllocator, _In_ OrtEpFactory* this_ptr, + _In_ const OrtMemoryInfo* memory_info, + _In_opt_ const OrtKeyValuePairs* allocator_options, + _Outptr_result_maybenull_ OrtAllocator** allocator); + + /** \brief Release an OrtAllocator created by the factory. + * + * \since Version 1.23. + */ + ORT_API_T(void, ReleaseAllocator, _In_ OrtEpFactory* this_ptr, _In_ OrtAllocator* allocator); + + /** \brief Create an OrtDataTransferImpl instance for the factory. + * + * This is used to create an IDataTransfer implementation that can be used to copy data between devices + * that the execution provider supports. + * + * \param[in] this_ptr The OrtEpFactory instance. + * \param[out] data_transfer The created OrtDataTransferImpl instance. Set to nullptr if not required. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(CreateDataTransfer, _In_ OrtEpFactory* this_ptr, + _Outptr_result_maybenull_ OrtDataTransferImpl** data_transfer); + + /** \brief Check if execution providers created by the factory are stream aware. + * + * \param[in] this_ptr The OrtEpFactory instance. + * \return True if the factory creates execution providers that are stream aware and it implements CreateSyncStreamForDevice. + * + * \since Version 1.23. + */ + ORT_API_T(bool, IsStreamAware, _In_ const OrtEpFactory* this_ptr); + + /** \brief Create a synchronization stream for the given memory device. + * + * This is used to create a synchronization stream for the memory device that can be used for operations outside of + * a session. + * + * \param[in] this_ptr The OrtEpFactory instance. + * \param[in] memory_device The OrtMemoryDevice to create the synchronization stream for. + * \param[in] stream_options Options for stream creation. May be nullptr. + * \param[out] stream The created OrtSyncStreamImpl instance. nullptr if the execution provider is not stream aware. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.23. + */ + ORT_API2_STATUS(CreateSyncStreamForDevice, _In_ OrtEpFactory* this_ptr, + _In_ const OrtMemoryDevice* memory_device, + _In_opt_ const OrtKeyValuePairs* stream_options, + _Outptr_ OrtSyncStreamImpl** stream); + + /** \brief Check for known incompatibility reasons between a hardware device and this execution provider. + * + * This function allows an execution provider to check if a specific hardware device is compatible + * with the execution provider. The EP can set specific incompatibility reasons via the + * OrtDeviceEpIncompatibilityDetails parameter using OrtEpApi::DeviceEpIncompatibilityDetails_SetDetails. + * + * \param[in] this_ptr The OrtEpFactory instance. + * \param[in] hw The hardware device to check for incompatibility. + * \param[in,out] details Pre-allocated incompatibility details object created and initialized by ORT. + * The EP can use OrtEpApi::DeviceEpIncompatibilityDetails_SetDetails to set + * incompatibility information. If the device is compatible, the EP can + * leave the object unchanged (it defaults to no incompatibility). + * + * \note Implementation of this function is optional. + * If not implemented, ORT will assume the device is compatible with this EP. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(GetHardwareDeviceIncompatibilityDetails, _In_ OrtEpFactory* this_ptr, + _In_ const OrtHardwareDevice* hw, + _Inout_ OrtDeviceEpIncompatibilityDetails* details); + + /** \brief Create an OrtExternalResourceImporterImpl for external resource import. + * + * This is used to create an external resource importer that enables zero-copy import of + * external GPU memory (e.g., D3D12 shared resources) and synchronization primitives + * (e.g., D3D12 timeline fences). + * + * EPs that support external resource import (via CUDA, HIP, Vulkan, or D3D12 APIs) can + * implement this to allow applications to share GPU resources without copies. + * + * \param[in] this_ptr The OrtEpFactory instance. + * \param[in] ep_device The OrtEpDevice to create the external resource importer for. + * \param[out] out_importer The created OrtExternalResourceImporterImpl instance. + * Set to nullptr if external resource import is not supported. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \note Implementation of this function is optional. + * An EP factory should only implement this if it supports external resource import. + * If not implemented or not supported, return ORT_NOT_IMPLEMENTED or set out_importer to nullptr. + * + * \since Version 1.24. + */ + ORT_API2_STATUS(CreateExternalResourceImporterForDevice, _In_ OrtEpFactory* this_ptr, + _In_ const OrtEpDevice* ep_device, + _Outptr_result_maybenull_ OrtExternalResourceImporterImpl** out_importer); + + /** \brief Returns the number of OrtCustomOpDomains that this factory provides. + * + * \param[in] this_ptr The OrtEpFactory instance. + * \param[out] num_domains Output parameter set to the number of provided OrtCustomOpDomain instances. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(GetNumCustomOpDomains, _In_ OrtEpFactory* this_ptr, _Out_ size_t* num_domains); + + /** \brief Gets the EP-specific OrtCustomOpDomains. + * + * This function is used when running inference on a model that contains EP-specific custom operations. + * + * Workflow: + * 1. The EP factory implements this function to supply a list of OrtCustomOpDomain instances. + * 2. The application either 1) calls SessionOptionsAppendExecutionProvider_V2() with an OrtEpDevice containing + * the plugin EP's factory or 2) enables auto ep selection. + * 3. 1) SessionOptionsAppendExecutionProvider_V2() appends the provided OrtCustomOpDomains to the + * session options or 2) ORT registers the OrtCustomOpDomains provided by the EP devices + * that could be potentially selected. + * + * As a result, any session created from these session options will have these custom op domains registered + * in ORT, ensuring that the custom ops are properly recognized and validated when the model is loaded. + * + * Plugin EPs can provide two types of custom ops: + * 1. A full OrtCustomOp with a concrete kernel implementation + * - A Plugin EP can supply an OrtCustomOp and a corresponding CustomKernel::Compute() implementation. + * - In GetCapability(), it calls EpGraphSupportInfo_AddSingleNode() to inform ORT + * that the custom node should NOT be fused or compiled. Instead, ORT should invoke + * the custom node's Compute() function at runtime. + * + * 2. A "placeholder" OrtCustomOp with an empty kernel implementation + * - A compile-based Plugin EP can supply an OrtCustomOp whose CustomKernel::Compute() + * does nothing. The purpose is to satisfy model validation during model loading by + * registering the custom op as a valid operator in the session. + * - In GetCapability(), the EP should call EpGraphSupportInfo_AddNodesToFuse() to + * notify ORT that this custom node should be fused and compiled by the EP. + * - In Compile(), the EP executes its compiled bits to perform inference for + * the fused custom node. + * + * Note: The OrtCustomOpDomain instances must be valid while any session is using them. + EP factory has the responsibility to release OrtCustomOpDomain instances it creates. It happens + * automatically if using the C++ Ort::CustomOpDomain class. + * + * \param[in] this_ptr The OrtEpFactory instance. + * \param[out] domains Array of `num_domains` elements pre-allocated by ORT that should be filled with + OrtCustomOpDomain instances created by the EP. The `num_domains` is the value returned by + GetNumCustomOpDomains(). + * \param[in] num_domains The size of the `domains` array pre-allocated by ORT. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \since Version 1.24. + */ + ORT_API2_STATUS(GetCustomOpDomains, _In_ OrtEpFactory* this_ptr, + _Out_writes_all_(num_domains) OrtCustomOpDomain** domains, _In_ size_t num_domains); +}; + +#ifdef __cplusplus +} +#endif diff --git a/dist/include/onnxruntime/onnxruntime_ep_device_ep_metadata_keys.h b/dist/include/onnxruntime/onnxruntime_ep_device_ep_metadata_keys.h new file mode 100644 index 0000000..5ea4261 --- /dev/null +++ b/dist/include/onnxruntime/onnxruntime_ep_device_ep_metadata_keys.h @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +// This file contains well-known keys for OrtEpDevice and OrtHardwareDevice metadata entries. +// It does NOT specify all available metadata keys. + +// Key for the execution provider version string. This should be available for all plugin EPs. +static const char* const kOrtEpDevice_EpMetadataKey_Version = "version"; + +// Key for the execution provider OS driver version. +static const char* const kOrtEpDevice_EpMetadataKey_OSDriverVersion = "os_driver_version"; + +// Prefix for execution provider compatibility information stored in model metadata. +// Used when generating EP context models to store compatibility strings for each EP. +// Full key format: "ep_compatibility_info." +static const char* const kOrtModelMetadata_EpCompatibilityInfoPrefix = "ep_compatibility_info."; + +// Key for the execution provider library path (for dynamically loaded EPs) +static const char* const kOrtEpDevice_EpMetadataKey_LibraryPath = "library_path"; + +// Optional metadata key to determine if a OrtHardwareDevice represents a virtual (non-hardware) device. +// Possible values: +// - "0": OrtHardwareDevice is not virtual (i.e., actual hardware device). This is the assumed default value +// if this metadata key is not present. +// - "1": OrtHardwareDevice is virtual. +static const char* const kOrtHardwareDevice_MetadataKey_IsVirtual = "is_virtual"; diff --git a/dist/include/onnxruntime/onnxruntime_float16.h b/dist/include/onnxruntime/onnxruntime_float16.h new file mode 100644 index 0000000..625a1e8 --- /dev/null +++ b/dist/include/onnxruntime/onnxruntime_float16.h @@ -0,0 +1,537 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include +#include +#include +#include + +namespace onnxruntime_float16 { + +namespace detail { + +enum class endian { +#if defined(_WIN32) + little = 0, + big = 1, + native = little, +#elif defined(__GNUC__) || defined(__clang__) + little = __ORDER_LITTLE_ENDIAN__, + big = __ORDER_BIG_ENDIAN__, + native = __BYTE_ORDER__, +#else +#error onnxruntime_float16::detail::endian is not implemented in this environment. +#endif +}; + +static_assert( + endian::native == endian::little || endian::native == endian::big, + "Only little-endian or big-endian native byte orders are supported."); + +} // namespace detail + +/// +/// Shared implementation between public and internal classes. CRTP pattern. +/// +template +struct Float16Impl { + protected: + /// + /// Converts from float to uint16_t float16 representation + /// + /// + /// + constexpr static uint16_t ToUint16Impl(float v) noexcept; + + /// + /// Converts float16 to float + /// + /// float representation of float16 value + float ToFloatImpl() const noexcept; + + /// + /// Creates an instance that represents absolute value. + /// + /// Absolute value + uint16_t AbsImpl() const noexcept { + return static_cast(val & ~kSignMask); + } + + /// + /// Creates a new instance with the sign flipped. + /// + /// Flipped sign instance + uint16_t NegateImpl() const noexcept { + return IsNaN() ? val : static_cast(val ^ kSignMask); + } + + public: + // uint16_t special values + static constexpr uint16_t kSignMask = 0x8000U; + static constexpr uint16_t kBiasedExponentMask = 0x7C00U; + static constexpr uint16_t kPositiveInfinityBits = 0x7C00U; + static constexpr uint16_t kNegativeInfinityBits = 0xFC00U; + static constexpr uint16_t kPositiveQNaNBits = 0x7E00U; + static constexpr uint16_t kNegativeQNaNBits = 0xFE00U; + static constexpr uint16_t kMaxValueBits = 0x7BFFU; // Largest normal number + static constexpr uint16_t kMinValueBits = 0xFBFFU; // Lowest normal number + static constexpr uint16_t kOneBits = 0x3C00U; + static constexpr uint16_t kMinusOneBits = 0xBC00U; + + uint16_t val{0}; + + Float16Impl() = default; + + /// + /// Checks if the value is negative + /// + /// true if negative + bool IsNegative() const noexcept { + return static_cast(val) < 0; + } + + /// + /// Tests if the value is NaN + /// + /// true if NaN + bool IsNaN() const noexcept { + return AbsImpl() > kPositiveInfinityBits; + } + + /// + /// Tests if the value is finite + /// + /// true if finite + bool IsFinite() const noexcept { + return AbsImpl() < kPositiveInfinityBits; + } + + /// + /// Tests if the value represents positive infinity. + /// + /// true if positive infinity + bool IsPositiveInfinity() const noexcept { + return val == kPositiveInfinityBits; + } + + /// + /// Tests if the value represents negative infinity + /// + /// true if negative infinity + bool IsNegativeInfinity() const noexcept { + return val == kNegativeInfinityBits; + } + + /// + /// Tests if the value is either positive or negative infinity. + /// + /// True if absolute value is infinity + bool IsInfinity() const noexcept { + return AbsImpl() == kPositiveInfinityBits; + } + + /// + /// Tests if the value is NaN or zero. Useful for comparisons. + /// + /// True if NaN or zero. + bool IsNaNOrZero() const noexcept { + auto abs = AbsImpl(); + return (abs == 0 || abs > kPositiveInfinityBits); + } + + /// + /// Tests if the value is normal (not zero, subnormal, infinite, or NaN). + /// + /// True if so + bool IsNormal() const noexcept { + auto abs = AbsImpl(); + return (abs < kPositiveInfinityBits) // is finite + && (abs != 0) // is not zero + && ((abs & kBiasedExponentMask) != 0); // is not subnormal (has a non-zero exponent) + } + + /// + /// Tests if the value is subnormal (denormal). + /// + /// True if so + bool IsSubnormal() const noexcept { + auto abs = AbsImpl(); + return (abs < kPositiveInfinityBits) // is finite + && (abs != 0) // is not zero + && ((abs & kBiasedExponentMask) == 0); // is subnormal (has a zero exponent) + } + + /// + /// Creates an instance that represents absolute value. + /// + /// Absolute value + Derived Abs() const noexcept { return Derived::FromBits(AbsImpl()); } + + /// + /// Creates a new instance with the sign flipped. + /// + /// Flipped sign instance + Derived Negate() const noexcept { return Derived::FromBits(NegateImpl()); } + + /// + /// IEEE defines that positive and negative zero are equal, this gives us a quick equality check + /// for two values by or'ing the private bits together and stripping the sign. They are both zero, + /// and therefore equivalent, if the resulting value is still zero. + /// + /// first value + /// second value + /// True if both arguments represent zero + static bool AreZero(const Float16Impl& lhs, const Float16Impl& rhs) noexcept { + return static_cast((lhs.val | rhs.val) & ~kSignMask) == 0; + } + + bool operator==(const Float16Impl& rhs) const noexcept { + if (IsNaN() || rhs.IsNaN()) { + // IEEE defines that NaN is not equal to anything, including itself. + return false; + } + return val == rhs.val; + } + + bool operator!=(const Float16Impl& rhs) const noexcept { return !(*this == rhs); } + + bool operator<(const Float16Impl& rhs) const noexcept { + if (IsNaN() || rhs.IsNaN()) { + // IEEE defines that NaN is unordered with respect to everything, including itself. + return false; + } + + const bool left_is_negative = IsNegative(); + if (left_is_negative != rhs.IsNegative()) { + // When the signs of left and right differ, we know that left is less than right if it is + // the negative value. The exception to this is if both values are zero, in which case IEEE + // says they should be equal, even if the signs differ. + return left_is_negative && !AreZero(*this, rhs); + } + return (val != rhs.val) && ((val < rhs.val) ^ left_is_negative); + } +}; + +// The following Float16_t conversions are based on the code from +// Eigen library. + +// The conversion routines are Copyright (c) Fabian Giesen, 2016. +// The original license follows: +// +// Copyright (c) Fabian Giesen, 2016 +// All rights reserved. +// Redistribution and use in source and binary forms, with or without +// modification, are permitted. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +namespace detail { +union float32_bits { + unsigned int u; + float f; +}; +} // namespace detail + +template +inline constexpr uint16_t Float16Impl::ToUint16Impl(float v) noexcept { + detail::float32_bits f{}; + f.f = v; + + constexpr detail::float32_bits f32infty = {255 << 23}; + constexpr detail::float32_bits f16max = {(127 + 16) << 23}; + constexpr detail::float32_bits denorm_magic = {((127 - 15) + (23 - 10) + 1) << 23}; + constexpr unsigned int sign_mask = 0x80000000u; + uint16_t val = static_cast(0x0u); + + unsigned int sign = f.u & sign_mask; + f.u ^= sign; + + // NOTE all the integer compares in this function can be safely + // compiled into signed compares since all operands are below + // 0x80000000. Important if you want fast straight SSE2 code + // (since there's no unsigned PCMPGTD). + + if (f.u >= f16max.u) { // result is Inf or NaN (all exponent bits set) + val = (f.u > f32infty.u) ? 0x7e00 : 0x7c00; // NaN->qNaN and Inf->Inf + } else { // (De)normalized number or zero + if (f.u < (113 << 23)) { // resulting FP16 is subnormal or zero + // use a magic value to align our 10 mantissa bits at the bottom of + // the float. as long as FP addition is round-to-nearest-even this + // just works. + f.f += denorm_magic.f; + + // and one integer subtract of the bias later, we have our final float! + val = static_cast(f.u - denorm_magic.u); + } else { + unsigned int mant_odd = (f.u >> 13) & 1; // resulting mantissa is odd + + // update exponent, rounding bias part 1 + // Equivalent to `f.u += ((unsigned int)(15 - 127) << 23) + 0xfff`, but + // without arithmetic overflow. + f.u += 0xc8000fffU; + // rounding bias part 2 + f.u += mant_odd; + // take the bits! + val = static_cast(f.u >> 13); + } + } + + val |= static_cast(sign >> 16); + return val; +} + +template +inline float Float16Impl::ToFloatImpl() const noexcept { + constexpr detail::float32_bits magic = {113 << 23}; + constexpr unsigned int shifted_exp = 0x7c00 << 13; // exponent mask after shift + detail::float32_bits o{}; + + o.u = (val & 0x7fff) << 13; // exponent/mantissa bits + unsigned int exp = shifted_exp & o.u; // just the exponent + o.u += (127 - 15) << 23; // exponent adjust + + // handle exponent special cases + if (exp == shifted_exp) { // Inf/NaN? + o.u += (128 - 16) << 23; // extra exp adjust + } else if (exp == 0) { // Zero/Denormal? + o.u += 1 << 23; // extra exp adjust + o.f -= magic.f; // re-normalize + } + + // Attempt to workaround the Internal Compiler Error on ARM64 + // for bitwise | operator, including std::bitset +#if (defined _MSC_VER) && (defined _M_ARM || defined _M_ARM64 || defined _M_ARM64EC) + if (IsNegative()) { + return -o.f; + } +#else + // original code: + o.u |= (val & 0x8000U) << 16U; // sign bit +#endif + return o.f; +} + +/// Shared implementation between public and internal classes. CRTP pattern. +template +struct BFloat16Impl { + protected: + /// + /// Converts from float to uint16_t float16 representation + /// + /// + /// + static uint16_t ToUint16Impl(float v) noexcept; + + /// + /// Converts bfloat16 to float + /// + /// float representation of bfloat16 value + float ToFloatImpl() const noexcept; + + /// + /// Creates an instance that represents absolute value. + /// + /// Absolute value + uint16_t AbsImpl() const noexcept { + return static_cast(val & ~kSignMask); + } + + /// + /// Creates a new instance with the sign flipped. + /// + /// Flipped sign instance + uint16_t NegateImpl() const noexcept { + return IsNaN() ? val : static_cast(val ^ kSignMask); + } + + public: + // uint16_t special values + static constexpr uint16_t kSignMask = 0x8000U; + static constexpr uint16_t kBiasedExponentMask = 0x7F80U; + static constexpr uint16_t kPositiveInfinityBits = 0x7F80U; + static constexpr uint16_t kNegativeInfinityBits = 0xFF80U; + static constexpr uint16_t kPositiveQNaNBits = 0x7FC1U; + static constexpr uint16_t kNegativeQNaNBits = 0xFFC1U; + static constexpr uint16_t kMaxValueBits = 0x7F7FU; + static constexpr uint16_t kMinValueBits = 0xFF7FU; + static constexpr uint16_t kRoundToNearest = 0x7FFFU; + static constexpr uint16_t kOneBits = 0x3F80U; + static constexpr uint16_t kMinusOneBits = 0xBF80U; + + uint16_t val{0}; + + BFloat16Impl() = default; + + /// + /// Checks if the value is negative + /// + /// true if negative + bool IsNegative() const noexcept { + return static_cast(val) < 0; + } + + /// + /// Tests if the value is NaN + /// + /// true if NaN + bool IsNaN() const noexcept { + return AbsImpl() > kPositiveInfinityBits; + } + + /// + /// Tests if the value is finite + /// + /// true if finite + bool IsFinite() const noexcept { + return AbsImpl() < kPositiveInfinityBits; + } + + /// + /// Tests if the value represents positive infinity. + /// + /// true if positive infinity + bool IsPositiveInfinity() const noexcept { + return val == kPositiveInfinityBits; + } + + /// + /// Tests if the value represents negative infinity + /// + /// true if negative infinity + bool IsNegativeInfinity() const noexcept { + return val == kNegativeInfinityBits; + } + + /// + /// Tests if the value is either positive or negative infinity. + /// + /// True if absolute value is infinity + bool IsInfinity() const noexcept { + return AbsImpl() == kPositiveInfinityBits; + } + + /// + /// Tests if the value is NaN or zero. Useful for comparisons. + /// + /// True if NaN or zero. + bool IsNaNOrZero() const noexcept { + auto abs = AbsImpl(); + return (abs == 0 || abs > kPositiveInfinityBits); + } + + /// + /// Tests if the value is normal (not zero, subnormal, infinite, or NaN). + /// + /// True if so + bool IsNormal() const noexcept { + auto abs = AbsImpl(); + return (abs < kPositiveInfinityBits) // is finite + && (abs != 0) // is not zero + && ((abs & kBiasedExponentMask) != 0); // is not subnormal (has a non-zero exponent) + } + + /// + /// Tests if the value is subnormal (denormal). + /// + /// True if so + bool IsSubnormal() const noexcept { + auto abs = AbsImpl(); + return (abs < kPositiveInfinityBits) // is finite + && (abs != 0) // is not zero + && ((abs & kBiasedExponentMask) == 0); // is subnormal (has a zero exponent) + } + + /// + /// Creates an instance that represents absolute value. + /// + /// Absolute value + Derived Abs() const noexcept { return Derived::FromBits(AbsImpl()); } + + /// + /// Creates a new instance with the sign flipped. + /// + /// Flipped sign instance + Derived Negate() const noexcept { return Derived::FromBits(NegateImpl()); } + + /// + /// IEEE defines that positive and negative zero are equal, this gives us a quick equality check + /// for two values by or'ing the private bits together and stripping the sign. They are both zero, + /// and therefore equivalent, if the resulting value is still zero. + /// + /// first value + /// second value + /// True if both arguments represent zero + static bool AreZero(const BFloat16Impl& lhs, const BFloat16Impl& rhs) noexcept { + // IEEE defines that positive and negative zero are equal, this gives us a quick equality check + // for two values by or'ing the private bits together and stripping the sign. They are both zero, + // and therefore equivalent, if the resulting value is still zero. + return static_cast((lhs.val | rhs.val) & ~kSignMask) == 0; + } +}; + +template +inline uint16_t BFloat16Impl::ToUint16Impl(float v) noexcept { + uint16_t result; + if (std::isnan(v)) { + result = kPositiveQNaNBits; + } else { + auto get_msb_half = [](float fl) { + uint16_t result; +#ifdef __cpp_if_constexpr + if constexpr (detail::endian::native == detail::endian::little) { +#else + if (detail::endian::native == detail::endian::little) { +#endif + std::memcpy(&result, reinterpret_cast(&fl) + sizeof(uint16_t), sizeof(uint16_t)); + } else { + std::memcpy(&result, &fl, sizeof(uint16_t)); + } + return result; + }; + + uint16_t upper_bits = get_msb_half(v); + union { + uint32_t U32; + float F32; + }; + F32 = v; + U32 += (upper_bits & 1) + kRoundToNearest; + result = get_msb_half(F32); + } + return result; +} + +template +inline float BFloat16Impl::ToFloatImpl() const noexcept { + if (IsNaN()) { + return std::numeric_limits::quiet_NaN(); + } + float result; + char* const first = reinterpret_cast(&result); + char* const second = first + sizeof(uint16_t); +#ifdef __cpp_if_constexpr + if constexpr (detail::endian::native == detail::endian::little) { +#else + if (detail::endian::native == detail::endian::little) { +#endif + std::memset(first, 0, sizeof(uint16_t)); + std::memcpy(second, &val, sizeof(uint16_t)); + } else { + std::memcpy(first, &val, sizeof(uint16_t)); + std::memset(second, 0, sizeof(uint16_t)); + } + return result; +} + +} // namespace onnxruntime_float16 diff --git a/dist/include/onnxruntime/onnxruntime_lite_custom_op.h b/dist/include/onnxruntime/onnxruntime_lite_custom_op.h new file mode 100644 index 0000000..81c2076 --- /dev/null +++ b/dist/include/onnxruntime/onnxruntime_lite_custom_op.h @@ -0,0 +1,1099 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// Summary +// The header has APIs to save custom op authors the trouble of defining schemas, +// which will be inferred by functions' signature, as long as their argument list has types supported here. +// Input could be: +// 1. Tensor of onnx data types. +// 2. Span of onnx data types. +// 3. Scalar of onnx data types. +// A input could be optional if indicated as std::optional<...>. +// For an output, it must be a tensor of onnx data types. +// Further, the header also has utility for a simple custom struct, where resources could be kept, to be registered as a custom op. +// For concrete examples, please search keyword "LiteCustomOpTest" under "/onnxruntime/test/". +// Note - all APIs in this header are ABI. + +#pragma once +#include "onnxruntime_cxx_api.h" +#include +#include +#include +#include + +namespace Ort { +namespace Custom { + +class ArgBase { + public: + ArgBase(OrtKernelContext* ctx, + size_t indice, + bool is_input) : ctx_(ctx), indice_(indice), is_input_(is_input) {} + virtual ~ArgBase() {}; + + protected: + struct KernelContext ctx_; + size_t indice_; + bool is_input_; +}; + +using ArgPtr = std::unique_ptr; +using ArgPtrs = std::vector; + +class TensorBase : public ArgBase { + public: + TensorBase(OrtKernelContext* ctx, + size_t indice, + bool is_input) : ArgBase(ctx, indice, is_input) {} + + operator bool() const { + return shape_.has_value(); + } + + const std::vector& Shape() const { + if (!shape_.has_value()) { + ORT_CXX_API_THROW("tensor shape is not yet initialized", OrtErrorCode::ORT_RUNTIME_EXCEPTION); + } + return shape_.value(); + } + + ONNXTensorElementDataType Type() const { + return type_; + } + + int64_t NumberOfElement() const { + if (shape_.has_value()) { + return std::accumulate(shape_->begin(), shape_->end(), 1LL, std::multiplies()); + } else { + return 0; + } + } + + std::string Shape2Str() const { + if (shape_.has_value()) { + std::string shape_str; + for (const auto& dim : *shape_) { + shape_str.append(std::to_string(dim)); + shape_str.append(", "); + } + return shape_str; + } else { + return "empty"; + } + } + + bool IsCpuTensor() const { + return strcmp("Cpu", mem_type_) == 0; + } + + virtual const void* DataRaw() const = 0; + virtual size_t SizeInBytes() const = 0; + + protected: + std::optional> shape_; + ONNXTensorElementDataType type_ = ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED; + const char* mem_type_ = "Cpu"; +}; + +template +struct Span { + const T* data_ = {}; + size_t size_ = {}; + void Assign(const T* data, size_t size) { + data_ = data; + size_ = size; + } + size_t size() const { return size_; } + T operator[](size_t indice) const { + return data_[indice]; + } + const T* data() const { return data_; } +}; + +template +class Tensor : public TensorBase { + public: + using TT = typename std::remove_reference::type; + Tensor(OrtKernelContext* ctx, size_t indice, bool is_input) : TensorBase(ctx, indice, is_input) { + if (is_input_) { + if (indice >= ctx_.GetInputCount()) { + ORT_CXX_API_THROW("invalid indice for Ort::Custom::Tensor", OrtErrorCode::ORT_INVALID_ARGUMENT); + } + const_value_ = ctx_.GetInput(indice); + auto type_shape_info = const_value_.GetTensorTypeAndShapeInfo(); + shape_ = type_shape_info.GetShape(); + } + } + const TT* Data() const { + return reinterpret_cast(const_value_.GetTensorRawData()); + } + TT* Allocate(const std::vector& shape) { + shape_ = shape; + if (!data_) { + shape_ = shape; + data_ = ctx_.GetOutput(indice_, shape).template GetTensorMutableData(); + } + return data_; + } + static TT GetT() { return (TT)0; } + const Span& AsSpan() { + if (!shape_.has_value() || shape_->size() != 1) { + ORT_CXX_API_THROW("invalid shape while trying to get a span out of Ort::Custom::Tensor", + OrtErrorCode::ORT_RUNTIME_EXCEPTION); + } + span_.Assign(Data(), static_cast((*shape_)[0])); + return span_; + } + const T& AsScalar() { + if (!shape_.has_value() || shape_->size() != 1 || (*shape_)[0] != 1) { + ORT_CXX_API_THROW("invalid shape while trying to get a scalar from Ort::Custom::Tensor", + OrtErrorCode::ORT_RUNTIME_EXCEPTION); + } + return *Data(); + } + const void* DataRaw() const override { + return reinterpret_cast(Data()); + } + + size_t SizeInBytes() const override { + return sizeof(TT) * static_cast(NumberOfElement()); + } + + private: + ConstValue const_value_; // for input + TT* data_{}; // for output + Span span_; +}; + +template <> +class Tensor : public TensorBase { + public: + using strings = std::vector; + + Tensor(OrtKernelContext* ctx, size_t indice, bool is_input) : TensorBase(ctx, indice, is_input) { + if (is_input_) { + if (indice >= ctx_.GetInputCount()) { + ORT_CXX_API_THROW("invalid indice for Ort::Custom::Tensor", OrtErrorCode::ORT_INVALID_ARGUMENT); + } + auto const_value = ctx_.GetInput(indice); + auto type_shape_info = const_value.GetTensorTypeAndShapeInfo(); + shape_ = type_shape_info.GetShape(); + auto num_chars = const_value.GetStringTensorDataLength(); + // note - there will be copy ... + auto num_strings = static_cast(NumberOfElement()); + if (num_strings) { + std::vector chars(num_chars + 1, '\0'); + std::vector offsets(num_strings); + const_value.GetStringTensorContent(static_cast(chars.data()), num_chars, offsets.data(), offsets.size()); + auto upper_bound = num_strings - 1; + input_strings_.resize(num_strings); + for (size_t i = upper_bound;; --i) { + if (i < upper_bound) { + chars[offsets[i + 1]] = '\0'; + } + input_strings_[i] = chars.data() + offsets[i]; + if (0 == i) { + break; + } + } + } + } + } + const strings& Data() const { + return input_strings_; + } + const void* DataRaw() const override { + if (input_strings_.size() != 1) { + ORT_CXX_API_THROW("DataRaw() only applies to string scalar", ORT_RUNTIME_EXCEPTION); + } + return reinterpret_cast(input_strings_[0].c_str()); + } + size_t SizeInBytes() const override { + if (input_strings_.size() != 1) { + ORT_CXX_API_THROW("SizeInBytes() only applies to string scalar", ORT_RUNTIME_EXCEPTION); + } + return input_strings_[0].size(); + } + void SetStringOutput(const strings& ss, const std::vector& dims) { + shape_ = dims; + std::vector raw; + for (const auto& s : ss) { + raw.push_back(s.data()); + } + auto output = ctx_.GetOutput(indice_, dims.data(), dims.size()); + // note - there will be copy ... + output.FillStringTensor(raw.data(), raw.size()); + } + const Span& AsSpan() { + ORT_CXX_API_THROW("span for TensorT of string not implemented", OrtErrorCode::ORT_RUNTIME_EXCEPTION); + } + const std::string& AsScalar() { + if (input_strings_.size() != 1) { + ORT_CXX_API_THROW("invalid shape while trying to get a scalar string from Ort::Custom::Tensor", + OrtErrorCode::ORT_RUNTIME_EXCEPTION); + } + return input_strings_[0]; + } + + private: + std::vector input_strings_; // for input +}; + +template <> +class Tensor : public TensorBase { + public: + using strings = std::vector; + using string_views = std::vector; + + Tensor(OrtKernelContext* ctx, size_t indice, bool is_input) : TensorBase(ctx, indice, is_input) { + if (is_input_) { + if (indice >= ctx_.GetInputCount()) { + ORT_CXX_API_THROW("invalid indice for Ort::Custom::Tensor", OrtErrorCode::ORT_INVALID_ARGUMENT); + } + auto const_value = ctx_.GetInput(indice); + auto type_shape_info = const_value.GetTensorTypeAndShapeInfo(); + shape_ = type_shape_info.GetShape(); + auto num_chars = const_value.GetStringTensorDataLength(); + chars_.resize(num_chars + 1, '\0'); + auto num_strings = static_cast(NumberOfElement()); + if (num_strings) { + std::vector offsets(num_strings); + const_value.GetStringTensorContent(static_cast(chars_.data()), num_chars, offsets.data(), offsets.size()); + offsets.push_back(num_chars); + for (size_t i = 0; i < num_strings; ++i) { + input_string_views_.emplace_back(chars_.data() + offsets[i], offsets[i + 1] - offsets[i]); + } + } + } + } + const string_views& Data() const { + return input_string_views_; + } + const void* DataRaw() const override { + if (input_string_views_.size() != 1) { + ORT_CXX_API_THROW("DataRaw() only applies to string scalar", ORT_RUNTIME_EXCEPTION); + } + return reinterpret_cast(input_string_views_[0].data()); + } + size_t SizeInBytes() const override { + if (input_string_views_.size() != 1) { + ORT_CXX_API_THROW("SizeInBytes() only applies to string scalar", ORT_RUNTIME_EXCEPTION); + } + return input_string_views_[0].size(); + } + void SetStringOutput(const strings& ss, const std::vector& dims) { + shape_ = dims; + std::vector raw; + for (const auto& s : ss) { + raw.push_back(s.data()); + } + auto output = ctx_.GetOutput(indice_, dims.data(), dims.size()); + // note - there will be copy ... + output.FillStringTensor(raw.data(), raw.size()); + } + const Span& AsSpan() { + ORT_CXX_API_THROW("span for TensorT of string view not implemented", OrtErrorCode::ORT_RUNTIME_EXCEPTION); + } + std::string_view AsScalar() { + if (input_string_views_.size() != 1) { + ORT_CXX_API_THROW("invalid shape while trying to get a scalar string view from Ort::Custom::Tensor", + OrtErrorCode::ORT_RUNTIME_EXCEPTION); + } + return input_string_views_[0]; + } + + private: + std::vector chars_; // for input + std::vector input_string_views_; // for input +}; + +using TensorPtr = std::unique_ptr; +using TensorPtrs = std::vector; + +struct TensorArray : public ArgBase { + TensorArray(OrtKernelContext* ctx, + size_t start_indice, + bool is_input) : ArgBase(ctx, + start_indice, + is_input) { + if (is_input) { + auto input_count = ctx_.GetInputCount(); + for (size_t ith_input = start_indice; ith_input < input_count; ++ith_input) { + auto const_value = ctx_.GetInput(start_indice); + auto type_shape_info = const_value.GetTensorTypeAndShapeInfo(); + auto type = type_shape_info.GetElementType(); + TensorPtr tensor; + switch (type) { + case ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL: + tensor = std::make_unique>(ctx, ith_input, true); + break; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT: + tensor = std::make_unique>(ctx, ith_input, true); + break; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE: + tensor = std::make_unique>(ctx, ith_input, true); + break; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8: + tensor = std::make_unique>(ctx, ith_input, true); + break; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8: + tensor = std::make_unique>(ctx, ith_input, true); + break; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16: + tensor = std::make_unique>(ctx, ith_input, true); + break; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16: + tensor = std::make_unique>(ctx, ith_input, true); + break; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32: + tensor = std::make_unique>(ctx, ith_input, true); + break; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32: + tensor = std::make_unique>(ctx, ith_input, true); + break; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64: + tensor = std::make_unique>(ctx, ith_input, true); + break; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64: + tensor = std::make_unique>(ctx, ith_input, true); + break; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING: + tensor = std::make_unique>(ctx, ith_input, true); + break; + default: + ORT_CXX_API_THROW("unknown input type", ORT_RUNTIME_EXCEPTION); + break; + } + tensors_.emplace_back(tensor.release()); + } // for + } + } + template + T* AllocateOutput(size_t ith_output, const std::vector& shape) { + // ith_output is the indice of output relative to the tensor array + // indice_ + ith_output is the indice relative to context + auto tensor = std::make_unique>(ctx_.GetOrtKernelContext(), indice_ + ith_output, false); + auto raw_output = tensor.get()->Allocate(shape); + tensors_.emplace_back(tensor.release()); + return raw_output; + } + Tensor& AllocateStringTensor(size_t ith_output) { + // ith_output is the indice of output relative to the tensor array + // indice_ + ith_output is the indice relative to context + auto tensor = std::make_unique>(ctx_.GetOrtKernelContext(), indice_ + ith_output, false); + Tensor& output = *tensor; + tensors_.emplace_back(tensor.release()); + return output; + } + size_t Size() const { + return tensors_.size(); + } + const TensorPtr& operator[](size_t ith_input) const { + // ith_input is the indice of output relative to the tensor array + return tensors_.at(ith_input); + } + + private: + TensorPtrs tensors_; +}; + +using Variadic = TensorArray; + +/* +Note: +OrtLiteCustomOp inherits from OrtCustomOp to bridge tween a custom func/struct and ort core. +The lifetime of an OrtLiteCustomOp instance is managed by customer code, not ort, so: +1. DO NOT cast OrtLiteCustomOp to OrtCustomOp and release since there is no virtual destructor in the hierarchy. +2. OrtLiteCustomFunc and OrtLiteCustomStruct, as two sub-structs, can be released in form of OrtLiteCustomOp since all members are kept in the OrtLiteCustomOp, + hence memory could still be recycled properly. +Further, OrtCustomOp is a c struct bearing no v-table, so offspring structs are by design to be of zero virtual functions to maintain cast safety. +*/ +struct OrtLiteCustomOp : public OrtCustomOp { + using ConstOptionalFloatTensor = std::optional&>; + using OptionalFloatTensor = std::optional>; + + // CreateTuple + template + static typename std::enable_if>::type + CreateTuple(OrtKernelContext*, ArgPtrs&, size_t, size_t, const std::string&) { + return std::make_tuple(); + } + + template + static typename std::enable_if::value, std::tuple>::type + CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) { + std::tuple current = std::tuple{context}; + auto next = CreateTuple(context, args, num_input, num_output, ep); + return std::tuple_cat(current, next); + } + + template + static typename std::enable_if::value, std::tuple>::type + CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) { + std::tuple current = std::tuple{*context}; + auto next = CreateTuple(context, args, num_input, num_output, ep); + return std::tuple_cat(current, next); + } + +#ifdef ORT_CUDA_CTX + template + static typename std::enable_if::value, std::tuple>::type + CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) { + thread_local CudaContext cuda_context; + cuda_context.Init(*context); + std::tuple current = std::tuple{cuda_context}; + auto next = CreateTuple(context, args, num_input, num_output, ep); + return std::tuple_cat(current, next); + } +#endif + + template + static typename std::enable_if::value, std::tuple>::type + CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) { + args.push_back(std::make_unique(context, ith_input, true)); + std::tuple current = std::tuple{reinterpret_cast(args.back().get())}; + auto next = CreateTuple(context, args, num_input, num_output, ep); + return std::tuple_cat(current, next); + } + + template + static typename std::enable_if::value, std::tuple>::type + CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) { + args.push_back(std::make_unique(context, ith_input, true)); + std::tuple current = std::tuple{reinterpret_cast(*args.back().get())}; + auto next = CreateTuple(context, args, num_input, num_output, ep); + return std::tuple_cat(current, next); + } + + template + static typename std::enable_if::value, std::tuple>::type + CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) { + args.push_back(std::make_unique(context, ith_output, false)); + std::tuple current = std::tuple{reinterpret_cast(args.back().get())}; + auto next = CreateTuple(context, args, num_input, num_output, ep); + return std::tuple_cat(current, next); + } + + template + static typename std::enable_if::value, std::tuple>::type + CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) { + args.push_back(std::make_unique(context, ith_output, false)); + std::tuple current = std::tuple{reinterpret_cast(*args.back().get())}; + auto next = CreateTuple(context, args, num_input, num_output, ep); + return std::tuple_cat(current, next); + } + +#define CREATE_TUPLE_INPUT(data_type) \ + template \ + static typename std::enable_if*>::value, std::tuple>::type \ + CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) { \ + args.push_back(std::make_unique>(context, ith_input, true)); \ + std::tuple current = std::tuple{reinterpret_cast(args.back().get())}; \ + auto next = CreateTuple(context, args, num_input, num_output, ep); \ + return std::tuple_cat(current, next); \ + } \ + template \ + static typename std::enable_if&>::value, std::tuple>::type \ + CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) { \ + args.push_back(std::make_unique>(context, ith_input, true)); \ + std::tuple current = std::tuple{reinterpret_cast(*args.back().get())}; \ + auto next = CreateTuple(context, args, num_input, num_output, ep); \ + return std::tuple_cat(current, next); \ + } \ + template \ + static typename std::enable_if*>>::value, std::tuple>::type \ + CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) { \ + if (ith_input < num_input) { \ + args.push_back(std::make_unique>(context, ith_input, true)); \ + std::tuple current = std::tuple{reinterpret_cast*>(args.back().get())}; \ + auto next = CreateTuple(context, args, num_input, num_output, ep); \ + return std::tuple_cat(current, next); \ + } else { \ + std::tuple current = std::tuple{}; \ + auto next = CreateTuple(context, args, num_input, num_output, ep); \ + return std::tuple_cat(current, next); \ + } \ + } \ + template \ + static typename std::enable_if*>::value, std::tuple>::type \ + CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) { \ + if ("CPUExecutionProvider" != ep) { \ + ORT_CXX_API_THROW("span input could only be applied to CPU EP", OrtErrorCode::ORT_RUNTIME_EXCEPTION); \ + } \ + args.push_back(std::make_unique>(context, ith_input, true)); \ + std::tuple current = std::tuple{&reinterpret_cast*>(args.back().get())->AsSpan()}; \ + auto next = CreateTuple(context, args, num_input, num_output, ep); \ + return std::tuple_cat(current, next); \ + } \ + template \ + static typename std::enable_if&>::value, std::tuple>::type \ + CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) { \ + if ("CPUExecutionProvider" != ep) { \ + ORT_CXX_API_THROW("span input could only be applied to CPU EP", OrtErrorCode::ORT_RUNTIME_EXCEPTION); \ + } \ + args.push_back(std::make_unique>(context, ith_input, true)); \ + std::tuple current = std::tuple{reinterpret_cast*>(args.back().get())->AsSpan()}; \ + auto next = CreateTuple(context, args, num_input, num_output, ep); \ + return std::tuple_cat(current, next); \ + } \ + template \ + static typename std::enable_if*>>::value, std::tuple>::type \ + CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) { \ + if (ith_input < num_input) { \ + if ("CPUExecutionProvider" != ep) { \ + ORT_CXX_API_THROW("span input could only be applied to CPU EP", OrtErrorCode::ORT_RUNTIME_EXCEPTION); \ + } \ + args.push_back(std::make_unique>(context, ith_input, true)); \ + std::tuple current = std::tuple{&reinterpret_cast*>(args.back().get())->AsSpan()}; \ + auto next = CreateTuple(context, args, num_input, num_output, ep); \ + return std::tuple_cat(current, next); \ + } else { \ + std::tuple current = std::tuple{}; \ + auto next = CreateTuple(context, args, num_input, num_output, ep); \ + return std::tuple_cat(current, next); \ + } \ + } \ + template \ + static typename std::enable_if::value, std::tuple>::type \ + CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) { \ + if ("CPUExecutionProvider" != ep) { \ + ORT_CXX_API_THROW("scalar input could only be applied to CPU EP", OrtErrorCode::ORT_RUNTIME_EXCEPTION); \ + } \ + args.push_back(std::make_unique>(context, ith_input, true)); \ + std::tuple current = std::tuple{reinterpret_cast*>(args.back().get())->AsScalar()}; \ + auto next = CreateTuple(context, args, num_input, num_output, ep); \ + return std::tuple_cat(current, next); \ + } \ + template \ + static typename std::enable_if>::value, std::tuple>::type \ + CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) { \ + if (ith_input < num_input) { \ + if ("CPUExecutionProvider" != ep) { \ + ORT_CXX_API_THROW("scalar input could only be applied to CPU EP", OrtErrorCode::ORT_RUNTIME_EXCEPTION); \ + } \ + args.push_back(std::make_unique>(context, ith_input, true)); \ + std::tuple current = std::tuple{reinterpret_cast*>(args.back().get())->AsScalar()}; \ + auto next = CreateTuple(context, args, num_input, num_output, ep); \ + return std::tuple_cat(current, next); \ + } else { \ + std::tuple current = std::tuple{}; \ + auto next = CreateTuple(context, args, num_input, num_output, ep); \ + return std::tuple_cat(current, next); \ + } \ + } +#define CREATE_TUPLE_OUTPUT(data_type) \ + template \ + static typename std::enable_if*>::value, std::tuple>::type \ + CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) { \ + args.push_back(std::make_unique>(context, ith_output, false)); \ + std::tuple current = std::tuple{reinterpret_cast(args.back().get())}; \ + auto next = CreateTuple(context, args, num_input, num_output, ep); \ + return std::tuple_cat(current, next); \ + } \ + template \ + static typename std::enable_if&>::value, std::tuple>::type \ + CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) { \ + args.push_back(std::make_unique>(context, ith_output, false)); \ + std::tuple current = std::tuple{reinterpret_cast(*args.back().get())}; \ + auto next = CreateTuple(context, args, num_input, num_output, ep); \ + return std::tuple_cat(current, next); \ + } \ + template \ + static typename std::enable_if*>>::value, std::tuple>::type \ + CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) { \ + if (ith_output < num_output) { \ + args.push_back(std::make_unique>(context, ith_output, false)); \ + std::tuple current = std::tuple{reinterpret_cast*>(args.back().get())}; \ + auto next = CreateTuple(context, args, num_input, num_output, ep); \ + return std::tuple_cat(current, next); \ + } else { \ + std::tuple current = std::tuple{}; \ + auto next = CreateTuple(context, args, num_input, num_output, ep); \ + return std::tuple_cat(current, next); \ + } \ + } +#define CREATE_TUPLE(data_type) \ + CREATE_TUPLE_INPUT(data_type) \ + CREATE_TUPLE_OUTPUT(data_type) + + CREATE_TUPLE(bool) + CREATE_TUPLE(float) + CREATE_TUPLE(Ort::Float16_t) + CREATE_TUPLE(Ort::BFloat16_t) + CREATE_TUPLE(double) + CREATE_TUPLE(int8_t) + CREATE_TUPLE(int16_t) + CREATE_TUPLE(int32_t) + CREATE_TUPLE(int64_t) + CREATE_TUPLE(uint8_t) + CREATE_TUPLE(uint16_t) + CREATE_TUPLE(uint32_t) + CREATE_TUPLE(uint64_t) + CREATE_TUPLE(std::string) + CREATE_TUPLE_INPUT(std::string_view) + CREATE_TUPLE(Ort::Float8E4M3FN_t) + CREATE_TUPLE(Ort::Float8E4M3FNUZ_t) + CREATE_TUPLE(Ort::Float8E5M2_t) + CREATE_TUPLE(Ort::Float8E5M2FNUZ_t) + + // ParseArgs ... + template + static typename std::enable_if<0 == sizeof...(Ts)>::type + ParseArgs(std::vector&, std::vector&) { + } + + template + static typename std::enable_if<0 <= sizeof...(Ts) && std::is_same::value>::type + ParseArgs(std::vector& input_types, std::vector& output_types) { + ParseArgs(input_types, output_types); + } + + template + static typename std::enable_if<0 <= sizeof...(Ts) && std::is_same::value>::type + ParseArgs(std::vector& input_types, std::vector& output_types) { + ParseArgs(input_types, output_types); + } + +#ifdef ORT_CUDA_CTX + template + static typename std::enable_if<0 <= sizeof...(Ts) && std::is_same::value>::type + ParseArgs(std::vector& input_types, std::vector& output_types) { + ParseArgs(input_types, output_types); + } +#endif + + template + static typename std::enable_if<0 <= sizeof...(Ts) && std::is_same::value>::type + ParseArgs(std::vector& input_types, std::vector& output_types) { + input_types.push_back(ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED); + ParseArgs(input_types, output_types); + } + + template + static typename std::enable_if<0 <= sizeof...(Ts) && std::is_same::value>::type + ParseArgs(std::vector& input_types, std::vector& output_types) { + input_types.push_back(ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED); + ParseArgs(input_types, output_types); + } + + template + static typename std::enable_if<0 <= sizeof...(Ts) && std::is_same::value>::type + ParseArgs(std::vector& input_types, std::vector& output_types) { + output_types.push_back(ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED); + ParseArgs(input_types, output_types); + } + + template + static typename std::enable_if<0 <= sizeof...(Ts) && std::is_same::value>::type + ParseArgs(std::vector& input_types, std::vector& output_types) { + output_types.push_back(ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED); + ParseArgs(input_types, output_types); + } + +#define PARSE_INPUT_BASE(pack_type, onnx_type) \ + template \ + static typename std::enable_if<0 <= sizeof...(Ts) && std::is_same::value>::type \ + ParseArgs(std::vector& input_types, std::vector& output_types) { \ + input_types.push_back(onnx_type); \ + ParseArgs(input_types, output_types); \ + } \ + template \ + static typename std::enable_if<0 <= sizeof...(Ts) && std::is_same>::value>::type \ + ParseArgs(std::vector& input_types, std::vector& output_types) { \ + input_types.push_back(onnx_type); \ + ParseArgs(input_types, output_types); \ + } \ + template \ + static typename std::enable_if<0 <= sizeof...(Ts) && std::is_same>::value>::type \ + ParseArgs(std::vector& input_types, std::vector& output_types) { \ + input_types.push_back(onnx_type); \ + ParseArgs(input_types, output_types); \ + } + +#define PARSE_INPUT(data_type, onnx_type) \ + PARSE_INPUT_BASE(const Custom::Tensor*, onnx_type) \ + PARSE_INPUT_BASE(const Custom::Tensor&, onnx_type) \ + PARSE_INPUT_BASE(const Custom::Span*, onnx_type) \ + PARSE_INPUT_BASE(const Custom::Span&, onnx_type) \ + PARSE_INPUT_BASE(data_type, onnx_type) + +#define PARSE_OUTPUT(data_type, onnx_type) \ + template \ + static typename std::enable_if<0 <= sizeof...(Ts) && std::is_same*>::value>::type \ + ParseArgs(std::vector& input_types, std::vector& output_types) { \ + output_types.push_back(onnx_type); \ + ParseArgs(input_types, output_types); \ + } \ + template \ + static typename std::enable_if<0 <= sizeof...(Ts) && std::is_same&>::value>::type \ + ParseArgs(std::vector& input_types, std::vector& output_types) { \ + output_types.push_back(onnx_type); \ + ParseArgs(input_types, output_types); \ + } \ + template \ + static typename std::enable_if<0 <= sizeof...(Ts) && std::is_same*>>::value>::type \ + ParseArgs(std::vector& input_types, std::vector& output_types) { \ + output_types.push_back(onnx_type); \ + ParseArgs(input_types, output_types); \ + } + +#define PARSE_ARGS(data_type, onnx_type) \ + PARSE_INPUT(data_type, onnx_type) \ + PARSE_OUTPUT(data_type, onnx_type) + + PARSE_ARGS(bool, ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL) + PARSE_ARGS(float, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT) + PARSE_ARGS(Ort::Float16_t, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16) + PARSE_ARGS(Ort::BFloat16_t, ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16) + PARSE_ARGS(double, ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE) + PARSE_ARGS(int8_t, ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8) + PARSE_ARGS(int16_t, ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16) + PARSE_ARGS(int32_t, ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32) + PARSE_ARGS(int64_t, ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64) + PARSE_ARGS(uint8_t, ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8) + PARSE_ARGS(uint16_t, ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16) + PARSE_ARGS(uint32_t, ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32) + PARSE_ARGS(uint64_t, ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64) + PARSE_ARGS(std::string, ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING) + PARSE_ARGS(std::string_view, ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING) // todo - remove string_view output + PARSE_ARGS(Ort::Float8E4M3FN_t, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E4M3FN) + PARSE_ARGS(Ort::Float8E4M3FNUZ_t, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E4M3FNUZ) + PARSE_ARGS(Ort::Float8E5M2_t, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E5M2) + PARSE_ARGS(Ort::Float8E5M2FNUZ_t, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E5M2FNUZ) + + OrtLiteCustomOp(const char* op_name, + const char* execution_provider, + ShapeInferFn shape_infer_fn, + int start_ver = 1, + int end_ver = MAX_CUSTOM_OP_END_VER) : op_name_(op_name), + execution_provider_(execution_provider), + shape_infer_fn_(shape_infer_fn), + start_ver_(start_ver), + end_ver_(end_ver) { + OrtCustomOp::version = ORT_API_VERSION; + + OrtCustomOp::GetName = [](const OrtCustomOp* op) { return static_cast(op)->op_name_.c_str(); }; + OrtCustomOp::GetExecutionProviderType = [](const OrtCustomOp* op) { return ((OrtLiteCustomOp*)op)->execution_provider_.c_str(); }; + OrtCustomOp::GetInputMemoryType = [](const OrtCustomOp*, size_t) { return OrtMemTypeDefault; }; + + OrtCustomOp::GetInputTypeCount = [](const OrtCustomOp* op) { + auto self = reinterpret_cast(op); + return self->input_types_.size(); + }; + + OrtCustomOp::GetInputType = [](const OrtCustomOp* op, size_t indice) { + auto self = reinterpret_cast(op); + return self->input_types_[indice]; + }; + + OrtCustomOp::GetOutputTypeCount = [](const OrtCustomOp* op) { + auto self = reinterpret_cast(op); + return self->output_types_.size(); + }; + + OrtCustomOp::GetOutputType = [](const OrtCustomOp* op, size_t indice) { + auto self = reinterpret_cast(op); + return self->output_types_[indice]; + }; + + OrtCustomOp::GetInputCharacteristic = [](const OrtCustomOp* op, size_t indice) { + auto self = reinterpret_cast(op); + return self->input_types_[indice] == ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED ? INPUT_OUTPUT_VARIADIC : INPUT_OUTPUT_OPTIONAL; + }; + + OrtCustomOp::GetOutputCharacteristic = [](const OrtCustomOp* op, size_t indice) { + auto self = reinterpret_cast(op); + return self->output_types_[indice] == ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED ? INPUT_OUTPUT_VARIADIC : INPUT_OUTPUT_OPTIONAL; + }; + + OrtCustomOp::GetVariadicInputMinArity = [](const OrtCustomOp*) { + return 1; + }; + + OrtCustomOp::GetVariadicInputHomogeneity = [](const OrtCustomOp*) { + return 0; + }; + + OrtCustomOp::GetVariadicOutputMinArity = [](const OrtCustomOp*) { + return 1; + }; + + OrtCustomOp::GetVariadicOutputHomogeneity = [](const OrtCustomOp*) { + return 0; + }; + + OrtCustomOp::GetVariadicInputMinArity = [](const OrtCustomOp*) { return 0; }; + OrtCustomOp::GetVariadicInputHomogeneity = [](const OrtCustomOp*) { return 0; }; + OrtCustomOp::GetVariadicOutputMinArity = [](const OrtCustomOp*) { return 0; }; + OrtCustomOp::GetVariadicOutputHomogeneity = [](const OrtCustomOp*) { return 0; }; + + OrtCustomOp::CreateKernelV2 = {}; + OrtCustomOp::KernelComputeV2 = {}; + OrtCustomOp::KernelCompute = {}; + + OrtCustomOp::InferOutputShapeFn = {}; + + OrtCustomOp::GetStartVersion = [](const OrtCustomOp* op) { + auto self = reinterpret_cast(op); + return self->start_ver_; + }; + + OrtCustomOp::GetEndVersion = [](const OrtCustomOp* op) { + auto self = reinterpret_cast(op); + return self->end_ver_; + }; + + OrtCustomOp::GetMayInplace = {}; + OrtCustomOp::ReleaseMayInplace = {}; + OrtCustomOp::GetAliasMap = {}; + OrtCustomOp::ReleaseAliasMap = {}; + } + + const std::string op_name_; + const std::string execution_provider_; + + std::vector input_types_; + std::vector output_types_; + + ShapeInferFn shape_infer_fn_ = {}; + + int start_ver_ = 1; + int end_ver_ = MAX_CUSTOM_OP_END_VER; + + void* compute_fn_ = {}; + void* compute_fn_return_status_ = {}; +}; + +//////////////////////////// OrtLiteCustomFunc //////////////////////////////// +// The struct is to implement function-as-op. +// E.g. a function might be defined as: +// void Filter(const Ort::Custom::Tensor& floats_in, Ort::Custom::Tensor& floats_out) { ... } +// It could be registered this way: +// Ort::CustomOpDomain v2_domain{"v2"}; +// std::unique_ptr fil_op_ptr{Ort::Custom::CreateLiteCustomOp("Filter", "CPUExecutionProvider", Filter)}; +// v2_domain.Add(fil_op_ptr.get()); +// session_options.Add(v2_domain); +// For the complete example, please search keyword "LiteCustomOpTest" under "/onnxruntime/test/". +template +struct OrtLiteCustomFunc : public OrtLiteCustomOp { + using ComputeFn = void (*)(Args...); + using ComputeFnReturnStatus = Status (*)(Args...); + using MyType = OrtLiteCustomFunc; + + struct Kernel { + size_t num_input_{}; + size_t num_output_{}; + ComputeFn compute_fn_{}; + ComputeFnReturnStatus compute_fn_return_status_{}; + std::string ep_{}; + }; + + OrtLiteCustomFunc(const char* op_name, + const char* execution_provider, + ComputeFn compute_fn, + ShapeInferFn shape_infer_fn = {}, + int start_ver = 1, + int end_ver = MAX_CUSTOM_OP_END_VER) : OrtLiteCustomOp(op_name, execution_provider, shape_infer_fn, start_ver, end_ver) { + compute_fn_ = reinterpret_cast(compute_fn); + ParseArgs(input_types_, output_types_); + + OrtCustomOp::KernelCompute = [](void* op_kernel, OrtKernelContext* context) { + auto kernel = reinterpret_cast(op_kernel); + std::vector args; + auto t = CreateTuple<0, 0, Args...>(context, args, kernel->num_input_, kernel->num_output_, kernel->ep_); + std::apply([kernel](Args const&... t_args) { kernel->compute_fn_(t_args...); }, t); + }; + + OrtCustomOp::CreateKernel = [](const OrtCustomOp* this_, const OrtApi* ort_api, const OrtKernelInfo* info) { + auto kernel = std::make_unique(); + auto me = static_cast(this_); + kernel->compute_fn_ = reinterpret_cast(me->compute_fn_); + Ort::ThrowOnError(ort_api->KernelInfo_GetInputCount(info, &kernel->num_input_)); + Ort::ThrowOnError(ort_api->KernelInfo_GetOutputCount(info, &kernel->num_output_)); + auto self = static_cast(this_); + kernel->ep_ = self->execution_provider_; + return reinterpret_cast(kernel.release()); + }; + + OrtCustomOp::KernelDestroy = [](void* op_kernel) { + delete reinterpret_cast(op_kernel); + }; + + if (shape_infer_fn_) { + OrtCustomOp::InferOutputShapeFn = [](const OrtCustomOp* op, OrtShapeInferContext* ort_ctx) -> OrtStatusPtr { + auto shape_info_fn = static_cast(op)->shape_infer_fn_; + ShapeInferContext ctx(&GetApi(), ort_ctx); + return shape_info_fn(ctx); + }; + } + } + + OrtLiteCustomFunc(const char* op_name, + const char* execution_provider, + ComputeFnReturnStatus compute_fn_return_status, + ShapeInferFn shape_infer_fn = {}, + int start_ver = 1, + int end_ver = MAX_CUSTOM_OP_END_VER) : OrtLiteCustomOp(op_name, execution_provider, shape_infer_fn, start_ver, end_ver) { + compute_fn_return_status_ = reinterpret_cast(compute_fn_return_status); + ParseArgs(input_types_, output_types_); + + OrtCustomOp::KernelComputeV2 = [](void* op_kernel, OrtKernelContext* context) -> OrtStatusPtr { + auto kernel = reinterpret_cast(op_kernel); + std::vector args; + auto t = CreateTuple<0, 0, Args...>(context, args, kernel->num_input_, kernel->num_output_, kernel->ep_); + return std::apply([kernel](Args const&... t_args) { Status status = kernel->compute_fn_return_status_(t_args...); return status.release(); }, t); + }; + + OrtCustomOp::CreateKernel = [](const OrtCustomOp* this_, const OrtApi* ort_api, const OrtKernelInfo* info) { + auto kernel = std::make_unique(); + auto me = static_cast(this_); + kernel->compute_fn_return_status_ = reinterpret_cast(me->compute_fn_return_status_); + Ort::ThrowOnError(ort_api->KernelInfo_GetInputCount(info, &kernel->num_input_)); + Ort::ThrowOnError(ort_api->KernelInfo_GetOutputCount(info, &kernel->num_output_)); + auto self = static_cast(this_); + kernel->ep_ = self->execution_provider_; + return reinterpret_cast(kernel.release()); + }; + + OrtCustomOp::KernelDestroy = [](void* op_kernel) { + delete reinterpret_cast(op_kernel); + }; + + if (shape_infer_fn_) { + OrtCustomOp::InferOutputShapeFn = [](const OrtCustomOp* op, OrtShapeInferContext* ort_ctx) -> OrtStatusPtr { + auto shape_info_fn = static_cast(op)->shape_infer_fn_; + ShapeInferContext ctx(&GetApi(), ort_ctx); + return shape_info_fn(ctx); + }; + } + } +}; // struct OrtLiteCustomFunc + +/////////////////////////// OrtLiteCustomStruct /////////////////////////// +// The struct is to implement struct-as-op. +// E.g. a struct might be defined as: +// struct Merge { +// Merge(const OrtApi* ort_api, const OrtKernelInfo* info) {...} +// void Compute(const Ort::Custom::Tensor& strings_in, +// std::string_view string_in, +// Ort::Custom::Tensor* strings_out) {...} +// bool reverse_ = false; +// }; +// It could be registered this way: +// Ort::CustomOpDomain v2_domain{"v2"}; +// std::unique_ptr mrg_op_ptr{Ort::Custom::CreateLiteCustomOp("Merge", "CPUExecutionProvider")}; +// v2_domain.Add(mrg_op_ptr.get()); +// session_options.Add(v2_domain); +// For the complete example, please search keyword "LiteCustomOpTest" under "/onnxruntime/test/". +template +struct OrtLiteCustomStruct : public OrtLiteCustomOp { + template + using CustomComputeFn = void (CustomOp::*)(Args...); + + template + using CustomComputeFnReturnStatus = Status (CustomOp::*)(Args...); + + using MyType = OrtLiteCustomStruct; + + struct Kernel { + size_t num_input_{}; + size_t num_output_{}; + std::unique_ptr custom_op_; + std::string ep_{}; + }; + + OrtLiteCustomStruct(const char* op_name, + const char* execution_provider, + int start_ver = 1, + int end_ver = MAX_CUSTOM_OP_END_VER) : OrtLiteCustomOp(op_name, execution_provider, {}, start_ver, end_ver) { + SetCompute(&CustomOp::Compute); + + OrtCustomOp::CreateKernel = [](const OrtCustomOp* this_, const OrtApi* ort_api, const OrtKernelInfo* info) { + auto kernel = std::make_unique(); + Ort::ThrowOnError(ort_api->KernelInfo_GetInputCount(info, &kernel->num_input_)); + Ort::ThrowOnError(ort_api->KernelInfo_GetOutputCount(info, &kernel->num_output_)); + kernel->custom_op_ = std::make_unique(ort_api, info); + auto self = static_cast(this_); + kernel->ep_ = self->execution_provider_; + return reinterpret_cast(kernel.release()); + }; + + OrtCustomOp::KernelDestroy = [](void* op_kernel) { + delete reinterpret_cast(op_kernel); + }; + + SetShapeInfer(0); + } + + template + void SetCompute(CustomComputeFn) { + ParseArgs(input_types_, output_types_); + OrtCustomOp::KernelCompute = [](void* op_kernel, OrtKernelContext* context) { + auto kernel = reinterpret_cast(op_kernel); + ArgPtrs args; + auto t = CreateTuple<0, 0, Args...>(context, args, kernel->num_input_, kernel->num_output_, kernel->ep_); + std::apply([kernel](Args const&... t_args) { kernel->custom_op_->Compute(t_args...); }, t); + }; + } + + template + void SetCompute(CustomComputeFnReturnStatus) { + ParseArgs(input_types_, output_types_); + OrtCustomOp::KernelComputeV2 = [](void* op_kernel, OrtKernelContext* context) -> OrtStatusPtr { + auto kernel = reinterpret_cast(op_kernel); + ArgPtrs args; + auto t = CreateTuple<0, 0, Args...>(context, args, kernel->num_input_, kernel->num_output_, kernel->ep_); + return std::apply([kernel](Args const&... t_args) { Status status = kernel->custom_op_->Compute(t_args...); return status.release(); }, t); + }; + } + + template + decltype(&C::InferOutputShape) SetShapeInfer(decltype(&C::InferOutputShape)) { + OrtCustomOp::InferOutputShapeFn = [](const OrtCustomOp*, OrtShapeInferContext* ort_ctx) -> OrtStatusPtr { + ShapeInferContext ctx(&GetApi(), ort_ctx); + return C::InferOutputShape(ctx); + }; + return {}; + } + + template + void SetShapeInfer(...) { + OrtCustomOp::InferOutputShapeFn = {}; + } +}; // struct OrtLiteCustomStruct + +/////////////////////////// CreateLiteCustomOp //////////////////////////// + +template +OrtLiteCustomOp* CreateLiteCustomOp(const char* op_name, + const char* execution_provider, + void (*custom_compute_fn)(Args...), + Status (*shape_infer_fn)(ShapeInferContext&) = {}, + int start_ver = 1, + int end_ver = MAX_CUSTOM_OP_END_VER) { + using LiteOp = OrtLiteCustomFunc; + return std::make_unique(op_name, execution_provider, custom_compute_fn, shape_infer_fn, start_ver, end_ver).release(); +} + +template +OrtLiteCustomOp* CreateLiteCustomOp(const char* op_name, + const char* execution_provider, + Status (*custom_compute_fn_v2)(Args...), + Status (*shape_infer_fn)(ShapeInferContext&) = {}, + int start_ver = 1, + int end_ver = MAX_CUSTOM_OP_END_VER) { + using LiteOp = OrtLiteCustomFunc; + return std::make_unique(op_name, execution_provider, custom_compute_fn_v2, shape_infer_fn, start_ver, end_ver).release(); +} + +template +OrtLiteCustomOp* CreateLiteCustomOp(const char* op_name, + const char* execution_provider, + int start_ver = 1, + int end_ver = MAX_CUSTOM_OP_END_VER) { + using LiteOp = OrtLiteCustomStruct; + return std::make_unique(op_name, execution_provider, start_ver, end_ver).release(); +} + +} // namespace Custom +} // namespace Ort diff --git a/dist/include/onnxruntime/onnxruntime_run_options_config_keys.h b/dist/include/onnxruntime/onnxruntime_run_options_config_keys.h new file mode 100644 index 0000000..f40ea65 --- /dev/null +++ b/dist/include/onnxruntime/onnxruntime_run_options_config_keys.h @@ -0,0 +1,54 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +/* + * This file defines RunOptions Config Keys and format of the Config Values. + * + * The Naming Convention for a RunOptions Config Key, + * "[Area][.[SubArea1].[SubArea2]...].[Keyname]" + * Such as "ep.cuda.use_arena" + * The Config Key cannot be empty + * The maximum length of the Config Key is 128 + * + * The string format of a RunOptions Config Value is defined individually for each Config. + * The maximum length of the Config Value is 1024 + */ + +// Key for enabling shrinkages of user listed device memory arenas. +// Expects a list of semi-colon separated key value pairs separated by colon in the following format: +// "device_0:device_id_0;device_1:device_id_1" +// No white-spaces allowed in the provided list string. +// Currently, the only supported devices are : "cpu", "gpu" (case sensitive). +// If "cpu" is included in the list, DisableCpuMemArena() API must not be called (i.e.) arena for cpu should be enabled. +// Example usage: "cpu:0;gpu:0" (or) "gpu:0" +// By default, the value for this key is empty (i.e.) no memory arenas are shrunk +static const char* const kOrtRunOptionsConfigEnableMemoryArenaShrinkage = "memory.enable_memory_arena_shrinkage"; + +// Set to '1' to not synchronize execution providers with CPU at the end of session run. +// Per default it will be set to '0' +// Taking CUDA EP as an example, it omit triggering cudaStreamSynchronize on the compute stream. +static const char* const kOrtRunOptionsConfigDisableSynchronizeExecutionProviders = "disable_synchronize_execution_providers"; + +// Set HTP performance mode for QNN HTP backend before session run. +// options for HTP performance mode: "burst", "balanced", "default", "high_performance", +// "high_power_saver", "low_balanced", "extreme_power_saver", "low_power_saver", "power_saver", +// "sustained_high_performance". Default to "default". +static const char* const kOrtRunOptionsConfigQnnPerfMode = "qnn.htp_perf_mode"; + +// Set HTP performance mode for QNN HTP backend post session run. +static const char* const kOrtRunOptionsConfigQnnPerfModePostRun = "qnn.htp_perf_mode_post_run"; + +// Set RPC control latency for QNN HTP backend +static const char* const kOrtRunOptionsConfigQnnRpcControlLatency = "qnn.rpc_control_latency"; + +// Set QNN Lora Config File for apply Lora in QNN context binary +static const char* const kOrtRunOptionsConfigQnnLoraConfig = "qnn.lora_config"; + +// Set graph annotation id for CUDA EP. Use with enable_cuda_graph=true. +// The value should be an integer. If the value is not set, the default value is 0 and +// ORT session only captures one cuda graph before another capture is requested. +// If the value is set to -1, cuda graph capture/replay is disabled in that run. +// User are not expected to set the value to 0 as it is reserved for internal use. +static const char* const kOrtRunOptionsConfigCudaGraphAnnotation = "gpu_graph_id"; diff --git a/dist/include/onnxruntime/onnxruntime_session_options_config_keys.h b/dist/include/onnxruntime/onnxruntime_session_options_config_keys.h new file mode 100644 index 0000000..abeb930 --- /dev/null +++ b/dist/include/onnxruntime/onnxruntime_session_options_config_keys.h @@ -0,0 +1,436 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +/* + * This file defines SessionOptions Config Keys and format of the Config Values. + * + * The Naming Convention for a SessionOptions Config Key, + * "[Area][.[SubArea1].[SubArea2]...].[Keyname]" + * Such as "ep.cuda.use_arena" + * The Config Key cannot be empty + * The maximum length of the Config Key is 1024 + * + * The string format of a SessionOptions Config Value is defined individually for each Config. + * The maximum length of the Config Value is 8192 + */ + +// Key for disable PrePacking, +// If the config value is set to "1" then the prepacking is disabled, otherwise prepacking is enabled (default value) +static const char* const kOrtSessionOptionsConfigDisablePrepacking = "session.disable_prepacking"; + +// A value of "1" means allocators registered in the env will be used. "0" means the allocators created in the session +// will be used. Use this to override the usage of env allocators on a per session level. +static const char* const kOrtSessionOptionsConfigUseEnvAllocators = "session.use_env_allocators"; + +// Set to 'ORT' (case sensitive) to load an ORT format model. +// If unset, model type will default to ONNX unless inferred from filename ('.ort' == ORT format) or bytes to be ORT +static const char* const kOrtSessionOptionsConfigLoadModelFormat = "session.load_model_format"; + +// Set to 'ORT' (case sensitive) to save optimized model in ORT format when SessionOptions.optimized_model_path is set. +// If unset, format will default to ONNX unless optimized_model_filepath ends in '.ort'. +static const char* const kOrtSessionOptionsConfigSaveModelFormat = "session.save_model_format"; + +// If a value is "1", flush-to-zero and denormal-as-zero are applied. The default is "0". +// When multiple sessions are created, a main thread doesn't override changes from succeeding session options, +// but threads in session thread pools follow option changes. +// When ORT runs with OpenMP, the same rule is applied, i.e. the first session option to flush-to-zero and +// denormal-as-zero is only applied to global OpenMP thread pool, which doesn't support per-session thread pool. +// Note that an alternative way not using this option at runtime is to train and export a model without denormals +// and that's recommended because turning this option on may hurt model accuracy. +static const char* const kOrtSessionOptionsConfigSetDenormalAsZero = "session.set_denormal_as_zero"; + +// It controls to run quantization model in QDQ (QuantizelinearDeQuantizelinear) format or not. +// "0": enable. ORT does fusion logic for QDQ format. +// "1": disable. ORT doesn't do fusion logic for QDQ format. +// Its default value is "0" unless the DirectML execution provider is registered, in which case it defaults to "1". +static const char* const kOrtSessionOptionsDisableQuantQDQ = "session.disable_quant_qdq"; + +// It controls whether to enable Double QDQ remover and Identical Children Consolidation +// "0": not to disable. ORT does remove the middle 2 Nodes from a Q->(QD->Q)->QD pairs +// "1": disable. ORT doesn't remove the middle 2 Nodes from a Q->(QD->Q)->QD pairs +// Its default value is "0" +static const char* const kOrtSessionOptionsDisableDoubleQDQRemover = "session.disable_double_qdq_remover"; + +// If set to "1", enables the removal of QuantizeLinear/DequantizeLinear node pairs once all QDQ handling has been +// completed. e.g. If after all QDQ handling has completed and we have -> FloatOp -> Q -> DQ -> FloatOp -> the +// Q -> DQ could potentially be removed. This will provide a performance benefit by avoiding going from float to +// 8-bit and back to float, but could impact accuracy. The impact on accuracy will be model specific and depend on +// other factors like whether the model was created using Quantization Aware Training or Post Training Quantization. +// As such, it's best to test to determine if enabling this works well for your scenario. +// The default value is "0" +// Available since version 1.11. +static const char* const kOrtSessionOptionsEnableQuantQDQCleanup = "session.enable_quant_qdq_cleanup"; + +// Enable or disable gelu approximation in graph optimization. "0": disable; "1": enable. The default is "0". +// GeluApproximation has side effects which may change the inference results. It is disabled by default due to this. +static const char* const kOrtSessionOptionsEnableGeluApproximation = "optimization.enable_gelu_approximation"; + +// Enable or disable Cast chain elimination in graph optimization. "0": disable; "1": enable. The default is "0". +// CastElimination with chain elimination has side effects which may change the inference results. It is disabled by default due to this. +static const char* const kOrtSessionOptionsEnableCastChainElimination = "optimization.enable_cast_chain_elimination"; + +// This setting controls whether to enable AheadOfTime function inlining. +// AOT function inlining examines the graph and attempts to inline as many locally defined functions in the model +// as possible with the help of enabled execution providers. +// This can reduce the number of function calls and improve performance because it is done before +// Level1 optimizers and constant folding. However, under some circumstances, when the EPs are not available, +// one can disable the AOT inlining, produce an optimized model and postpone AOT until run time. +// "0": enable; "1": disable. +// Its default value is "0". +static const char* const kOrtSessionOptionsDisableAheadOfTimeFunctionInlining = "session.disable_aot_function_inlining"; + +#ifdef ENABLE_TRAINING +// Specifies a path of the file containing a list of memory optimization configurations. +// The value should be a string indicating the file path of the config file. +// The content of the config file is a JSON struct like this: +// [ +// "Gelu+Cast+:1:0", +// "Dropout+:1:1" +// ] +// Taking the example of "Gelu+Cast+:1:0", +// > "Gelu+Cast+" is the subgraph string, a valid "subgraph string" should be one subgraph representation +// output by ORT graph transformations. +// > "1" is "optimization strategy", valid values: 0 - disabled, 1 - recompute. +// > "0" is "number of subgraph to apply" which is used to control how many subgraphs to apply optimization, +// to avoid "oversaving" the memory. +static const char* const kOrtSessionOptionsMemoryOptimizerApplyConfig = "optimization.memory_optimizer_config"; + +// Specifies the config for detecting subgraphs for memory footprint reduction. +// The value should be a string contains int separated using commas. The default value is "0:0". +static const char* const kOrtSessionOptionsMemoryOptimizerProbeConfig = "optimization.enable_memory_probe_recompute_config"; +#endif + +// This setting if set should contain a comma separated list of optimizers names that should be disabled. +// Optimizers may take time to execute and affect model loading time. If you feel that a specific optimizer +// does not provider runtime benefits, but affects your model loading time you may disable it using this config +// entry. This option is not enabled in ORT_MINIMAL_BUILD build. +// A list of optimizes is available in onnxruntime/core/optimizer/graph_transformer_utils.cc +// +// Default is an empty string which means no optimizers are disabled. +static const char* const kOrtSessionOptionsDisableSpecifiedOptimizers = "optimization.disable_specified_optimizers"; + +// It controls whether to run graph optimizations in loop or not. +// +// "0": disable. Graph Optimization Loop is disabled. +// ``` +// Level 2 --> Level 3 --> InsertCastTransforms --> Level 4 +// ^ | +// | "No Loop" | +// | | +// X xxxxxxxxxxx X +// ``` +// "1": enable. Graph Optimization Loop is enabled, such that, if optimizations at Level 4 are applied then +// the loop will check for any other valid optimization that can happen. +// ``` +// Level 2 --> Level 3 --> InsertCastTransforms --> Level 4 +// ^ | +// | "Loop only depending on Level 4" | +// | | +// --------------------------------------------------- +// ``` +// "2": enable. Graph Optimization Loop is enabled, such that, if optimizations at Level 2 or above are applied then +// The loop will check for any other valid optimization that can happen. +// ``` +// Level 2 --> Level 3 --> InsertCastTransforms --> Level 4 +// ^ | +// | "Loop" | +// | | +// --------------------------------------------------- +// ``` +// Default value is set to "1". +static const char* const kOrtSessionOptionsGraphOptimizationsLoopLevel = "session.graph_optimizations_loop_level"; + +// Enable or disable using device allocator for allocating initialized tensor memory. "1": enable; "0": disable. The default is "0". +// Using device allocators means the memory allocation is made using malloc/new. +static const char* const kOrtSessionOptionsUseDeviceAllocatorForInitializers = "session.use_device_allocator_for_initializers"; + +// Configure whether to allow the inter_op/intra_op threads spinning a number of times before blocking +// "0": thread will block if found no job to run +// "1": thread will spin a number of times before blocking +// The default is "0" when ORT is built with "ORT_CLIENT_PACKAGE_BUILD" and "1" otherwise. +// Thread spinning is disabled by default for client/on-device workloads to reduce cpu utilization and improve power efficiency. +static const char* const kOrtSessionOptionsConfigAllowInterOpSpinning = "session.inter_op.allow_spinning"; +static const char* const kOrtSessionOptionsConfigAllowIntraOpSpinning = "session.intra_op.allow_spinning"; + +// Key for using model bytes directly for ORT format +// If a session is created using an input byte array contains the ORT format model data, +// By default we will copy the model bytes at the time of session creation to ensure the model bytes +// buffer is valid. +// Setting this option to "1" will disable copy the model bytes, and use the model bytes directly. The caller +// has to guarantee that the model bytes are valid until the ORT session using the model bytes is destroyed. +static const char* const kOrtSessionOptionsConfigUseORTModelBytesDirectly = "session.use_ort_model_bytes_directly"; + +/// +/// Key for using the ORT format model flatbuffer bytes directly for initializers. +/// This avoids copying the bytes and reduces peak memory usage during model loading and initialization. +/// Requires `session.use_ort_model_bytes_directly` to be true. +/// If set, the flatbuffer bytes provided when creating the InferenceSession MUST remain valid for the entire +/// duration of the InferenceSession. +/// +static const char* const kOrtSessionOptionsConfigUseORTModelBytesForInitializers = + "session.use_ort_model_bytes_for_initializers"; + +// This should only be specified when exporting an ORT format model for use on a different platform. +// If the ORT format model will be used on ARM platforms set to "1". For other platforms set to "0" +// Available since version 1.11. +static const char* const kOrtSessionOptionsQDQIsInt8Allowed = "session.qdqisint8allowed"; + +// x64 SSE4.1/AVX2/AVX512(with no VNNI) has overflow problem with quantizied matrix multiplication with U8S8. +// To avoid this we need to use slower U8U8 matrix multiplication instead. This option, if +// turned on, use slower U8U8 matrix multiplications. Only effective with AVX2 or AVX512 +// platforms. +static const char* const kOrtSessionOptionsAvx2PrecisionMode = "session.x64quantprecision"; + +// Specifies how minimal build graph optimizations are handled in a full build. +// These optimizations are at the extended level or higher. +// Possible values and their effects are: +// "save": Save runtime optimizations when saving an ORT format model. +// "apply": Only apply optimizations available in a minimal build. +// ""/: Apply optimizations available in a full build. +// Available since version 1.11. +static const char* const kOrtSessionOptionsConfigMinimalBuildOptimizations = + "optimization.minimal_build_optimizations"; + +// Note: The options specific to an EP should be specified prior to appending that EP to the session options object in +// order for them to take effect. + +// Specifies a list of stop op types. Nodes of a type in the stop op types and nodes downstream from them will not be +// run by the NNAPI EP. +// The value should be a ","-delimited list of op types. For example, "Add,Sub". +// If not specified, the default set of stop ops is used. To specify an empty stop ops types list and disable stop op +// exclusion, set the value to "". +static const char* const kOrtSessionOptionsConfigNnapiEpPartitioningStopOps = "ep.nnapi.partitioning_stop_ops"; + +// Enabling dynamic block-sizing for multithreading. +// With a positive value, thread pool will split a task of N iterations to blocks of size starting from: +// N / (num_of_threads * dynamic_block_base) +// As execution progresses, the size will decrease according to the diminishing residual of N, +// meaning the task will be distributed in smaller granularity for better parallelism. +// For some models, it helps to reduce the variance of E2E inference latency and boost performance. +// The feature will not function by default, specify any positive integer, e.g. "4", to enable it. +// Available since version 1.11. +static const char* const kOrtSessionOptionsConfigDynamicBlockBase = "session.dynamic_block_base"; + +// This option allows to decrease CPU usage between infrequent +// requests and forces any TP threads spinning stop immediately when the last of +// concurrent Run() call returns. +// Spinning is restarted on the next Run() call. +// Applies only to internal thread-pools +static const char* const kOrtSessionOptionsConfigForceSpinningStop = "session.force_spinning_stop"; + +// "1": all inconsistencies encountered during shape and type inference +// will result in failures. +// "0": in some cases warnings will be logged but processing will continue. The default. +// May be useful to expose bugs in models. +static const char* const kOrtSessionOptionsConfigStrictShapeTypeInference = "session.strict_shape_type_inference"; + +// "1": every model using a more recent opset than the latest released one will fail +// "0": the model may or may not work if onnxruntime cannot find an implementation, this option +// is used for development purpose. +static const char* const kOrtSessionOptionsConfigStrictAllowReleasedOpsetsOnly = "session.allow_released_opsets_only"; + +// The file saves configuration for partitioning node among logic streams +static const char* const kNodePartitionConfigFile = "session.node_partition_config_file"; + +// This Option allows setting affinities for intra op threads. +// Affinity string follows format: +// logical_processor_id,logical_processor_id;logical_processor_id,logical_processor_id +// Semicolon isolates configurations among threads, while comma split processors where ith thread expected to attach to. +// e.g.1,2,3;4,5 +// specifies affinities for two threads, with the 1st thread attach to the 1st, 2nd, and 3rd processor, and 2nd thread to the 4th and 5th. +// To ease the configuration, an "interval" is also allowed: +// e.g. 1-8;8-16;17-24 +// orders that the 1st thread runs on first eight processors, 2nd thread runs on next eight processors, and so forth. +// Note: +// 1. Once set, the number of thread affinities must equal to intra_op_num_threads - 1, since ort does not set affinity on the main thread which +// is started and managed by the calling app; +// 2. For windows, ort will infer the group id from a logical processor id, for example, assuming there are two groups with each has 64 logical processors, +// an id of 64 will be inferred as the last processor of the 1st group, while 65 will be interpreted as the 1st processor of the second group. +// Hence 64-65 is an invalid configuration, because a windows thread cannot be attached to processors across group boundary. +static const char* const kOrtSessionOptionsConfigIntraOpThreadAffinities = "session.intra_op_thread_affinities"; + +// This option will dump out the model to assist debugging any issues with layout transformation, +// and is primarily intended for developer usage. It is only relevant if an execution provider that requests +// NHWC layout is enabled such as NNAPI, XNNPACK or QNN. +// +// Default is off. Set to "1" to enable. +// +// If modified by layout transformation the model will be dumped after these steps: +// 1) insertion of the layout transformation Transpose nodes +// 2) after those are optimized using the transpose optimizer, +// 3) after the L1 transformers are applied to the updated graph. +// The model will be saved to filename post_layout_transform_step_.onnx. +static const char* const kDebugLayoutTransformation = "session.debug_layout_transformation"; + +// Graph nodes that are not supported by the execution providers (EPs) explicitly added to the session are +// assigned (i.e., "fallback") to the CPU EP by default. +// +// This option allows the user to disable the fallback of unsupported graph nodes to the CPU EP. +// If this option is set to "1", session creation will fail if the execution providers other than the CPU EP cannot +// fully support all of the nodes in the graph. +// +// It is invalid to set this option and explicitly add the CPU EP to the session. In this case, session creation +// will also fail with an error. +// +// Option values: +// - "0": CPU EP fallback is not disabled. [DEFAULT] +// - "1": CPU EP fallback is disabled. +static const char* const kOrtSessionOptionsDisableCPUEPFallback = "session.disable_cpu_ep_fallback"; + +// Use this config when serializing a large model after optimization to specify an external initializers file +static const char* const kOrtSessionOptionsOptimizedModelExternalInitializersFileName = + "session.optimized_model_external_initializers_file_name"; + +// Use this config to control the minimum size of the initializer when externalizing it during serialization +static const char* const kOrtSessionOptionsOptimizedModelExternalInitializersMinSizeInBytes = + "session.optimized_model_external_initializers_min_size_in_bytes"; + +// When loading model from memory buffer and the model has external initializers +// Use this config to set the external data file folder path +// All external data files should be in the same folder +static const char* const kOrtSessionOptionsModelExternalInitializersFileFolderPath = + "session.model_external_initializers_file_folder_path"; + +// Use this config when saving pre-packed constant initializers to an external data file. +// This allows you to memory map pre-packed initializers on model load and leave it to +// to the OS the amount of memory consumed by the pre-packed initializers. Otherwise, +// pre-packed data resides on the heap. +// +// - "0": Default is not save pre-packed initializers to a data file. +// - "1": Save pre-packed constant initializers to an external data file. +// Sample usage: sess_options.add_session_config_entry(kOrtSessionOptionsSavePrePackedConstantInitializers, "1") +static const char* const kOrtSessionOptionsSavePrePackedConstantInitializers = + "session.save_external_prepacked_constant_initializers"; + +// Use this config when you want to collect memory stats for each node in the graph. +// The file format is a CSV file with the following columns: +// The file will be created if it does not exist, and will be overwritten if it does. +// +// The content of the file can be used to estimate memory requirements at run time including +// the temporary allocations. This operation is preferably done on a CPU device, as the model may exceed +// device memory limits in constrained environments. When enabling this option, it is important to disable +// memory patterns, as they tend to allocate large blocks to avoid fragmentation and accommodate needs of multiple +// kernels. Memory patterns may make it difficult to allocate on a device with limited memory. +// +// The collected stats then can be used to partition the graph among the devices in a way that only the +// required memory is allocated on each device. +// +// node_name, initializers_memory, dynamic_outputs_sizes, temp_allocations_size +// +// - "full path to file": there is not a default for this option. If the file can not be opened for writing, an error will be returned. +static const char* const kOrtSessionOptionsCollectNodeMemoryStatsToFile = "session.collect_node_memory_stats_to_file"; + +/// This is a composite CSV setting formatted as "memory limit in kb,file name for collected stats" +/// "limit > 0": enables Capacity Aware Partitioning for Cuda EP. `limit` is optional and when absent +/// the provider may attempt to figure out the memory available automatically. +/// The setting with no limit is expected to look like: ",file name for collected stats" +/// The EP will place nodes on device "file name" : +/// this file is expected to be found at the same folder with the model. The file contains +/// pre-recorded stats collected when running with kOrtSessionOptionsCollectNodeMemoryStatsToFile enforce (see above) +static const char* const kOrtSessionOptionsResourceCudaPartitioningSettings = + "session.resource_cuda_partitioning_settings"; + +// Enable EP context feature to dump the partitioned graph which includes the EP context into Onnx file. +// The dumped Onnx model with EP context can be used for future inference to avoid the EP graph partitioning/compile overhead. +// "0": disable. (default) +// "1": enable. +static const char* const kOrtSessionOptionEpContextEnable = "ep.context_enable"; + +// Specify the file path for the Onnx model which has EP context. +// Default to original_file_name_ctx.onnx if not specified +// Folder is not a valid option +static const char* const kOrtSessionOptionEpContextFilePath = "ep.context_file_path"; + +// Flag to specify whether to dump the EP context into the Onnx model. +// "0": dump the EP context into separate file, keep the file name in the Onnx model. (default). +// "1": dump the EP context into the Onnx model. +static const char* const kOrtSessionOptionEpContextEmbedMode = "ep.context_embed_mode"; + +// Specify the EPContext node name prefix to make it unique +// in case user need to merge/connect multiple EPContext nodes in one model +static const char* const kOrtSessionOptionEpContextNodeNamePrefix = "ep.context_node_name_prefix"; + +// Share EP related resources across sessions +static const char* const kOrtSessionOptionShareEpContexts = "ep.share_ep_contexts"; + +// Stop to share EP related resources across sessions from then on +static const char* const kOrtSessionOptionStopShareEpContexts = "ep.stop_share_ep_contexts"; + +// Used only for context model generation. +// This configuration is used when some nodes are partitioned on the CPU EP and those nodes have external initializers. +// When generating the EP context model, the new model should not rely on the old external data file used by the source ONNX model. +// Use this setting when dumping the EP context model with an external initializers file. +// If specified, all initializers will be placed inside the external data file. +// Otherwise, all initializers will be embedded inside the generated ONNX file. +// By default, this option is not set, meaning all initializers will be included within the ONNX file. +static const char* const kOrtSessionOptionsEpContextModelExternalInitializersFileName = + "ep.context_model_external_initializers_file_name"; + +// Gemm fastmath mode provides fp32 gemm acceleration with bfloat16 based matmul. +// Option values: +// - "0": Gemm FastMath mode is not enabled. [DEFAULT] +// - "1": Gemm FastMath mode is enabled. +static const char* const kOrtSessionOptionsMlasGemmFastMathArm64Bfloat16 = "mlas.enable_gemm_fastmath_arm64_bfloat16"; + +// Use LUT (Lookup Table) based GEMM for quantized models when available. +// Option values: +// - "0": Do not use LUT based GEMM. [DEFAULT] +// - "1": Use LUT based GEMM when available. +static const char* const kOrtSessionOptionsMlasLutGemm = "mlas.use_lut_gemm"; + +// When converting DQ + MatMul -> MatMulNBits, the accuracy level of the MatMulNBits is controlled by this option. +// Refer to MatMulNBits op schema for more details. +// If not provided, default is 4. +static const char* const kOrtSessionOptionsQDQMatMulNBitsAccuracyLevel = "session.qdq_matmulnbits_accuracy_level"; + +// Enable the DQ->MatMulNBits fusion graph transformer. +// "0": disabled (default). "1": enabled. +// This is typically set automatically by InferenceSession when the NvTensorRTRTX EP is registered. +static const char* const kOrtSessionOptionsEnableDQMatMulNBitsFusion = "session.enable_dq_matmulnbits_fusion"; + +// THIS OPTION IS NOT A REGULAR SESSION OPTION SINCE IT CAN BE MODIFIED AT ANY TIME +// Meant to be used with SetEpDynamicOptions +// Specify the type of workload for this session. +// "Default": OS determines the scheduling priority and processor performance to service this workload. [Default] +// "Efficient": OS treats this workload is efficiency oriented with low scheduling priority and efficient processor performance. +static const char* const kOrtEpDynamicOptionsWorkloadType = "ep.dynamic.workload_type"; + +// Disables model compilation during session initialization. +// +// If this option is set to "1", inference session creation will fail with error code ORT_MODEL_REQUIRES_COMPILATION +// if compilation is required to run the model on any Execution Provider added to the session. +// Only the following kinds of models are valid when this option is set to "1": +// - Pre-compiled models that have EPContext nodes for the compiling Execution Providers in the session. +// - Non-compiled models that run only on non-compiling Execution Providers, like CPU EP. +// +// See \href https://onnxruntime.ai/docs/execution-providers/EP-Context-Design.html for details about +// compiled models with EPContext nodes. +// +// Option values: +// - "0": EP compile is not disabled. [DEFAULT] +// - "1": EP compile is disabled. +static const char* const kOrtSessionOptionsDisableModelCompile = "session.disable_model_compile"; + +// Controls behavior when compiled model compatibility is SUPPORTED_PREFER_RECOMPILATION. +// "0": Allow execution with suboptimal performance. [DEFAULT] +// "1": Fail session creation to require recompilation for optimal performance. +// Note: UNSUPPORTED models always fail regardless of this setting. +static const char* const kOrtSessionOptionsFailOnSuboptimalCompiledModel = + "session.fail_on_suboptimal_compiled_model"; + +// THIS OPTION IS NOT A REGULAR SESSION OPTION SINCE IT CAN BE MODIFIED AT ANY TIME +// Meant to be used with SetEpDynamicOptions +// options for HTP performance mode: "burst", "balanced", "default", "high_performance", +// "high_power_saver", "low_balanced", "extreme_power_saver", "low_power_saver", "power_saver", +// "sustained_high_performance". Default to "default". +static const char* const kOrtEpDynamicOptionsQnnHtpPerformanceMode = "ep.dynamic.qnn_htp_performance_mode"; + +// Enables the session to record information about the subgraphs/nodes assigned to execution providers. +// When enabled, an application may call Session_GetEpGraphAssignmentInfo() to retrieve the information. +// +// Option values: +// - "0": Recording of EP graph assignment information is disabled. [DEFAULT] +// - "1": Recording of EP graph assignment information is enabled. +static const char* const kOrtSessionOptionsRecordEpGraphAssignmentInfo = "session.record_ep_graph_assignment_info"; diff --git a/dist/jni/kazeia_mel.cpp b/dist/jni/kazeia_mel.cpp new file mode 100644 index 0000000..c35ca21 --- /dev/null +++ b/dist/jni/kazeia_mel.cpp @@ -0,0 +1,236 @@ +// Implementation kazeia_mel — voir kazeia_mel.h pour l'API publique. +#include "kazeia_mel.h" + +#include +#include +#include +#include +#include + +namespace kazeia_mel { + +// ============================================================================ +// Configurations pré-faites +// ============================================================================ +MelConfig config_qwen3_tts_speaker() { + MelConfig c; + c.sample_rate = 24000; + c.n_fft = 1024; + c.hop_size = 256; + c.win_size = 1024; + c.n_mels = 128; + c.pad_amount = (1024 - 256) / 2; // 384, reflect (TTS convention) + c.fixed_n_frames = 0; + c.clip_eps = 1e-5f; + c.window = WindowKind::HANN_SYMMETRIC; + c.comp = Compression::LOG_NATURAL; + c.post = PostNorm::NONE; + return c; +} + +MelConfig config_whisper() { + MelConfig c; + c.sample_rate = 16000; + c.n_fft = 400; + c.hop_size = 160; + c.win_size = 400; + c.n_mels = 80; + c.pad_amount = 200; // n_fft/2 (HF whisper) + c.fixed_n_frames = 3000; // 30s padding fixe + c.clip_eps = 1e-10f; + c.window = WindowKind::HANN_PERIODIC; + c.comp = Compression::LOG10; + c.post = PostNorm::WHISPER; + return c; +} + +// ============================================================================ +// Helpers internes +// ============================================================================ + +// FFT radix-2 in-place (n doit être puissance de 2). Si n_fft pas puissance de 2 +// (cas Whisper N_FFT=400), on zero-pad jusqu'à la puissance de 2 supérieure. +static void fft_radix2(float* real, float* imag, int n) { + int j = 0; + for (int i = 1; i < n; i++) { + int bit = n >> 1; + while (j & bit) { j ^= bit; bit >>= 1; } + j ^= bit; + if (i < j) { std::swap(real[i], real[j]); std::swap(imag[i], imag[j]); } + } + for (int len = 2; len <= n; len <<= 1) { + int half = len / 2; + double angle = -2.0 * M_PI / len; + float wR = (float)std::cos(angle), wI = (float)std::sin(angle); + for (int i = 0; i < n; i += len) { + float cR = 1.f, cI = 0.f; + for (int k = 0; k < half; k++) { + float tR = cR * real[i+k+half] - cI * imag[i+k+half]; + float tI = cR * imag[i+k+half] + cI * real[i+k+half]; + real[i+k+half] = real[i+k] - tR; + imag[i+k+half] = imag[i+k] - tI; + real[i+k] += tR; + imag[i+k] += tI; + float ncR = cR * wR - cI * wI; + cI = cR * wI + cI * wR; + cR = ncR; + } + } + } +} + +static int next_pow2(int n) { + int p = 1; while (p < n) p <<= 1; return p; +} + +static std::vector make_window(int n, WindowKind kind) { + std::vector w(n); + if (kind == WindowKind::HANN_SYMMETRIC) { + // torch.hann_window(N) default = symmetric : 0.5 * (1 - cos(2*pi*n/(N-1))) + for (int i = 0; i < n; i++) + w[i] = 0.5f * (1.0f - std::cos(2.0f * (float)M_PI * (float)i / (float)(n - 1))); + } else { + // Periodic (HF whisper, numpy default) : 0.5 * (1 - cos(2*pi*n/N)) + for (int i = 0; i < n; i++) + w[i] = 0.5f * (1.0f - std::cos(2.0f * (float)M_PI * (float)i / (float)n)); + } + return w; +} + +// Pad reflect : miroir interne (n'inclut PAS le sample bord). Équivalent torch reflect. +static std::vector pad_reflect(const std::vector& x, int padding) { + const int n = (int)x.size(); + std::vector out((size_t)n + 2 * padding); + for (int i = 0; i < padding; i++) out[i] = x[padding - i]; + for (int i = 0; i < n; i++) out[padding + i] = x[i]; + for (int i = 0; i < padding; i++) out[padding + n + i] = x[n - 2 - i]; + return out; +} + +// ============================================================================ +// load_mel_basis : fichier binaire f32 [n_mels, n_fft/2+1] +// ============================================================================ +bool load_mel_basis(const char * path, const MelConfig & cfg, std::vector & out) { + const int fft_bins = cfg.n_fft / 2 + 1; + const size_t expected = (size_t)cfg.n_mels * fft_bins; + std::ifstream f(path, std::ios::binary | std::ios::ate); + if (!f) { fprintf(stderr, "kazeia_mel::load_mel_basis: open %s FAIL\n", path); return false; } + size_t n = (size_t)f.tellg() / sizeof(float); + if (n != expected) { + fprintf(stderr, "kazeia_mel::load_mel_basis: %s contient %zu f32, attendu %zu (=%d mels x %d bins)\n", + path, n, expected, cfg.n_mels, fft_bins); + return false; + } + f.seekg(0); + out.assign(expected, 0.0f); + f.read((char*)out.data(), expected * sizeof(float)); + return (bool)f; +} + +// ============================================================================ +// compute : STFT (radix-2 zero-pad) + projection mel + log + post-norm +// ============================================================================ +std::vector compute(const std::vector & wav_in, + const std::vector & mel_basis, + const MelConfig & cfg, + int & out_T) { + const int fft_bins = cfg.n_fft / 2 + 1; + const int pad_amount = (cfg.pad_amount >= 0) ? cfg.pad_amount : (cfg.n_fft - cfg.hop_size) / 2; + + // Optionnel : padding audio à T_fixe (Whisper 30s) + std::vector wav = wav_in; + if (cfg.fixed_n_frames > 0) { + int target_samples = cfg.fixed_n_frames * cfg.hop_size; + if ((int)wav.size() > target_samples) wav.resize(target_samples); + else if ((int)wav.size() < target_samples) wav.resize(target_samples, 0.0f); + } + + auto pad = pad_reflect(wav, pad_amount); + auto hann = make_window(cfg.win_size, cfg.window); + const int len_pad = (int)pad.size(); + if (len_pad < cfg.n_fft) { + fprintf(stderr, "kazeia_mel::compute: wav trop court (%d < %d)\n", len_pad, cfg.n_fft); + out_T = 0; return {}; + } + + // T_compute : sans fixed_n_frames -> calcule autant que possible (center=False). + // Avec fixed_n_frames -> exactement N_FRAMES (le padding ci-dessus garantit la place). + int T; + if (cfg.fixed_n_frames > 0) T = cfg.fixed_n_frames; + else T = (len_pad - cfg.n_fft) / cfg.hop_size + 1; + out_T = T; + + // FFT en puissance de 2 >= n_fft (Whisper N_FFT=400 -> FFT 512). + const int fftN = next_pow2(cfg.n_fft); + + std::vector spec((size_t)fft_bins * T, 0.0f); + std::vector re(fftN), im(fftN); + + for (int t = 0; t < T; t++) { + const int off = t * cfg.hop_size; + // Window sur win_size samples, zero-pad jusqu'à fftN + for (int i = 0; i < cfg.win_size; i++) { + re[i] = pad[off + i] * hann[i]; + im[i] = 0.0f; + } + for (int i = cfg.win_size; i < fftN; i++) { re[i] = 0.0f; im[i] = 0.0f; } + + fft_radix2(re.data(), im.data(), fftN); + + // Power spectrum [0..n_fft/2] + for (int k = 0; k < fft_bins; k++) { + spec[(size_t)k * T + t] = re[k]*re[k] + im[k]*im[k]; + } + } + + // mel = basis [n_mels, fft_bins] @ spec [fft_bins, T] -> [n_mels, T] + std::vector mel((size_t)cfg.n_mels * T, 0.0f); + for (int m = 0; m < cfg.n_mels; m++) { + for (int t = 0; t < T; t++) { + float s = 0.0f; + for (int k = 0; k < fft_bins; k++) { + s += mel_basis[m * fft_bins + k] * spec[(size_t)k * T + t]; + } + mel[(size_t)m * T + t] = std::max(s, cfg.clip_eps); + } + } + + // Compression (log naturel ou log10) + sqrt si TTS (variante: TTS prend sqrt magnitude + // AVANT projection mel ; Whisper prend power direct projetée puis log10). + // Distinction empirique du speaker encoder Qwen3-TTS : il calcule magnitude = sqrt(re²+im²+1e-9) + // puis projette puis log. Pour rester strictement bit-correct avec le code spk d'origine, + // on adapte si compression == LOG_NATURAL : sqrt avant log. + if (cfg.comp == Compression::LOG_NATURAL) { + // TTS : sqrt(power+1e-9) puis mel_proj puis log(max(., eps)) + // Refait correctement : revert la projection power, refais en magnitude. + // Implémenter ainsi pour bit-match : + std::vector mag_spec((size_t)fft_bins * T); + for (int k = 0; k < fft_bins; k++) + for (int t = 0; t < T; t++) + mag_spec[(size_t)k * T + t] = std::sqrt(spec[(size_t)k * T + t] + 1e-9f); + for (int m = 0; m < cfg.n_mels; m++) { + for (int t = 0; t < T; t++) { + float s = 0.0f; + for (int k = 0; k < fft_bins; k++) + s += mel_basis[m * fft_bins + k] * mag_spec[(size_t)k * T + t]; + mel[(size_t)m * T + t] = std::log(std::max(s, cfg.clip_eps)); + } + } + } else { // LOG10 (Whisper) + for (auto & v : mel) v = std::log10(v); + } + + // Post-norm (Whisper : clamp à max-8, puis (x+4)/4) + if (cfg.post == PostNorm::WHISPER) { + float maxv = *std::max_element(mel.begin(), mel.end()); + float floor_v = maxv - 8.0f; + for (auto & v : mel) { + if (v < floor_v) v = floor_v; + v = (v + 4.0f) * 0.25f; + } + } + + return mel; +} + +} // namespace kazeia_mel diff --git a/dist/jni/kazeia_mel.h b/dist/jni/kazeia_mel.h new file mode 100644 index 0000000..26af821 --- /dev/null +++ b/dist/jni/kazeia_mel.h @@ -0,0 +1,63 @@ +// Mel spectrogram extractor unifié Kazeia-Engine. +// Couvre Whisper (16kHz, 80 mel, log10 + Whisper norm) et Qwen3-TTS speaker encoder +// (24kHz, 128 mel, log naturel + clip 1e-5). +// +// Bit-correct vs librosa/torch reference pour les deux profils. +// +// Layout sortie : [N_MELS, T] = data[mel * T + frame] (PyTorch [B, C, T] / ggml +// ne[0]=T fastest). Compatible directement avec speaker_encoder_encode_waveform() +// et Whisper encoder ONNX (qui veut [1, n_mels, n_frames] fp16). +#pragma once +#include +#include +#include + +namespace kazeia_mel { + +enum class WindowKind { + HANN_SYMMETRIC, // torch.hann_window(N) = 0.5*(1-cos(2*pi*n/(N-1))) (TTS) + HANN_PERIODIC, // numpy/HF whisper = 0.5*(1-cos(2*pi*n/N)) (STT) +}; + +enum class Compression { + LOG_NATURAL, // log(max(mel, eps)) (TTS Qwen3-TTS) + LOG10, // log10(max(mel, eps)) (STT Whisper) +}; + +enum class PostNorm { + NONE, // pas de post-process (TTS) + WHISPER, // clamp à max-8, puis (x+4)/4 (STT Whisper) +}; + +struct MelConfig { + int sample_rate = 16000; + int n_fft = 400; + int hop_size = 160; + int win_size = 400; + int n_mels = 80; + int pad_amount = -1; // -1 -> auto : whisper n_fft/2, tts (n_fft-hop)/2 + int fixed_n_frames = 0; // 0 -> variable ; >0 -> pad audio à T fixe (whisper=3000=30s) + float clip_eps = 1e-5f; // valeur min avant log + WindowKind window = WindowKind::HANN_SYMMETRIC; + Compression comp = Compression::LOG_NATURAL; + PostNorm post = PostNorm::NONE; +}; + +// Configs pré-faites — utiliser comme défauts puis customiser au besoin. +MelConfig config_qwen3_tts_speaker(); // SR=24000, N_FFT=1024, hop=256, n_mels=128, log naturel +MelConfig config_whisper(); // SR=16000, N_FFT=400, hop=160, n_mels=80, log10 + WhisperNorm + +// Charge mel_basis [n_mels, n_fft/2+1] depuis un fichier binaire f32 (librosa pré-calculé +// ou mel_filters.json converti). Le caller fournit le path et la lib vérifie la taille +// attendue n_mels * (n_fft/2+1). +bool load_mel_basis(const char * path, const MelConfig & cfg, std::vector & out); + +// Compute mel spectrogram. wav = mono f32 [-1..1] @ cfg.sample_rate. mel_basis attendu +// dans le layout librosa [n_mels, n_fft/2+1]. Retourne mel [n_mels, T] dans le layout +// data[mel * T + frame]. out_T renvoie le nombre de frames calculés. +std::vector compute(const std::vector & wav, + const std::vector & mel_basis, + const MelConfig & cfg, + int & out_T); + +} // namespace kazeia_mel diff --git a/dist/jni/speaker_encoder.cpp b/dist/jni/speaker_encoder.cpp index 7093dd5..fc849e5 100644 --- a/dist/jni/speaker_encoder.cpp +++ b/dist/jni/speaker_encoder.cpp @@ -15,6 +15,7 @@ // Compile avec -DSPK_STANDALONE pour le binaire CLI : // usage : kazeia_speaker_encode #include "speaker_encoder.h" +#include "kazeia_mel.h" #include "ggml.h" #include "gguf.h" #include "ggml-cpu.h" @@ -29,111 +30,11 @@ #include // ============================================================================ -// Constantes mel (cf /opt/Kazeia/qnn_venv/.../modeling_qwen3_tts.py mel_spectrogram) +// Constantes mel : maintenant centralisées dans kazeia_mel::config_qwen3_tts_speaker() // ============================================================================ static const int SAMPLE_RATE = 24000; -static const int N_FFT = 1024; -static const int HOP_SIZE = 256; -static const int WIN_SIZE = 1024; static const int N_MELS = 128; -static const int FFT_BINS = N_FFT / 2 + 1; // 513 -static const float MEL_CLIP_VAL = 1e-5f; - -// ============================================================================ -// FFT radix-2 (in-place, n doit être puissance de 2). Identique à mel_extractor.cpp. -// ============================================================================ -static void fft_radix2(float* real, float* imag, int n) { - // Bit-reversal - int j = 0; - for (int i = 1; i < n; i++) { - int bit = n >> 1; - while (j & bit) { j ^= bit; bit >>= 1; } - j ^= bit; - if (i < j) { std::swap(real[i], real[j]); std::swap(imag[i], imag[j]); } - } - for (int len = 2; len <= n; len <<= 1) { - int half = len / 2; - double angle = -2.0 * M_PI / len; - float wR = (float)std::cos(angle), wI = (float)std::sin(angle); - for (int i = 0; i < n; i += len) { - float cR = 1.f, cI = 0.f; - for (int k = 0; k < half; k++) { - float tR = cR * real[i+k+half] - cI * imag[i+k+half]; - float tI = cR * imag[i+k+half] + cI * real[i+k+half]; - real[i+k+half] = real[i+k] - tR; - imag[i+k+half] = imag[i+k] - tI; - real[i+k] += tR; - imag[i+k] += tI; - float ncR = cR * wR - cI * wI; - cI = cR * wI + cI * wR; - cR = ncR; - } - } - } -} - -// Hann window (cosine), torch.hann_window(N) = 0.5*(1 - cos(2π * n / (N-1))) -static std::vector make_hann(int n) { - std::vector w(n); - for (int i = 0; i < n; i++) - w[i] = 0.5f * (1.0f - std::cos(2.0f * (float)M_PI * (float)i / (float)(n - 1))); - return w; -} - -// Pad reflect : insère padding samples sur chaque côté, miroir interne (sans répéter -// la 1ère/dernière). Équivalent torch.nn.functional.pad(..., mode="reflect"). -static std::vector pad_reflect(const std::vector& x, int padding) { - const int n = (int)x.size(); - std::vector out((size_t)n + 2 * padding); - for (int i = 0; i < padding; i++) out[i] = x[padding - i]; // miroir gauche - for (int i = 0; i < n; i++) out[padding + i] = x[i]; // milieu - for (int i = 0; i < padding; i++) out[padding + n + i] = x[n - 2 - i]; // miroir droite - return out; -} - -// Compute mel-spectrogram [n_mels=128, T] depuis waveform mono 24kHz. -// Réplique exactement qwen_tts.core.models.modeling_qwen3_tts.mel_spectrogram. -static std::vector mel_spectrogram(const std::vector& wav, - const std::vector& mel_basis, // [128, 513] - int& out_T) { - const int padding = (N_FFT - HOP_SIZE) / 2; // 384 - auto pad = pad_reflect(wav, padding); - auto hann = make_hann(WIN_SIZE); - - // T = floor((len_padded - n_fft) / hop) + 1 (center=False) - const int len_pad = (int)pad.size(); - if (len_pad < N_FFT) { fprintf(stderr, "wav too short (%d < %d)\n", len_pad, N_FFT); return {}; } - const int T = (len_pad - N_FFT) / HOP_SIZE + 1; - out_T = T; - - std::vector spec(FFT_BINS * T, 0.0f); // [513, T] magnitude - std::vector re(N_FFT), im(N_FFT); - for (int t = 0; t < T; t++) { - const int off = t * HOP_SIZE; - for (int i = 0; i < N_FFT; i++) { - re[i] = pad[off + i] * hann[i]; - im[i] = 0.0f; - } - fft_radix2(re.data(), im.data(), N_FFT); - // Magnitude sqrt(re² + im² + 1e-9), onesided -> bins 0..N_FFT/2 - for (int k = 0; k < FFT_BINS; k++) { - float m = re[k]*re[k] + im[k]*im[k] + 1e-9f; - spec[k * T + t] = std::sqrt(m); - } - } - - // mel = mel_basis [128, 513] @ spec [513, T] -> [128, T] - std::vector mel(N_MELS * T, 0.0f); - for (int m = 0; m < N_MELS; m++) { - for (int t = 0; t < T; t++) { - float s = 0.0f; - for (int k = 0; k < FFT_BINS; k++) - s += mel_basis[m * FFT_BINS + k] * spec[k * T + t]; - mel[m * T + t] = std::log(std::max(s, MEL_CLIP_VAL)); // dynamic_range_compression - } - } - return mel; -} +static const int FFT_BINS = 513; // n_fft/2+1 pour n_fft=1024 // ============================================================================ // Forward ECAPA-TDNN ggml C++ — bit-match qwen_tts.modeling_qwen3_tts @@ -540,7 +441,8 @@ SpeakerEncoder * speaker_encoder_load(const char * gguf_path, const char * mel_b std::vector speaker_encoder_encode_waveform(SpeakerEncoder * spk, const std::vector & wav) { if (!spk) return {}; int T_mel = 0; - auto mel = mel_spectrogram(wav, spk->mel_basis, T_mel); + const auto cfg = kazeia_mel::config_qwen3_tts_speaker(); + auto mel = kazeia_mel::compute(wav, spk->mel_basis, cfg, T_mel); if (mel.empty()) return {}; return spk_forward(*spk, mel, T_mel); } diff --git a/dist/jni/stt_engine.cpp b/dist/jni/stt_engine.cpp new file mode 100644 index 0000000..41c7a56 --- /dev/null +++ b/dist/jni/stt_engine.cpp @@ -0,0 +1,167 @@ +// Implementation stt_engine — voir stt_engine.h pour l'API publique. +// +// S1.3 (cette session) : squelette compilable, VAD complet, mel via kazeia_mel. +// stt_engine_load / transcribe / free : stubs renvoyant err=-99 jusqu'au port ORT +// QNN en S2. +#include "stt_engine.h" +#include "kazeia_mel.h" + +#include +#include +#include +#include +#include +#include + +// ============================================================================ +// VAD RMS énergie (port C++ de VadStage.kt) +// ============================================================================ +// +// État interne : compteur de frames parole / silence consécutifs, sliding sur +// chunks PCM. L'appelant push des chunks de taille frame_size; chaque push fait +// avancer l'état. is_speech / is_end_of_speech consultent l'état. + +struct SttVadState { + int sample_rate; + int frame_size; // 1600 = 100 ms @ 16 kHz + int rms_threshold; // 150 sur PCM16 brut (prod KazeiaService) + int min_speech_frames; // 3 -> 300 ms de parole pour déclencher + int silence_end_frames; // 8 -> 800 ms de silence pour finir + + int speech_count = 0; + int silence_count = 0; + bool in_speech = false; + bool end_of_speech = false; +}; + +SttVadState * stt_vad_new(int sample_rate, int frame_size, int rms_threshold, + int min_speech_frames, int silence_end_frames) { + auto * s = new SttVadState(); + s->sample_rate = sample_rate; + s->frame_size = frame_size; + s->rms_threshold = rms_threshold; + s->min_speech_frames = min_speech_frames; + s->silence_end_frames = silence_end_frames; + return s; +} + +void stt_vad_reset(SttVadState * s) { + if (!s) return; + s->speech_count = 0; + s->silence_count = 0; + s->in_speech = false; + s->end_of_speech = false; +} + +void stt_vad_free(SttVadState * s) { delete s; } + +bool stt_vad_is_speech(const SttVadState * s) { return s && s->in_speech; } +bool stt_vad_is_end_of_speech(const SttVadState * s) { return s && s->end_of_speech; } + +// Calcule RMS d'un buffer PCM16, renvoie l'énergie en unités PCM brutes +// (compatible seuil 150 prod). +static float rms_pcm16(const int16_t * pcm, int n) { + if (n <= 0) return 0.0f; + double acc = 0.0; + for (int i = 0; i < n; ++i) { double v = (double)pcm[i]; acc += v * v; } + return (float)std::sqrt(acc / (double)n); +} + +bool stt_vad_push(SttVadState * s, const int16_t * pcm, int n_samples) { + if (!s || !pcm || n_samples <= 0) return false; + s->end_of_speech = false; + + // Traite par chunks de frame_size. Si n_samples < frame_size, on accumulera + // mentalement mais ici on calcule simplement la RMS sur ce qu'on a (l'app + // appelante pousse des chunks pleins de 100 ms en pratique). + int offset = 0; + while (offset + s->frame_size <= n_samples) { + float r = rms_pcm16(pcm + offset, s->frame_size); + if (r >= (float)s->rms_threshold) { + s->speech_count += 1; + s->silence_count = 0; + if (!s->in_speech && s->speech_count >= s->min_speech_frames) { + s->in_speech = true; + } + } else { + s->silence_count += 1; + // Pas de reset speech_count tant qu'on est in_speech, c'est l'enchainement + // silence consécutifs qui clôt. + if (s->in_speech && s->silence_count >= s->silence_end_frames) { + s->in_speech = false; + s->end_of_speech = true; + s->speech_count = 0; + } else if (!s->in_speech) { + s->speech_count = 0; // reset compteur si on n'a pas atteint le seuil + } + } + offset += s->frame_size; + } + return true; +} + +// ============================================================================ +// SttEngine (stubs — port ORT QNN en S2) +// ============================================================================ + +struct SttEngine { + SttEngineLoadCfg cfg; + std::vector mel_basis; // [80, 201] librosa whisper + // S2 : + // void * encoder_session; // Ort::Session* + // void * decoder_session; + // std::vector vocab_ids; + // std::unordered_map token_to_id; + bool loaded = false; +}; + +SttEngine * stt_engine_load(const SttEngineLoadCfg & cfg) { + if (!cfg.model_dir) { + fprintf(stderr, "stt_engine_load: model_dir requis\n"); return nullptr; + } + auto * eng = new SttEngine(); + eng->cfg = cfg; + // Tente charger mel_filters.bin (préféré) ou mel_filters.json (sera porté en S2). + std::string mel_path = std::string(cfg.model_dir) + "/mel_filters.bin"; + auto mel_cfg = kazeia_mel::config_whisper(); + if (!kazeia_mel::load_mel_basis(mel_path.c_str(), mel_cfg, eng->mel_basis)) { + fprintf(stderr, "stt_engine_load: mel_filters.bin absent ou mauvaise taille à %s\n", + mel_path.c_str()); + // Pas fatal : à S2 on chargera depuis mel_filters.json. Pour S1.3 on continue + // pour permettre les tests VAD. + } + eng->loaded = true; + return eng; +} + +SttTranscribeResult stt_engine_transcribe(SttEngine * eng, const SttTranscribeCfg & cfg) { + SttTranscribeResult R{}; + if (!eng || !eng->loaded || !cfg.pcm16 || cfg.n_samples <= 0) { + R.err = -1; return R; + } + // Étape 1 : mel via kazeia_mel (Whisper config). Implémentée en S1.3 pour exercer + // la lib partagée ; le encoder/decoder ORT viennent en S2. + std::vector wav((size_t)cfg.n_samples); + for (int i = 0; i < cfg.n_samples; ++i) wav[i] = (float)cfg.pcm16[i] / 32768.0f; + + if (eng->mel_basis.empty()) { + fprintf(stderr, "stt_engine_transcribe: mel_basis non chargé (mel_filters.bin manquant)\n"); + R.err = -2; return R; + } + auto mel_cfg = kazeia_mel::config_whisper(); + int T = 0; + auto mel = kazeia_mel::compute(wav, eng->mel_basis, mel_cfg, T); + if (mel.empty()) { R.err = -3; return R; } + R.mel_ms = 0; // TODO: timer + + // Étape 2..N : encoder NPU + decoder loop + tokenizer BPE -> S2. + fprintf(stderr, "stt_engine_transcribe: stubs (port S2 requis). mel %d frames OK.\n", T); + R.err = -99; // not implemented yet + R.detected_language = cfg.language; + return R; +} + +void stt_engine_free(SttEngine * eng) { + if (!eng) return; + delete eng; +} diff --git a/dist/jni/stt_engine.h b/dist/jni/stt_engine.h new file mode 100644 index 0000000..095265c --- /dev/null +++ b/dist/jni/stt_engine.h @@ -0,0 +1,103 @@ +// Engine STT Whisper-Small Qualcomm AI Hub (HfWhisper KV-cache) in-process. +// Load une fois (encoder QAIRT context + decoder QAIRT context + mel_filters + +// vocab), transcribe N fois (PCM 16kHz mono -> texte FR/EN). +// +// Backend : ONNX Runtime + QNN ExecutionProvider (HTP V79 sur SM8750). +// Empreinte RAM : ~545 MB (encoder ctx 201 MB + decoder ctx 345 MB). +// +// Bench de référence prod (Whisper-Small, FR, audio 1.6s) : +// mel 189 ms (CPU C++) + encoder 125 ms (NPU) + decoder 510 ms (NPU, 22 tokens) +// total 825 ms, RTF 0.51, RAM peak 545 MB. +// +// Construit pour remplacer WhisperHybridEngine.kt + libmel_extractor.so + VadStage.kt +// derrière une API unifiée Kazeia-Engine. La sélection de backend (HTP vs CPU) est +// automatique via QNN EP : si le contexte QAIRT est trouvé, HTP ; sinon CPU fallback. +#pragma once +#include +#include +#include + +struct SttEngine; // opaque + +struct SttEngineLoadCfg { + // Dir contenant : HfWhisperEncoder.onnx + HfWhisperEncoder_qairt_context.bin, + // HfWhisperDecoder.onnx + HfWhisperDecoder_qairt_context.bin, + // mel_filters.json (ou .bin), vocab.json (BPE byte-level). + const char * model_dir = nullptr; + + // 1 = QNN HTP (NPU, défaut prod). 0 = CPU fallback (ONNX Runtime CPU EP). + // QNN EP requiert que libQnnHtp.so + libQnnHtpV79Skel.so + libcdsprpc.so soient + // dans LD_LIBRARY_PATH (côté tablette : push depuis dist/lib/). + bool use_htp = true; + + int n_threads = 6; + int max_decode_steps = 200; // borne hard du loop decoder autorégressif +}; + +struct SttTranscribeCfg { + // PCM mono 16-bit signed, n'importe quelle longueur (le mel pad fait à 30s). + // Si vous avez du PCM 24kHz, resamplez externe avant (ou utilisez SttTranscribeWaveform). + const int16_t * pcm16 = nullptr; + int n_samples = 0; + int sample_rate = 16000; + + // "fr", "en", "auto", etc. Whisper-Small supporte 99 langues. "auto" = détection + // via la langue token de Whisper. + const char * language = "fr"; + + // 1 = force le mode "transcribe" (FR -> texte FR). 0 = laisse Whisper choisir + // entre transcribe et translate (translate force la sortie en EN, pas voulu pour + // Kazeia FR). Défaut : 1 (cf override prod KazeiaService). + bool force_transcribe = true; + + // Inclure les timestamps (segments + offsets) dans la sortie. + bool with_timestamps = false; +}; + +struct SttSegment { + int t0_ms; // début segment (ms relatif au début audio) + int t1_ms; // fin segment + std::string text; +}; + +struct SttTranscribeResult { + int err = 0; // 0 = ok, <0 = err + std::string text; // texte concaténé (sans timestamps) + std::vector segments; // vide si !with_timestamps + std::string detected_language; // "fr", "en", ... + + // Métriques perf (ms) + int mel_ms = 0; + int encoder_ms = 0; + int decoder_ms = 0; + int total_ms = 0; + float rtf = 0.0f; // total_ms / (audio_duration_ms) + int n_tokens = 0; +}; + +// Charge encoder + decoder + mel basis + vocab. Retourne nullptr si échec. +// Idempotent vis-à-vis de llama_backend_init() (cohabite avec tts_engine). +SttEngine * stt_engine_load(const SttEngineLoadCfg & cfg); + +// Transcrit une fenêtre audio. Stateful pour le warm path NPU (sessions ORT +// gardées en RAM entre appels). +SttTranscribeResult stt_engine_transcribe(SttEngine * eng, const SttTranscribeCfg & cfg); + +// Libère tout (encoder ctx, decoder ctx, sessions ORT, vocab). +void stt_engine_free(SttEngine * eng); + +// Helper VAD RMS énergie : renvoie true si le buffer PCM contient de la parole +// (énergie > seuil). Compatible drop-in avec VadStage.kt prod (frame=1600 @ 16kHz, +// seuil 150 PCM, ≥3 frames consécutives parole, ≥8 frames silence). +// Tu peux passer le PCM en streaming par chunks de 100ms et lire is_speech / is_end_of_speech. +struct SttVadState; +SttVadState * stt_vad_new(int sample_rate = 16000, + int frame_size = 1600, + int rms_threshold = 150, + int min_speech_frames = 3, + int silence_end_frames = 8); +bool stt_vad_push(SttVadState * s, const int16_t * pcm, int n_samples); +bool stt_vad_is_speech(const SttVadState * s); +bool stt_vad_is_end_of_speech(const SttVadState * s); +void stt_vad_reset(SttVadState * s); +void stt_vad_free(SttVadState * s); diff --git a/dist/jni/stt_test.cpp b/dist/jni/stt_test.cpp new file mode 100644 index 0000000..b8e5c1c --- /dev/null +++ b/dist/jni/stt_test.cpp @@ -0,0 +1,113 @@ +// Test standalone S1.3 : exercer kazeia_mel (config_whisper) + VAD RMS. +// Usage : kazeia_stt_test +// +// Vérifie que le mel produit a la bonne shape [80 x 3000] et que les valeurs sont +// dans la plage Whisper (~[-1, 0.5] après norm). Permet de valider le mel partagé +// AVANT de brancher ORT/QNN en S2. +#include "kazeia_mel.h" +#include "stt_engine.h" + +#include +#include +#include +#include +#include + +static std::vector read_wav_pcm16(const char* path, int & sr_out, int & channels_out) { + FILE* f = fopen(path, "rb"); + if (!f) { fprintf(stderr, "open %s FAIL\n", path); return {}; } + char riff[12]; + if (fread(riff, 1, 12, f) != 12 || memcmp(riff, "RIFF", 4) != 0 || memcmp(riff+8, "WAVE", 4) != 0) { + fprintf(stderr, "not RIFF/WAVE\n"); fclose(f); return {}; + } + uint16_t channels = 0, bps = 0; + uint32_t sr = 0, data_sz = 0; + long data_off = -1; + while (!feof(f)) { + char cid[4]; uint32_t csz; + if (fread(cid, 1, 4, f) != 4) break; + if (fread(&csz, 4, 1, f) != 1) break; + if (memcmp(cid, "fmt ", 4) == 0) { + std::vector buf(csz); + fread(buf.data(), 1, csz, f); + channels = *(uint16_t*)&buf[2]; + sr = *(uint32_t*)&buf[4]; + bps = *(uint16_t*)&buf[14]; + } else if (memcmp(cid, "data", 4) == 0) { + data_sz = csz; data_off = ftell(f); break; + } else { + fseek(f, csz, SEEK_CUR); + } + } + if (data_off < 0 || channels != 1 || bps != 16) { + fprintf(stderr, "WAV mono 16-bit attendu (got ch=%d bps=%d)\n", channels, bps); + fclose(f); return {}; + } + sr_out = (int)sr; channels_out = channels; + fseek(f, data_off, SEEK_SET); + std::vector pcm(data_sz / 2); + fread(pcm.data(), 2, pcm.size(), f); + fclose(f); + return pcm; +} + +int main(int argc, char** argv) { + setvbuf(stderr, nullptr, _IONBF, 0); + setvbuf(stdout, nullptr, _IONBF, 0); + if (argc < 3) { + fprintf(stderr, "usage: %s \n", argv[0]); + return 1; + } + + // 1) Mel basis 80x201 (Whisper) + auto cfg = kazeia_mel::config_whisper(); + std::vector mel_basis; + if (!kazeia_mel::load_mel_basis(argv[1], cfg, mel_basis)) return 2; + fprintf(stderr, "mel_basis Whisper [%d, %d] OK (%zu f32)\n", + cfg.n_mels, cfg.n_fft/2+1, mel_basis.size()); + + // 2) WAV + int sr = 0, ch = 0; + auto pcm = read_wav_pcm16(argv[2], sr, ch); + if (pcm.empty()) return 3; + fprintf(stderr, "wav : %zu samples @ %d Hz = %.2f s\n", + pcm.size(), sr, (double)pcm.size() / sr); + if (sr != 16000) fprintf(stderr, "WARNING: sr=%d != 16000 (resample externe requis)\n", sr); + + // 3) Convert to f32 + std::vector wav(pcm.size()); + for (size_t i = 0; i < pcm.size(); ++i) wav[i] = (float)pcm[i] / 32768.0f; + + // 4) Mel via kazeia_mel + int T = 0; + auto mel = kazeia_mel::compute(wav, mel_basis, cfg, T); + fprintf(stderr, "mel computed : [%d mels, %d frames] = %zu f32 (cible 80 x 3000 Whisper)\n", + cfg.n_mels, T, mel.size()); + + float mn = 1e9f, mx = -1e9f; double sum = 0; + for (float v : mel) { if (v < mn) mn = v; if (v > mx) mx = v; sum += v; } + fprintf(stderr, "mel stats : min=%.4f max=%.4f mean=%.4f\n", + mn, mx, (float)(sum / mel.size())); + fprintf(stderr, "(plage Whisper attendue ~[-1.0, 0.5])\n"); + + // 5) VAD test : push tout le PCM par chunks de 1600 samples + fprintf(stderr, "\n=== VAD RMS test (frame=1600=100ms, seuil=150) ===\n"); + SttVadState * vad = stt_vad_new(16000, 1600, 150, 3, 8); + int speech_seen = 0, eos_count = 0; + for (size_t off = 0; off + 1600 <= pcm.size(); off += 1600) { + stt_vad_push(vad, pcm.data() + off, 1600); + if (stt_vad_is_speech(vad)) speech_seen += 1; + if (stt_vad_is_end_of_speech(vad)) eos_count += 1; + } + fprintf(stderr, "VAD : %d frames de parole / %zu total, %d end-of-speech\n", + speech_seen, pcm.size() / 1600, eos_count); + stt_vad_free(vad); + + // 6) Dump mel pour comparaison externe si KZSTT_DUMP_MEL=path + if (const char * p = std::getenv("KZSTT_DUMP_MEL")) { + std::ofstream f(p, std::ios::binary); + f.write((char*)mel.data(), mel.size() * sizeof(float)); + fprintf(stderr, "mel dumped -> %s\n", p); + } + return 0; +} diff --git a/dist/jni/tts_pipeline.cpp b/dist/jni/tts_pipeline.cpp index 5139307..f091f69 100644 --- a/dist/jni/tts_pipeline.cpp +++ b/dist/jni/tts_pipeline.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -67,11 +68,16 @@ int main(int argc, char** argv) { // l'utilise comme override pour ce synth (clonage vocal in-process). std::vector ref_xvec; if (const char * ref_wav = getenv("KZTTS_REF_WAV")) { + struct timespec t0, t1; + clock_gettime(CLOCK_MONOTONIC, &t0); ref_xvec = tts_engine_encode_speaker_wav(eng, ref_wav); + clock_gettime(CLOCK_MONOTONIC, &t1); + double enc_s = (t1.tv_sec - t0.tv_sec) + (t1.tv_nsec - t0.tv_nsec) * 1e-9; if (ref_xvec.empty()) { fprintf(stderr, "tts_pipeline: KZTTS_REF_WAV=%s FAIL (encoder missing or read FAIL)\n", ref_wav); } else { - fprintf(stderr, "tts_pipeline: KZTTS_REF_WAV=%s -> x_vector[%zu] OK\n", ref_wav, ref_xvec.size()); + fprintf(stderr, "tts_pipeline: KZTTS_REF_WAV=%s -> x_vector[%zu] in %.3fs\n", + ref_wav, ref_xvec.size(), enc_s); } } diff --git a/dist/lib/README_ORT.md b/dist/lib/README_ORT.md new file mode 100644 index 0000000..9017448 --- /dev/null +++ b/dist/lib/README_ORT.md @@ -0,0 +1,15 @@ +# libonnxruntime.so (Microsoft ONNX Runtime + QNN EP) + +Cette lib est requise par stt_engine.cpp pour exécuter Whisper-Small sur HTP V79. +Non versionnée dans git à cause de sa taille (20 MB). + +Source : Maven `com.microsoft.onnxruntime:onnxruntime-android-qnn:1.24.3`. + +Pour récupérer : extraire de l'AAR Gradle cache : +```bash +AAR=~/.gradle/caches/8.12/transforms/*/transformed/onnxruntime-android-qnn-1.24.3 +cp $AAR/jni/arm64-v8a/libonnxruntime.so /opt/Kazeia-engine/dist/lib/ +cp $AAR/headers/*.h /opt/Kazeia-engine/dist/include/onnxruntime/ +``` + +Headers C++ : dans `dist/include/onnxruntime/` (versionnés, ~1 MB).