From 7f6adcb2b8bceb99a27e4ca369d3493c38864bc2 Mon Sep 17 00:00:00 2001 From: Glen Cao Date: Mon, 3 Jun 2024 11:41:52 -0700 Subject: [PATCH 01/11] Initial support for EP context models for VitisAI EP --- .../shared_library/provider_interfaces.h | 73 ++++ .../shared_library/provider_wrappedtypes.h | 85 +++++ .../providers/vitisai/imp/ep_context_utils.cc | 333 ++++++++++++++++++ .../vitisai/include/ep_context_utils.h | 57 +++ .../vitisai/vitisai_execution_provider.cc | 129 ++++++- .../vitisai/vitisai_execution_provider.h | 26 +- .../vitisai/vitisai_provider_factory.cc | 0 .../core/session/provider_bridge_ort.cc | 80 ++++- 8 files changed, 773 insertions(+), 10 deletions(-) create mode 100644 onnxruntime/core/providers/vitisai/imp/ep_context_utils.cc create mode 100644 onnxruntime/core/providers/vitisai/include/ep_context_utils.h mode change 100755 => 100644 onnxruntime/core/providers/vitisai/vitisai_provider_factory.cc diff --git a/onnxruntime/core/providers/shared_library/provider_interfaces.h b/onnxruntime/core/providers/shared_library/provider_interfaces.h index cc3b13f696a96..ad63bd895b81a 100644 --- a/onnxruntime/core/providers/shared_library/provider_interfaces.h +++ b/onnxruntime/core/providers/shared_library/provider_interfaces.h @@ -421,6 +421,7 @@ struct ProviderHost { virtual int NodeProto__attribute_size(ONNX_NAMESPACE::NodeProto* p) = 0; virtual const ONNX_NAMESPACE::AttributeProto& NodeProto__attribute(const ONNX_NAMESPACE::NodeProto* p, int index) const = 0; virtual ONNX_NAMESPACE::AttributeProto* NodeProto__mutable_attribute(ONNX_NAMESPACE::NodeProto* p, int index) = 0; + virtual ONNX_NAMESPACE::AttributeProto* NodeProto__add_attribute(ONNX_NAMESPACE::NodeProto* p) = 0; // TensorProto virtual std::unique_ptr TensorProto__construct() = 0; @@ -489,6 +490,75 @@ struct ProviderHost { virtual const ONNX_NAMESPACE::ValueInfoProto& ValueInfoProtos__operator_array(const ONNX_NAMESPACE::ValueInfoProtos* p, int index) = 0; + // FunctionProto + virtual std::unique_ptr FunctionProto__construct() = 0; + virtual void FunctionProto__operator_delete(ONNX_NAMESPACE::FunctionProto* p) = 0; + + virtual bool FunctionProto__SerializeToString(const ONNX_NAMESPACE::FunctionProto* p, std::string& string) = 0; + virtual bool FunctionProto__SerializeToOstream(const ONNX_NAMESPACE::FunctionProto* p, std::ostream& output) = 0; + virtual bool FunctionProto__ParseFromString(ONNX_NAMESPACE::FunctionProto* p, const std::string& data) = 0; + virtual std::string FunctionProto__SerializeAsString(const ONNX_NAMESPACE::FunctionProto* p) = 0; + + virtual bool FunctionProto__has_name(const ONNX_NAMESPACE::FunctionProto* p) = 0; + virtual const std::string& FunctionProto__name(const ONNX_NAMESPACE::FunctionProto* p) const = 0; + virtual void FunctionProto__set_name(ONNX_NAMESPACE::FunctionProto* p, const ::std::string& name) = 0; + + virtual bool FunctionProto__has_since_version(const ONNX_NAMESPACE::FunctionProto* p) = 0; + virtual int FunctionProto__since_version(const ONNX_NAMESPACE::FunctionProto* p) const = 0; + virtual void FunctionProto__set_since_version(ONNX_NAMESPACE::FunctionProto* p, int since_version) = 0; + + bool FunctionProto__has_status(const ONNX_NAMESPACE::FunctionProto* p) = 0; + virtual const ONNX_NAMESPACE::OperatorStatus& FunctionProto__status(const ONNX_NAMESPACE::FunctionProto* p) const = 0; + virtual void FunctionProto__set_status(ONNX_NAMESPACE::FunctionProto* p, const ONNX_NAMESPACE::OperatorStatus& status) = 0; + + virtual bool FunctionProto__has_doc_string(const ONNX_NAMESPACE::FunctionProto* p) = 0; + virtual const std::string& FunctionProto__doc_string(const ONNX_NAMESPACE::FunctionProto* p) const = 0; + virtual void FunctionProto__set_doc_string(ONNX_NAMESPACE::FunctionProto* p, const ::std::string& doc_string) = 0; + + virtual bool FunctionProto__has_domain(const ONNX_NAMESPACE::FunctionProto* p) = 0; + virtual const std::string& FunctionProto__domain(const ONNX_NAMESPACE::FunctionProto* p) = 0; + virtual void FunctionProto__set_domain(ONNX_NAMESPACE::FunctionProto* p, const ::std::string& domain) = 0; + + virtual const std::string& FunctionProto__input(const ONNX_NAMESPACE::FunctionProto* p, int index) = 0; + virtual std::vector* FunctionProto__mutable_input(ONNX_NAMESPACE::FunctionProto* p) = 0; + virtual std::string* FunctionProto__mutable_input(ONNX_NAMESPACE::FunctionProto* p, int index) = 0; + virtual int FunctionProto__input_size(const ONNX_NAMESPACE::FunctionProto* p) = 0; + virtual void FunctionProto__add_input(ONNX_NAMESPACE::FunctionProto* p, const string& value) = 0; + + virtual const std::string& FunctionProto__output(const ONNX_NAMESPACE::FunctionProto* p, int index) = 0; + virtual std::vector* FunctionProto__mutable_output(ONNX_NAMESPACE::FunctionProto* p) = 0; + virtual std::string* FunctionProto__mutable_output(ONNX_NAMESPACE::FunctionProto* p, int index) = 0; + virtual int FunctionProto__output_size(const ONNX_NAMESPACE::FunctionProto* p) = 0; + virtual void FunctionProto__add_output(ONNX_NAMESPACE::FunctionProto* p, const string& value) = 0; + + virtual const std::string& FunctionProto__attribute(const ONNX_NAMESPACE::FunctionProto* p, int index) = 0; + virtual std::vector* FunctionProto__mutable_attribute(ONNX_NAMESPACE::FunctionProto* p) = 0; + virtual std::string* FunctionProto__mutable_attribute(ONNX_NAMESPACE::FunctionProto* p, int index) = 0; + virtual int FunctionProto__attribute_size(const ONNX_NAMESPACE::FunctionProto* p) = 0; + virtual void FunctionProto__add_attribute(ONNX_NAMESPACE::FunctionProto* p, const string& value) = 0; + + virtual const ONNX_NAMESPACE::AttributeProto& FunctionProto__attribute_proto(const ONNX_NAMESPACE::FunctionProto* p, int index) = 0; + virtual ONNX_NAMESPACE::AttributeProto* FunctionProto__mutable_attribute_proto(ONNX_NAMESPACE::FunctionProto* p, int index) = 0; + virtual int FunctionProto__attribute_proto_size(const ONNX_NAMESPACE::FunctionProto* p) = 0; + virtual ONNX_NAMESPACE::AttributeProto* FunctionProto__add_attribution_proto(ONNX_NAMESPACE::FunctionProto* p) = 0; + + virtual const ONNX_NAMESPACE::NodeProto& FunctionProto__node(const ONNX_NAMESPACE::FunctionProto* p, int index) = 0; + virtual ONNX_NAMESPACE::NodeProto* FunctionProto__mutable_node(ONNX_NAMESPACE::FunctionProto* p, int index) = 0; + virtual int FunctionProto__node_size(const ONNX_NAMESPACE::FunctionProto* p) = 0; + virtual ONNX_NAMESPACE::NodeProto* FunctionProto__add_node(ONNX_NAMESPACE::FunctionProto* p) = 0; + + virtual const ONNX_NAMESPACE::ValueInfoProto& FunctionProto__value_info(const ONNX_NAMESPACE::FunctionProto* p, int index) = 0; + virtual ONNX_NAMESPACE::ValueInfoProtos* FunctionProto__mutable_value_info(ONNX_NAMESPACE::FunctionProto* p) = 0; + virtual ONNX_NAMESPACE::ValueInfoProto* FunctionProto__mutable_value_info(ONNX_NAMESPACE::FunctionProto* p, int index) = 0; + virtual int FunctionProto__value_info_size(const ONNX_NAMESPACE::FunctionProto* p) = 0; + virtual ONNX_NAMESPACE::ValueInfoProto* FunctionProto__add_value_info(ONNX_NAMESPACE::FunctionProto* p) = 0; + + virtual const ONNX_NAMESPACE::StringStringEntryProto& FunctionProto__metadata_props(const ONNX_NAMESPACE::FunctionProto* p, int index) = 0; + virtual ONNX_NAMESPACE::StringStringEntryProtos* FunctionProto__mutable_metadata_props(ONNX_NAMESPACE::FunctionProto* p) = 0; + virtual ONNX_NAMESPACE::StringStringEntryProto* FunctionProto__mutable_metadata_props(ONNX_NAMESPACE::FunctionProto* p, int index) = 0; + virtual int FunctionProto__metadata_props_size(const ONNX_NAMESPACE::FunctionProto* p) = 0; + virtual ONNX_NAMESPACE::StringStringEntryProto* FunctionProto__add_metadata_props(ONNX_NAMESPACE::FunctionProto* p) = 0; + virtual void RegisterSchema(const std::string& domain, const OrtCustomOp* op, int type) = 0; // ConfigOptions @@ -540,6 +610,9 @@ struct ProviderHost { virtual void IndexedSubGraph__SetMetaDef(IndexedSubGraph* p, std::unique_ptr&& meta_def_) = 0; virtual const IndexedSubGraph_MetaDef* IndexedSubGraph__GetMetaDef(const IndexedSubGraph* p) = 0; + virtual void IndexedSubGraph__SetSchemaSource(IndexedSubGraph* p, SourceOfSchema schema_source) = 0; + virtual SourceOfSchema IndexedSubGraph__GetSchemaSource(const IndexedSubGraph* p) = 0; + // KernelDef virtual void KernelDef__operator_delete(KernelDef* p) = 0; virtual int KernelDef__ExecQueueId(const KernelDef* p) = 0; diff --git a/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h b/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h index fd2540b42a3db..831ec6235503d 100644 --- a/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h +++ b/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h @@ -190,6 +190,7 @@ struct NodeProto final { int attribute_size() { return g_host->NodeProto__attribute_size(this); } const AttributeProto& attribute(int index) const { return g_host->NodeProto__attribute(this, index); } AttributeProto* mutable_attribute(int index) { return g_host->NodeProto__mutable_attribute(this, index); } + AttributeProto* add_attribute() { return g_host->NodeProto__add_attribute(this); } NodeProto() = delete; NodeProto(const NodeProto&) = delete; @@ -372,6 +373,80 @@ struct ValueInfoProtos final { PROVIDER_DISALLOW_ALL(ValueInfoProtos) }; + +struct FunctionProto final { + static std::unique_ptr Create() { return g_host->FunctionProto__construct(); } + static void operator delete(void* p) { g_host->FunctionProto__operator_delete(reinterpret_cast(p)); } + + bool SerializeToString(std::string& string) const { return g_host->FunctionProto__SerializeToString(this, string); } + bool SerializeToOstream(std::ostream& output) const { return g_host->FunctionProto__SerializeToOstream(this, output); } + bool ParseFromString(const std::string& data) { return g_host->FunctionProto__ParseFromString(this, data); } + std::string SerializeAsString() const { return g_host->FunctionProto__SerializeAsString(this); } + + bool has_name() const { return g_host->FunctionProto__has_name(this); } + const std::string& name() const { return g_host->FunctionProto__name(this); } + void set_name(const std::string& name) { g_host->FunctionProto__set_name(this, name); } + + bool has_since_version() const { return g_host->FunctionProto__has_since_version(this); } + int since_version() const { return g_host->FunctionProto__since_version(this); } + void set_since_version(int since_version) { g_host->FunctionProto__set_since_version(this, since_version); } + + bool has_status() const { return g_host->FunctionProto__has_status(this); } + const OperatorStatus& status() const { return g_host->FunctionProto__status(this); } + void set_status(const OperatorStatus& status) { g_host->FunctionProto__set_status(this, status); } + + bool has_doc_string() const { return g_host->FunctionProto__has_doc_string(this); } + const std::string& doc_string() const { return g_host->FunctionProto__doc_string(this); } + void set_doc_string(const std::string& doc_string) { g_host->FunctionProto__set_doc_string(this, doc_string); } + + bool has_domain() const { return g_host->FunctionProto__has_domain(this); } + const std::string& domain() const { return g_host->FunctionProto__domain(this); } + void set_domain(const std::string& domain) { g_host->FunctionProto__set_domain(this, domain); } + + const std::string& input(int index) const { return g_host->FunctionProto__input(this, index); } + std::vector* mutable_input() { return g_host->FunctionProto__input(this); } + std::string* mutable_input(int index) { return g_host->FunctionProto__mutable_input(this, index); } + int input_size() const { return g_host->FunctionProto__input_size(this); } + void add_input(const std::string& value) { g_host->FunctionProto__add_input(this, value); } + + const std::string& output(int index) const { return g_host->FunctionProto__output(this, index); } + std::vector* mutable_output() { return g_host->FunctionProto__output(this); } + std::string* mutable_output(int index) { return g_host->FunctionProto__mutable_output(this, index); } + int output_size() const { return g_host->FunctionProto__output_size(this); } + void add_output(const std::string& value) { g_host->FunctionProto__add_output(this, value); } + + const std::string& attribute(int index) const { return g_host->FunctionProto__attribute(this, index); } + std::vector* mutable_attribute() { return g_host->FunctionProto__attribute(this); } + std::string* mutable_attribute(int index) { return g_host->FunctionProto__mutable_attribute(this, index); } + int attribute_size() const { return g_host->FunctionProto__attribute_size(this); } + void add_attribute(const std::string& value) { g_host->FunctionProto__add_attribute(this, value); } + + const AttributeProto& attribute_proto(int index) const { return g_host->FunctionProto__attribute_proto(this, index); } + AttributeProto* mutable_attribute_proto(int index) { return g_host->FunctionProto__mutable_attribute_proto(this, index); } + int attribute_proto_size() const { return g_host->FunctionProto__attribute_proto_size(this); } + AttributeProto* add_attribute_proto() { return g_host->FunctionProto__add_attribute_proto(this); } + + const NodeProto& node(int index) const { return g_host->FunctionProto__node(this, index); } + NodeProto* mutable_node(int index) { return g_host->FunctionProto__mutable_node(this, index); } + int node_size() const { return g_host->FunctionProto__node_size(this); } + NodeProto* add_node() { return g_host->FunctionProto__add_node(this); } + + const ValueInfoProto& value_info(int index) const { return g_host->FunctionProto__value_info(this, index); } + ValueInfoProtos* mutable_value_info() { return g_host->FunctionProto__mutable_value_info(this); } + ValueInfoProto* mutable_value_info(int index) { return g_host->FunctionProto__mutable_value_info(this, index); } + int value_info_size() const { return g_host->FunctionProto__value_info_size(this); } + ValueInfoProto* add_value_info() { return g_host->FunctionProto__add_value_info(this); } + + const StringStringEntryProto& metadata_props(int index) const { return g_host->FunctionProto__metadata_props(this, index); } + StringStringEntryProtos* mutable_metadata_props() { return g_host->FunctionProto__mutable_metadata_props(this); } + StringStringEntryProto* mutable_metadata_props(int index) { return g_host->FunctionProto__mutable_metadata_props(this, index); } + int metadata_props_size() const { return g_host->FunctionProto__metadata_props_size(this); } + StringStringEntryProto* add_metadata_props() { return g_host->FunctionProto__add_metadata_props(this); } + + FunctionProto() = delete; + FunctionProto(const FunctionProto&) = delete; + void operator=(const FunctionProto&) = delete; +}; } // namespace ONNX_NAMESPACE namespace onnxruntime { @@ -450,6 +525,13 @@ struct IndexedSubGraph_MetaDef final { }; struct IndexedSubGraph final { + // Mirroring the enum defined in indexed_sub_graph.h. + enum class SourceOfSchema : uint8_t { + CREATE, + REUSE_OR_CREATE, + EXISTING, + }; + static std::unique_ptr Create() { return g_host->IndexedSubGraph__construct(); } static void operator delete(void* p) { g_host->IndexedSubGraph__operator_delete(reinterpret_cast(p)); } @@ -458,6 +540,9 @@ struct IndexedSubGraph final { void SetMetaDef(std::unique_ptr&& meta_def_) { return g_host->IndexedSubGraph__SetMetaDef(this, std::move(*reinterpret_cast*>(&meta_def_))); } const IndexedSubGraph_MetaDef* GetMetaDef() const { return reinterpret_cast(g_host->IndexedSubGraph__GetMetaDef(this)); } + void SetSchemaSource(SourceOfSchema schema_source) { return g_host->IndexedSubGraph__SetSchemaSource(this, schema_source); } + SourceOfSchema GetSchemaSource() const { return g_host->IndexedSubGraph__GetSchemaSource(this); } + IndexedSubGraph() = delete; IndexedSubGraph(const IndexedSubGraph&) = delete; void operator=(const IndexedSubGraph&) = delete; diff --git a/onnxruntime/core/providers/vitisai/imp/ep_context_utils.cc b/onnxruntime/core/providers/vitisai/imp/ep_context_utils.cc new file mode 100644 index 0000000000000..bac6f699fc801 --- /dev/null +++ b/onnxruntime/core/providers/vitisai/imp/ep_context_utils.cc @@ -0,0 +1,333 @@ +#include "ep_context_utils.h" + +// 1st-party headers/libs. +#include "core/graph/graph.h" + + +namespace fs = std::filesystem; + +namespace onnxruntime { + +std::unique_ptr +VitisAIExecutionProvider::ConvertIndexedSubGraphToFunctionProto( + const IndexedSubGraph& sub_graph, const Graph& parent_graph) { + auto p_func_proto = FunctionProto::Create(); + const auto* p_meta_def = sub_graph.GetMetaDef(); + if (p_meta_def) { + p_func_proto->set_name(p_meta_def->name()); + p_func_proto->set_domain(p_meta_def->domain()); + p_func_proto->set_since_version(p_meta_def->since_version()); + p_func_proto->set_status(p_meta_def->status()); + for (const auto& input : p_meta_def->inputs()) { + p_func_proto->add_input(input); + } + auto* p_metadata_props_0 = p_func_proto->add_metadata_props(); + *(p_metadata_props_0->mutable_key()) = "meta_def_inputs_size"; + *(p_metadata_props_0->mutable_value()) = std::to_string(p_meta_def->inputs().size()); + for (const auto& output : p_meta_def->outputs()) { + p_func_proto->add_output(output); + } + // XXX: SerDes with different fields. + for (const auto& initializer : p_meta_def->constant_initializers()) { + p_func_proto->add_input(initializer); + } + // XXX: SerDes with different numbers of fields. + for (const auto& attr_pair : p_meta_def->attributes()) { + p_func_proto->add_attribute(attr_pair.first); + auto* p_attr_proto = p_func_proto->add_attribute_proto(); + *p_attr_proto = attr_pair.second; + } + p_func_proto->set_doc_string(p_meta_def->doc_string()); + // TODO: `MetaDef::type_and_shape_inference_function`. + } + auto p_parent_graph_proto = parent_graph.ToGraphProto(); + for (auto node_index : sub_graph.Nodes()) { + auto* p_node_proto = p_parent_graph_proto->mutable_node(node_index); + auto* p_attr_proto = p_node_proto->add_attribute(); + p_attr_proto->set_name("parent_graph_node_index"); + p_attr_proto->set_type(ONNX_NAMESPACE::AttributeProto::INT); + p_attr_proto->set_i(node_index); + *(p_func_proto.add_node()) = *p_node_proto; + } +#if 0 + // Alternative. + for (const auto node_index : sub_graph.Nodes()) { + const auto* p_node = parent_graph.GetNode(node_index); + auto p_node_proto = ONNX_NAMESPACE::NodeProto::Create(); + // XXX + p_node->ToProto(*p_node_proto, true); + p_attr_proto->set_name("parent_graph_node_index"); + p_attr_proto->set_type(ONNX_NAMESPACE::AttributeProto::INT); + p_attr_proto->set_i(node_index); + *(p_func_proto.add_node()) = *p_node_proto; + } +#endif + auto* p_metadata_props_1 = p_func_proto->add_metadata_props(); + *p_metadata_props_1->mutable_key() = "schema_source"; + *p_metadata_props_1->mutable_value() = std::to_string(static_cast(sub_graph.GetSchemaSource())); + return p_func_proto; +} + +std::unique_ptr ConvertFunctionProtoToIndexedSubGraph( + const std::unique_ptr& p_func_proto) { + auto p_isg = IndexedSubGraph::Create(); + // "meta_def_inputs_size" (optional) and "schema_source". + int func_metadata_props_size = p_func_proto->metadata_props_size(); + // Precisely, func_metadata_props_size == 2, which implies + // `IndexedSubGraph::meta_def_` is not null and `IndexedSubGraph::nodes` > 1. + if (func_metadata_props_size > 1) { + auto& prop = p_func_proto->metadata_props(0); + int isg_meta_def_inputs_size = std::stoi(*(prop.mutable_value())); + auto p_meta_def = IndexedSubGraph_MetaDef::Create(); + p_meta_def->name() = p_func_proto->name(); + p_meta_def->domain() = p_func_proto->domain(); + p_meta_def->since_version() = p_func_proto->since_version(); + p_meta_def->status() = p_func_proto->status(); + auto& meta_def_inputs = p_meta_def->inputs(); + for (int i = 0; i < isg_meta_def_inputs_size; i++) { + meta_def_inputs.push_back(p_func_proto->input(i)); + } + auto& meta_def_outputs = p_meta_def->outputs(); + for (int i = 0, l = p_func_proto->output_size(); i < l; i++) { + meta_def_outputs.push_back(p_func_proto->output(i)); + } + auto& meta_def_initializers = p_meta_def->constant_initializers(); + for (int i = isg_metadef_inputs_size, l = p_func_proto->input_size(); i < l; i++) { + meta_def_initializers.push_back(p_func_proto->input(i)); + } + auto& meta_def_attrs = p_meta_def->attributes(); + for (int i = 0, l = p_func_proto->attribute_size(); i < l; i++) { + meta_def_attrs.emplace(p_func_proto->attribute(i), p_func_proto->attribute_proto(i)); + } + p_meta_def->doc_string() = p_func_proto->doc_string(); + // TODO: `IndexedSubGraph::type_and_shape_inference_function`. + p_isg->SetMetaDef(std::move(p_meta_def)); + } + auto& isg_nodes = p_isg->Nodes(); + for (int j = 0, l = p_func_proto->node_size(); j < l; j++) { + isg_nodes.push_back(p_func_proto->node(j).i()); + } + auto schema_source = static_cast( + std::stoi(*(p_func_proto->metadata_props(func_metadata_props_size - 1).mutable_value()))); + p_isg->SetSchemaSource(schema_source); + return p_isg; +} + +std::string SerializeCapabilities( + const std::vector>& capability_ptrs, + const Graph& graph) { + std::stringstream ss; + for (const auto& p : capability_ptrs) { + auto& p_subgraph = p->Subgraph(); + auto p_func_proto = ConvertIndexedSubGraphToFunctionProto(*p_subgraph, graph); + std::string func_proto_buf; + p_func_proto->SerializeToString(&func_proto_buf); + size_t buf_len = func_proto_buf.length(); + ss.write(reinterpret_cast(&buf_len), sizeof(buf_len)); + ss.write(func_proto_buf.data(), buf_len); + } + static_assert(ss.good(), "Serialization stream bad"); + return ss.str(); +} + +void DeserializeCapabilities(const string& ser_capabilities, + std::vector>& capability_ptrs) { + std::stringstream ss(ser_capabilities); + while (!ss.eof()) { + size_t buf_len; + ss.read(reinterpret_cast(&buf_len), sizeof(buf_len)); + std::string buf(buf_len, '\0'); + ss.read(&buf[0], buf_len); + auto p_func_proto = ONNX_NAMESPACE::FunctionProto::Create(); + p_func_proto->ParseFromString(buf); + auto p_subgraph = ConvertFunctionProtoToIndexedSubGraph(p_func_proto); + capability_ptrs.push_back(ComputeCapability::Create(std::move(p_subgraph))); + } +} + +std::unique_ptr CreateEPContexModel( + const GraphViewer& graph_viewer, + const std::string& serialized_ctx_cache, + const std::string& ctx_cache_file_loc, + const int64_t embed_mode, + const logging::Logger* p_logger) { + // Create a new graph/model, reusing the graph name, + // the op-domain-to-opset-version map, + // and the op schema registry of the current graph. + auto& ep_ctx_graph = graph_viewer.CreateModel(*p_logger)->MainGraph(); + + std::vector input_node_arg_ptrs; + // XXX: vs `GraphViewer::GetInputsIncludingInitializers()`. + for (const auto* p_node_arg : graph_viewer.GetInputs()) { + auto& temp_node_arg = ep_ctx_graph.GetOrCreateNodeArg( + p_node_arg->Name(), p_node_arg->TypeAsProto()); + input_node_arg_ptrs.push_back(&temp_node_arg); + } + std::vector output_node_arg_ptrs; + for (const auto* p_node_arg: graph_viewer.GetOutputs()) { + auto& temp_node_arg = ep_ctx_graph.GetOrCreateNodeArg(output->Name(), output->TypeAsProto()); + output_node_arg_ptrs.push_back(&temp_node_arg); + } + + // Attr "embed_mode". + auto* p_attr_0 = ONNX_NAMESPACE::AttributeProto::Create(); + p_attr_0->set_name(kEmbedModeAttr); + //p_attr_0->set_type(onnx::AttributeProto_AttributeType_INT); + p_attr_0->set_type(ONNX_NAMESPACE::AttributeProto::INT); + p_attr_0->seti(embed_mode); + // Attr "ep_cache_context". + auto* p_attr_1 = ONNX_NAMESPACE::AttributeProto::Create(); + p_attr_1->set_name(kEPCacheContextAttr); + //p_attr_1->set_type(onnx::AttributeProto_AttributeType_STRING); + p_attr_1->set_type(ONNX_NAMESPACE::AttributeProto::STRING); + p_attr_1->set_s(embed_mode == 0 ? ctx_cache_file_loc : serialized_ctx_cache); + // Attr "source". + auto* p_attr_2 = ONNX_NAMESPACE::AttributeProto::Create(); + p_attr_2->set_name(kSourceAttr); + //p_attr_2->set_type(onnx::AttributeProto_AttributeType_STRING); + p_attr_2->set_type(ONNX_NAMESPACE::AttributeProto::STRING); + p_attr_2->set_s(kVitisAIExecutionProvider); + + auto p_node_attrs = NodeAttributes::Create(); + constexpr int num_attrs = 3; + p_node_attrs->reserve(num_attrs); + p_node_attrs->emplace(kEmbedModeAttr, *p_attr_0); + p_node_attrs->emplace(kEPCacheContextAttr, *p_attr_1); + p_node_attrs->emplace(kSourceAttr, *p_attr_2); + + ep_ctx_graph.AddNode(kEPContextOp, kEPContextOp, "", input_node_arg_ptrs, output_node_arg_ptrs, p_node_attrs.get(), kEPContextOpDomain); + ORT_ENFORCE(ep_ctx_graph.Resolve().IsOK()); + auto p_ep_ctx_graph_viewer = ep_ctx_graph.CreateGraphViewer(); + auto p_ep_ctx_model = p_ep_ctx_graph_viewer->CreateModel(*p_logger); + auto p_ep_ctx_model_proto = p_ep_ctx_model->ToProto(); + p_ep_ctx_model_proto->set_ir_version(ONNX_NAMESPACE::Version::IR_VERSION); + p_ep_ctx_graph_viewer->ToProto(*(p_ep_ctx_model_proto->mutable_graph()), true, true); + + return p_ep_ctx_model; +} + +void DumpEPContextModel( + const std::unique_ptr& p_model, const std::string& ep_ctx_model_file_loc) { + std::fstream dump_stream(ep_ctx_model_file_loc, std::ios::out | std::ios::trunc | std::ios::binary); + p_model->ToProto()->SerializeToOstream(dump_stream); + LOGS_DEFAULT(VERBOSE) << "[VitisAI EP] Dumped " << ep_ctx_model_file_loc; +} + +bool ValidateEPContextNode(const Graph& graph) { + // TODO: Support for multi-node EP context model. + assert(graph.Nodes().size() == 1); + auto* p_node = graph.GetNode(0); + assert(p_node->OpType() == kEPContextOp); + auto& attrs = p_node->GetAttributes(); + assert(attrs.count(kEmbedModeAttr) > 0); + assert(attrs.count(kEPCacheContextAttr) > 0); + assert(attrs.count(kSourceAttr) > 0); + return true; +} + +std::string RetrieveEPContextCache(const Graph& graph) { + if (!ValidateEPContextNode(graph)) { + ORT_THROW("Invalid EP context model for Vitis AI"); + } + // TODO: Support for multi-node EP context model. + auto* p_node = graph.GetNode(0); + auto& attrs = p_node->GetAttributes(); + int64_t embed_mode = attrs.at(kEmbedModeAttr).i(); + std::string& ep_ctx_cache = attrs.at(kEPCacheContextAttr).s(); + if (embed_mode) { + return ep_ctx_cache; + } + fs::path ep_ctx_file_loc(ep_ctx_cache); + // TODO: Validaion of the file location to make sure security is met. + if (!fs::exists(ep_ctx_file_loc) || !fs::is_regular_file(ep_ctx_file_loc)) { + ORT_THROW("File for EP context cache is missing"); + } + std::ifstream ifs(ep_ctx_cache, std::ios::binary | std::ios::in); + if (!ifs.is_open()) { + ORT_THROW("Exception opening EP context cache file"); + } + ifs.seekg(0, ifs.end); + int cache_len = ifs.tellg(); + ifs.seekg(0, ifs.beg); + char* buf = new char[cache_len]; + ifs.read(buf, cache_len); + if (!ifs.good()) { + ifs.close(); + ORT_THROW("Exception reading EP context cache file"); + } + ifs.close(); + std::string cache_payload(buf); + delete[] buf; + return cache_payload; +} + +bool GraphHasEPContextNode(const GraphViewer& graph_viewer) { + for (size_t i = 0, l = static_cast(graph_viewer.MaxNodeIndex()); i < l; i++) { + auto* p_node = graph_viewer.GetNode(i); + if (p_node != nullptr && p_node->OpType() == kEPContextOp) { + NodeAttrHelper node_attr_helper(*p_node); + if (node_attr_helper.Get(kSourceAttr, "") == kVitisAIExecutionProvider) { + return true; + } + } + } + return false; +} + +bool FusedGraphHasEPContextNode( + const std::vector& fused_nodes_and_graphs) { + for (const auto& fused_node_graph : fused_nodes_and_graphs) { + bool has_node = GraphHasEPContextNode(fused_node_graph.filtered_graph); + if (has_node) { + return true; + } + } + return false; +} + +const Path& GetTopLevelModelPath(const GraphViewer& graph_viewer) { + const auto& graph = graph_viewer.GetGraph(); + const Graph* p_graph = &graph; + while (p_graph->IsSubgraph()) { + p_graph = p_graph->ParentGraph(); + } + return p_graph->ModelPath(); +} + +bool GetEPContextModelFileLocation( + const std::string& ep_ctx_model_path_cfg, + const PathString& model_path_str, + bool is_ep_ctx_model, + PathString& ep_ctx_model_file_loc) { + //if (!ep_ctx_model_file_loc.empty()) { + // return true; + //} + if (!ep_ctx_model_path_cfg.emtpy()) { + ep_ctx_model_file_loc = ToPathString(ep_ctx_model_path_cfg); + } else if (!model_path_str.emtpy()) { + if (is_ep_ctx_model) { + ep_ctx_model_file_loc = model_path_str; + } else { + ep_ctx_model_file_loc = + ToPathString(fs::path(model_path_str).stem().string() + "_ctx.onnx"); + } + } + return !ep_ctx_model_file_loc.empty() && fs::exists(ep_ctx_model_file_loc) && fs::is_regular_file(ep_ctx_model_file_loc); +} + +// The file for EP context binary is in the same folder as the EP context model file. +PathString GetEPContextCacheFileLocation( + const PathString& ep_ctx_model_file_loc, const PathString& model_path_str) { + if (!ep_ctx_model_file_loc.emtpy()) { + fs::path ep_ctx_model_fs_path(ep_ctx_model_file_loc); + auto ep_ctx_cache_fs_path = + ep_ctx_model_fs_path.replace_extension(fs::path("__ep_ctx_cache.bin")); + return ToPathString(ep_ctx_cache_fs_path.string()); + } + fs::path model_fs_path(model_path_str); + auto ep_ctx_cache_fs_path = + model_fs_path.replace_extension(fs::path("__ep_ctx_cache.bin")); + return ToPathString(ep_ctx_cache_fs_path.string()); +} + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/vitisai/include/ep_context_utils.h b/onnxruntime/core/providers/vitisai/include/ep_context_utils.h new file mode 100644 index 0000000000000..4531354b7e693 --- /dev/null +++ b/onnxruntime/core/providers/vitisai/include/ep_context_utils.h @@ -0,0 +1,57 @@ +#pragma once + +// Standard headers/libs. +#include +#include +#include + +// 1st-party headers/libs. +#include "core/providers/shared_library/provider_api.h" + + +namespace onnxruntime { + +static constexpr const char* kEPContextOp = "EPContext"; +static constexpr const char* kMainContextAttr = "main_context"; +static constexpr const char* kEPCacheContextAttr = "ep_cache_context"; +static constexpr const char* kEmbedModeAttr = "embed_mode"; +static constexpr const char* kPartitionNameAttr = "partition_name"; +static constexpr const char* kSourceAttr = "source"; +static constexpr const char* kEPSDKVersionAttr = "ep_sdk_version"; +static constexpr const char* kEPContextOpDomain = "com.microsoft"; + +std::unique_ptr +ConvertIndexedSubGraphToFunctionProto(const IndexedSubGraph&, const Graph&); + +std::unique_ptr ConvertFunctionProtoToIndexedSubGraph( + const std::unique_ptr&); + +std::string SerializeCapabilities( + const std::vector>&, const Graph&); + +void DeserializeCapabilities( + const string&, std::vector>&); + +std::unique_ptr CreateEPContexModel(const GraphViewer&, const std::string&, + const std::string&, const int64_t, const logging::Logger*); + +void DumpEPContextModel(std::unique_ptr&, const std::string&); + +bool ValidateEPContextNode(const Graph&); + +std::string RetrieveEPContextCache(const Graph&); + +bool GraphHasEPContextNode(const GraphViewer&); + +bool FusedGraphHasEPContextNode( + const std::vector&); + +const Path& GetTopLevelModelPath(const GraphViewer&); + +bool GetEPContextModelFileLocation( + const std::string&, const PathString&, bool, PathString&); + +// The file for EP context binary is in the same folder as the EP context model file. +PathString GetEPContextCacheFileLocation(const PathString&, const PathString&); + +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/vitisai/vitisai_execution_provider.cc b/onnxruntime/core/providers/vitisai/vitisai_execution_provider.cc index 6fc09f3495aa1..ffc55dbbe9e69 100644 --- a/onnxruntime/core/providers/vitisai/vitisai_execution_provider.cc +++ b/onnxruntime/core/providers/vitisai/vitisai_execution_provider.cc @@ -5,19 +5,69 @@ #include #include #include +#include #include "vaip/capability.h" #include "vaip/global_api.h" +#include "ep_context_utils.h" using namespace ONNX_NAMESPACE; +namespace fs = std::filesystem; + namespace onnxruntime { constexpr const char* VITISAI = "VITISAI"; VitisAIExecutionProvider::VitisAIExecutionProvider( - const ProviderOptions& info) - : IExecutionProvider{onnxruntime::kVitisAIExecutionProvider}, info_(info) { + const ProviderOptions& info, const SessionOptions* p_sess_opts) + : IExecutionProvider{onnxruntime::kVitisAIExecutionProvider}, info_(info) { CreateKernelRegistry(); + + if (p_sess_opts) { + ep_ctx_enabled_ = p_sess_opts->config_options.GetConfigOrDefault( + kOrtSessionOptionEpContextEnable, "0") == "1"; + std::string embed_mode = p_sess_opts->config_options.GetConfigOrDefault( + kOrtSessionOptionEpContextEmbedMode, "1"); + if ("1" == embed_mode) { + ep_ctx_embed_mode_ = true; + } else if ("0" == embed_mode) { + ep_ctx_embed_mode_ = false; + } else { + LOGS_DEFAULT(VERBOSE) << "Invalid ep.context_embed_mode: " << embed_mode << " only 0 or 1 allowed. Set to 1."; + ep_ctx_embed_mode_ = true; + } + ep_ctx_model_path_cfg_ = p_sess_opts->config_options.GetConfigOrDefault( + kOrtSessionOptionEpContextFilePath, ""); + } else { + auto it = info_.find("ep_context_enable"); + ep_ctx_enabled_ = it != info_.end() && it->second == "1"; + it = info_.find("ep_context_embed_mode"); + ep_ctx_embed_mode_ = it != info_.end() && it->second != "0"; + //ep_ctx_embed_mode_ = it == info_.end() || it->second != "0"; + it = info_.find("ep_context_file_path"); + ep_ctx_model_path_cfg_ = it == info_.end() ? "" : it->second; + } + LOGS_DEFAULT(VERBOSE) << "EP Context cache enabled: " << ep_ctx_enabled_; + LOGS_DEFAULT(VERBOSE) << "EP context cache embed mode: " << ep_ctx_embed_mode_; + LOGS_DEFAULT(VERBOSE) << "User specified EP context cache path: " << ep_ctx_model_path_cfg_; +} + +#if 0 +VitisAIExecutionProvider::~VitisAIExecutionProvider() { + // TODO: EP context related sources. +} +#endif + +void VitisAIExecutionProvider::LoadEPContexModelFromFile() { + // XXX: should "p_ep_ctx_model_" be checked or not? + if (!p_ep_ctx_model_ && !ep_ctx_model_file_loc_.empty()) { + auto p_model_proto = ONNX_NAMESPACE::ModelProto::Create(); + auto status = Model::Load(ep_ctx_model_file_loc_, *p_model_proto); + static_assert(status.IsOK(), "Loading EP context model failed"); + auto& logger = logging::LoggingManager::DefaultLogger(); + p_ep_ctx_model_ = Model::Create(std::move(*p_model_proto), ep_ctx_model_file_loc_, nullptr, logger); + LOGS_DEFAULT(VERBOSE) << "Loaded EP context model from: " << ep_ctx_model_file_loc_; + } } void VitisAIExecutionProvider::CreateKernelRegistry() { @@ -30,9 +80,71 @@ void VitisAIExecutionProvider::CreateKernelRegistry() { std::shared_ptr VitisAIExecutionProvider::GetKernelRegistry() const { return get_kernel_registry_vitisaiep(); } +// This method is called after both `GetComputeCapabilityOps()` and `Compile()`. +// This timing is required to work with both compliation-based EPs and non-compilation-based EPs. +const InlinedVector VitisAIExecutionProvider::GetEpContextNodes() const { + InlinedVector ep_context_node_ptrs; + // All preconditions are supposed to have happened. + if (p_ep_ctx_model_) { + auto& graph = p_ep_ctx_model_->MainGraph(); + for (const auto* p_node : graph.Nodes()) { + ep_context_node_ptrs.push_back(p_node); + } + } + return ep_context_node_ptrs; +} + +// Create EP context model and dump it for future use. +// This implementation here is only working for non-compilation-based EPs. +void VitisAIExecutionProvider::FulfillEPContextEnablement( + const std::vector>& capability_ptrs, + const onnxruntime::GraphViewer& graph_viewer) { + auto& logger = logging::LoggingManager::DefaultLogger() + auto& model_path_str = GetTopLevelModelPath(graph_viewer).ToPathString(); + auto ep_ctx_payload = SerializeCapabilities(capability_ptrs, graph_viewer.GetGraph()); + if (!ep_ctx_embed_mode_) { + GetEPContextModelFileLocation(ep_ctx_model_path_cfg_, model_path_str, false, ep_ctx_model_file_loc_); + auto ep_ctx_cache_fs_path = GetEPContextCacheFileLocation(ep_ctx_model_file_loc_, model_path_str); + std::ofstream ep_ctx_cache_ofs(ep_ctx_cache_fs_path.c_str(), std::ios::trunc | std::ios::binary); + static_assert(ep_ctx_cache_ofs.is_open(), "Failed to open a file to write EP context cache"); + ep_ctx_cache_ofs.write(ep_ctx_payload.c_str(), ep_ctx_payload.length()); + if (!ep_ctx_cache_ofs.good()) { + ep_ctx_cache_ofs.close(); + ORT_THROW("Exception writing EP context cache file"); + } + ep_ctx_cache_ofs.close(); + p_ep_ctx_model_ = CreateEPContexModel(graph_viewer, ep_ctx_payload, ep_ctx_cache_fs_path.string(), 0, &logger); + } else { + p_ep_ctx_model_ = CreateEPContexModel(graph_viewer, ep_ctx_payload, "", 1, &logger); + } + DumpEPContextModel(p_ep_ctx_model_, ep_ctx_model_file_loc_); +} + std::vector> VitisAIExecutionProvider::GetCapability( - const onnxruntime::GraphViewer& graph, const IKernelLookup& /*kernel_lookup*/) const { - if (graph.IsSubgraph()) { + const onnxruntime::GraphViewer& graph_viewer, const IKernelLookup& /*kernel_lookup*/) const { + bool is_ep_ctx_model = GraphHasEPContextNode(graph_viewer); + if (is_ep_ctx_model) { + auto ep_ctx_payload = RetrieveEPContextCache(graph_viewer.GetGraph()); + std::vector> capability_ptrs; + DeserializeCapabilities(ep_ctx_payload, capability_ptrs); + return capability_ptrs; + } else { + // FIXME: Will it make sense to do this? + // One of the potential problems is the existing EP-context model file may be stale. + auto& model_path_str = GetTopLevelModelPath(graph_viewer).ToPathString(); + if (GetEPContextModelFileLocation( + ep_ctx_model_path_cfg_, model_path_str, false, ep_ctx_model_file_loc_)) { + LOGS_DEFAULT(WARNING) << "The inference session was created with a normal ONNX model " + << "but a model file with EP context cache exists at " << ep_ctx_model_file_loc_.c_str(); + LoadEPContexModelFromFile(); + auto ep_ctx_payload = RetrieveEPContextCache(p_ep_ctx_model_->MainGraph()); + std::vector> capability_ptrs; + DeserializeCapabilities(ep_ctx_payload, capability_ptrs); + return capability_ptrs; + } + } + + if (graph_viewer.IsSubgraph()) { // VITIS AI EP not support sungraph. Assigned to CPU. return {}; } @@ -40,13 +152,16 @@ std::vector> VitisAIExecutionProvider::GetCap // Only compiling a model once is currently supported return {}; } - execution_providers_ = std::make_unique(compile_onnx_model(graph, *GetLogger(), info_)); - auto result = vaip::GetComputeCapabilityOps(graph, execution_providers_.get(), vitisai_optypes_); + execution_providers_ = std::make_unique(compile_onnx_model(graph_viewer, *GetLogger(), info_)); + auto result = vaip::GetComputeCapabilityOps(graph_viewer, execution_providers_.get(), vitisai_optypes_); size_t index = 0u; for (auto& ep : **execution_providers_) { - result.emplace_back(vaip::XirSubgraphToComputeCapability1(graph, ep.get(), index)); + result.emplace_back(vaip::XirSubgraphToComputeCapability1(graph_viewer, ep.get(), index)); index = index + 1; } + if (ep_ctx_enabled_) { + FulfillEPContextEnablement(result, graph_viewer); + } return result; } diff --git a/onnxruntime/core/providers/vitisai/vitisai_execution_provider.h b/onnxruntime/core/providers/vitisai/vitisai_execution_provider.h index 186427be4fab2..15ec59c9691b0 100644 --- a/onnxruntime/core/providers/vitisai/vitisai_execution_provider.h +++ b/onnxruntime/core/providers/vitisai/vitisai_execution_provider.h @@ -9,8 +9,10 @@ #include #include +#include "core/framework/session_options.h" #include "core/providers/shared_library/provider_api.h" #include "core/session/onnxruntime_c_api.h" +#include "core/common/inlined_containers_fwd.h" // we cannot include vaip/vaip.hpp here because header file referred by // onnxruntime_pybind_state_common.cc @@ -23,10 +25,11 @@ namespace onnxruntime { // Logical device representation. class VitisAIExecutionProvider : public IExecutionProvider { public: - explicit VitisAIExecutionProvider(const ProviderOptions& info); + explicit VitisAIExecutionProvider(const ProviderOptions& info, + const SessionOptions* p_sess_opts = nullptr); ~VitisAIExecutionProvider() = default; - std::vector> GetCapability(const onnxruntime::GraphViewer& graph, + std::vector> GetCapability(const onnxruntime::GraphViewer& graph_viewer, const IKernelLookup& /*kernel_lookup*/) const override; int GetDeviceId() const { return 0; } @@ -35,6 +38,10 @@ class VitisAIExecutionProvider : public IExecutionProvider { std::vector& node_compute_funcs) override; std::shared_ptr GetKernelRegistry() const override; + // This method is called after both `GetComputeCapabilityOps()` and `Compile()`. + // This timing is required to work with both compliation-based EPs and non-compilation-based EPs. + const InlinedVector GetEpContextNodes() const override; + private: void CreateKernelRegistry(); using my_ep_t = vaip_core::DllSafe>>; @@ -45,6 +52,21 @@ class VitisAIExecutionProvider : public IExecutionProvider { std::vector custom_op_domains_; std::shared_ptr registry_; std::set vitisai_optypes_; + // EP context related. + bool ep_ctx_enabled_ = false; + bool ep_ctx_embed_mode_ = true; + std::string ep_ctx_model_path_cfg_{""}; + PathString ep_ctx_model_file_loc_{""}; + // FIXME: This might not be needed. + std::unique_ptr p_ep_ctx_model_; + // It might need to be called before loading + // the EP context model that is compiled AOT/offline. + void LoadEPContexModelFromFile(); + // Create EP context model and dump it for future use. + // This implementation here is only working for non-compilation-based EPs. + void FulfillEPContextEnablement( + const std::vector>&, + const onnxruntime::GraphViewer&); }; } // namespace onnxruntime diff --git a/onnxruntime/core/providers/vitisai/vitisai_provider_factory.cc b/onnxruntime/core/providers/vitisai/vitisai_provider_factory.cc old mode 100755 new mode 100644 diff --git a/onnxruntime/core/session/provider_bridge_ort.cc b/onnxruntime/core/session/provider_bridge_ort.cc index d18b3ac40d489..ad24e0b643f30 100644 --- a/onnxruntime/core/session/provider_bridge_ort.cc +++ b/onnxruntime/core/session/provider_bridge_ort.cc @@ -526,6 +526,7 @@ struct ProviderHostImpl : ProviderHost { int NodeProto__attribute_size(ONNX_NAMESPACE::NodeProto* p) override { return p->attribute_size(); } const ONNX_NAMESPACE::AttributeProto& NodeProto__attribute(const ONNX_NAMESPACE::NodeProto* p, int index) const override { return p->attribute(index); } ONNX_NAMESPACE::AttributeProto* NodeProto__mutable_attribute(ONNX_NAMESPACE::NodeProto* p, int index) override { return p->mutable_attribute(index); } + ONNX_NAMESPACE::AttributeProto* NodeProto__add_attribute(ONNX_NAMESPACE::NodeProto* p) override { return p->add_attribute(); } // TensorProto (wrapped) std::unique_ptr TensorProto__construct() override { return std::make_unique(); } @@ -600,6 +601,75 @@ struct ProviderHostImpl : ProviderHost { const ONNX_NAMESPACE::ValueInfoProto& ValueInfoProtos__operator_array(const ONNX_NAMESPACE::ValueInfoProtos* p, int index) override { return (*p)[index]; } + // FunctionProto (wrapped) + std::unique_ptr FunctionProto__construct() override { return std::make_unique(); } + void FunctionProto__operator_delete(ONNX_NAMESPACE::FunctionProto* p) override { delete p; } + + bool FunctionProto__SerializeToString(const ONNX_NAMESPACE::FunctionProto* p, std::string& string) override { return p->SerializeToString(&string); } + bool FunctionProto__SerializeToOstream(const ONNX_NAMESPACE::FunctionProto* p, std::ostream& output) override { return p->SerializeToOstream(&output); } + bool FunctionProto__ParseFromString(ONNX_NAMESPACE::FunctionProto* p, const std::string& data) override { return p->ParseFromString(data); } + std::string FunctionProto__SerializeAsString(const ONNX_NAMESPACE::FunctionProto* p) override { return p->SerializeAsString(); } + + bool FunctionProto__has_name(const ONNX_NAMESPACE::FunctionProto* p) override { return p->has_name(); } + const std::string& FunctionProto__name(const ONNX_NAMESPACE::FunctionProto* p) const override { return p->name(); } + void FunctionProto__set_name(ONNX_NAMESPACE::FunctionProto* p, const std::string& name) override { p->set_name(name); } + + bool FunctionProto__has_since_version(const ONNX_NAMESPACE::FunctionProto* p) override { return p->has_since_version(); } + int FunctionProto__since_version(const ONNX_NAMESPACE::FunctionProto* p) const override { return p->since_version(); } + void FunctionProto__set_since_version(ONNX_NAMESPACE::FunctionProto* p, int since_version) override { p->set_since_version(since_version); } + + bool FunctionProto__has_status(const ONNX_NAMESPACE::FunctionProto* p) override { return p->has_status(); } + const ONNX_NAMESPACE::OperatorStatus& FunctionProto__status(const ONNX_NAMESPACE::FunctionProto* p) const override { return p->status(); } + void FunctionProto__set_status(ONNX_NAMESPACE::FunctionProto* p, const ONNX_NAMESPACE::OperatorStatus& status) override { p->set_status(status); } + + bool FunctionProto__has_doc_string(const ONNX_NAMESPACE::FunctionProto* p) override { return p->has_doc_string(); } + const std::string& FunctionProto__doc_string(const ONNX_NAMESPACE::FunctionProto* p) const override { return p->doc_string(); } + void FunctionProto__set_doc_string(ONNX_NAMESPACE::FunctionProto* p, const std::string& doc_string) override { p->set_doc_string(doc_string); } + + bool FunctionProto__has_domain(const ONNX_NAMESPACE::FunctionProto* p) override { return p->has_domain(); } + const std::string& FunctionProto__domain(const ONNX_NAMESPACE::FunctionProto* p) const override { return p->domain(); } + void FunctionProto__set_domain(ONNX_NAMESPACE::FunctionProto* p, const std::string& domain) override { p->set_domain(domain); } + + const std::string& FunctionProto__input(const ONNX_NAMESPACE::FunctionProto* p, int index) override { return p->input(index); } + std::vector* FunctionProto__mutable_input(ONNX_NAMESPACE::FunctionProto* p) override { return p->mutable_input(); } + std::string* FunctionProto__mutable_input(ONNX_NAMESPACE::FunctionProto* p, int index) override { return p->mutable_input(index); } + int FunctionProto__input_size(const ONNX_NAMESPACE::FunctionProto* p) override { return p->input_size(); } + void FunctionProto__add_input(ONNX_NAMESPACE::FunctionProto* p, const std::string& value) override { p->add_input(value); } + + const std::string& FunctionProto__output(const ONNX_NAMESPACE::FunctionProto* p, int index) override { return p->output(index); } + std::vector* FunctionProto__mutable_output(ONNX_NAMESPACE::FunctionProto* p) override { return p->mutable_output(); } + std::string* FunctionProto__mutable_output(ONNX_NAMESPACE::FunctionProto* p, int index) override { return p->mutable_output(index); } + int FunctionProto__output_size(const ONNX_NAMESPACE::FunctionProto* p) override { return p->output_size(); } + void FunctionProto__add_output(ONNX_NAMESPACE::FunctionProto* p, const std::string& value) override { p->add_output(value); } + + const std::string& FunctionProto__attribute(const ONNX_NAMESPACE::FunctionProto* p, int index) override { return p->attribute(index); } + std::vector* FunctionProto__mutable_attribute(ONNX_NAMESPACE::FunctionProto* p) override { return p->mutable_attribute(); } + std::string* FunctionProto__mutable_attribute(ONNX_NAMESPACE::FunctionProto* p, int index) override { return p->mutable_attribute(index); } + int FunctionProto__attribute_size(const ONNX_NAMESPACE::FunctionProto* p) override { return p->attribute_size(); } + void FunctionProto__add_attribute(ONNX_NAMESPACE::FunctionProto* p, const std::string& value) override { p->add_attribute(value); } + + const ONNX_NAMESPACE::AttributeProto& FunctionProto__attribute_proto(const ONNX_NAMESPACE::FunctionProto* p, int index) override { return p->attribute_proto(index); } + ONNX_NAMESPACE::AttributeProto* FunctionProto__mutable_attribute_proto(ONNX_NAMESPACE::FunctionProto* p, int index) override { return p->mutable_attribute_proto(index); } + int FunctionProto__attribute_size(const ONNX_NAMESPACE::FunctionProto* p) override { return p->attribute_proto_size(); } + ONNX_NAMESPACE::AttributeProto* FunctionProto__add_attribute_proto(ONNX_NAMESPACE::FunctionProto* p) override { return p->add_attribute_proto(); } + + const ONNX_NAMESPACE::NodeProto& FunctionProto__node(const ONNX_NAMESPACE::FunctionProto* p, int index) override { return p->node(index); } + ONNX_NAMESPACE::NodeProto* FunctionProto__mutable_node(ONNX_NAMESPACE::FunctionProto* p, int index) override { return p->mutable_node(index); } + int FunctionProto__node_size(const ONNX_NAMESPACE::FunctionProto* p) override { return p->node_size(); } + ONNX_NAMESPACE::NodeProto* FunctionProto__add_node(ONNX_NAMESPACE::FunctionProto* p) override { return p->add_node(); } + + const ONNX_NAMESPACE::ValueInfoProto& FunctionProto__value_info(const ONNX_NAMESPACE::FunctionProto* p, int index) override { return p->value_info(index); } + ONNX_NAMESPACE::ValueInfoProto* FunctionProto__mutable_value_info(ONNX_NAMESPACE::FunctionProto* p, int index) override { return p->mutable_value_info(index); } + ONNX_NAMESPACE::ValueInfoProtos* FunctionProto__mutable_value_info(ONNX_NAMESPACE::FunctionProto* p) override { return p->mutable_value_info(); } + int FunctionProto__value_info_size(const ONNX_NAMESPACE::FunctionProto* p) override { return p->value_info_size(); } + ONNX_NAMESPACE::ValueInfoProto* FunctionProto__add_value_info(ONNX_NAMESPACE::FunctionProto* p) override { return p->add_value_info(); } + + const ONNX_NAMESPACE::StringStringEntryProto& FunctionProto__metadata_props(const ONNX_NAMESPACE::FunctionProto* p, int index) override { return p->metadata_props(index); } + ONNX_NAMESPACE::StringStringEntryProto* FunctionProto__mutable_metadata_props(ONNX_NAMESPACE::FunctionProto* p, int index) override { return p->mutable_metadata_props(index); } + ONNX_NAMESPACE::StringStringEntryProtos* FunctionProto__mutable_metadata_props(ONNX_NAMESPACE::FunctionProto* p) override { return p->mutable_metadata_props(); } + int FunctionProto__metadata_props_size(const ONNX_NAMESPACE::FunctionProto* p) override { return p->metadata_props_size(); } + ONNX_NAMESPACE::StringStringEntryProto* FunctionProto__add_metadata_props(ONNX_NAMESPACE::FunctionProto* p) override { return p->add_metadata_props(); } + static void xir_shape_infer(ONNX_NAMESPACE::InferenceContext& ctx) { auto* shape = ctx.getAttribute("shape"); auto* data_type = ctx.getAttribute("data_type"); @@ -734,9 +804,12 @@ struct ProviderHostImpl : ProviderHost { std::vector& IndexedSubGraph__Nodes(IndexedSubGraph* p) override { return p->nodes; } - void IndexedSubGraph__SetMetaDef(IndexedSubGraph* p, std::unique_ptr&& meta_def_) override { return p->SetMetaDef(std::move(meta_def_)); } + void IndexedSubGraph__SetMetaDef(IndexedSubGraph* p, std::unique_ptr&& meta_def_) override { p->SetMetaDef(std::move(meta_def_)); } const IndexedSubGraph_MetaDef* IndexedSubGraph__GetMetaDef(const IndexedSubGraph* p) override { return p->GetMetaDef(); } + void IndexedSubGraph__SetSchemaSource(IndexedSubGraph* p, SourceOfSchema schema_source) override { p->schema_source = schema_source; } + SourceOfSchema IndexedSubGraph__GetSchemaSource(const IndexedSubGraph* p) override { return p->schema_source; } + // KernelDef (wrapped) void KernelDef__operator_delete(KernelDef* p) override { delete p; } void KernelDef__SinceVersion(const KernelDef* p, int* start, int* end) override { return p->SinceVersion(start, end); } @@ -2753,6 +2826,11 @@ ORT_API_STATUS_IMPL(OrtApis::SessionOptionsAppendExecutionProvider_VitisAI, _In_ provider_options[provider_options_keys[i]] = provider_options_values[i]; } + // EP context related session config options. + provider_options["ep_context_enable"] = options->value.config_options.GetConfigOrDefault(kOrtSessionOptionEpContextEnable, "0"); + provider_options["ep_context_embed_mode"] = options->value.config_options.GetConfigOrDefault(kOrtSessionOptionEpContextEmbedMode, "1"); + provider_options["ep_context_file_path"] = options->value.config_options.GetConfigOrDefault(kOrtSessionOptionEpContextFilePath, ""); + auto factory = onnxruntime::VitisAIProviderFactoryCreator::Create(provider_options); if (!factory) { return OrtApis::CreateStatus(ORT_FAIL, "SessionOptionsAppendExecutionProvider_VitisAI: Failed to load shared library"); From a94ad9bdf9f9c694eede0b6e0436af62f803f737 Mon Sep 17 00:00:00 2001 From: Glen Cao Date: Mon, 3 Jun 2024 13:54:46 -0700 Subject: [PATCH 02/11] Tried fixing a few errors --- .../core/providers/shared_library/provider_api.h | 1 + .../providers/shared_library/provider_interfaces.h | 12 ++++++------ .../core/providers/vitisai/imp/ep_context_utils.cc | 3 --- onnxruntime/core/session/provider_bridge_ort.cc | 4 ++-- 4 files changed, 9 insertions(+), 11 deletions(-) diff --git a/onnxruntime/core/providers/shared_library/provider_api.h b/onnxruntime/core/providers/shared_library/provider_api.h index 7cdfb0ffc19f2..6959b8a59efa3 100644 --- a/onnxruntime/core/providers/shared_library/provider_api.h +++ b/onnxruntime/core/providers/shared_library/provider_api.h @@ -120,6 +120,7 @@ struct TypeProto_Sequence; struct TypeProto; struct ValueInfoProto; struct ValueInfoProtos; // RepeatedPtrField +struct FunctionProto; struct InferenceContext; class GraphInferencer; using InferenceFunction = std::function; diff --git a/onnxruntime/core/providers/shared_library/provider_interfaces.h b/onnxruntime/core/providers/shared_library/provider_interfaces.h index ad63bd895b81a..ad140c03eca68 100644 --- a/onnxruntime/core/providers/shared_library/provider_interfaces.h +++ b/onnxruntime/core/providers/shared_library/provider_interfaces.h @@ -507,7 +507,7 @@ struct ProviderHost { virtual int FunctionProto__since_version(const ONNX_NAMESPACE::FunctionProto* p) const = 0; virtual void FunctionProto__set_since_version(ONNX_NAMESPACE::FunctionProto* p, int since_version) = 0; - bool FunctionProto__has_status(const ONNX_NAMESPACE::FunctionProto* p) = 0; + virtual bool FunctionProto__has_status(const ONNX_NAMESPACE::FunctionProto* p) = 0; virtual const ONNX_NAMESPACE::OperatorStatus& FunctionProto__status(const ONNX_NAMESPACE::FunctionProto* p) const = 0; virtual void FunctionProto__set_status(ONNX_NAMESPACE::FunctionProto* p, const ONNX_NAMESPACE::OperatorStatus& status) = 0; @@ -523,19 +523,19 @@ struct ProviderHost { virtual std::vector* FunctionProto__mutable_input(ONNX_NAMESPACE::FunctionProto* p) = 0; virtual std::string* FunctionProto__mutable_input(ONNX_NAMESPACE::FunctionProto* p, int index) = 0; virtual int FunctionProto__input_size(const ONNX_NAMESPACE::FunctionProto* p) = 0; - virtual void FunctionProto__add_input(ONNX_NAMESPACE::FunctionProto* p, const string& value) = 0; + virtual void FunctionProto__add_input(ONNX_NAMESPACE::FunctionProto* p, const std::string& value) = 0; virtual const std::string& FunctionProto__output(const ONNX_NAMESPACE::FunctionProto* p, int index) = 0; virtual std::vector* FunctionProto__mutable_output(ONNX_NAMESPACE::FunctionProto* p) = 0; virtual std::string* FunctionProto__mutable_output(ONNX_NAMESPACE::FunctionProto* p, int index) = 0; virtual int FunctionProto__output_size(const ONNX_NAMESPACE::FunctionProto* p) = 0; - virtual void FunctionProto__add_output(ONNX_NAMESPACE::FunctionProto* p, const string& value) = 0; + virtual void FunctionProto__add_output(ONNX_NAMESPACE::FunctionProto* p, const std::string& value) = 0; virtual const std::string& FunctionProto__attribute(const ONNX_NAMESPACE::FunctionProto* p, int index) = 0; virtual std::vector* FunctionProto__mutable_attribute(ONNX_NAMESPACE::FunctionProto* p) = 0; virtual std::string* FunctionProto__mutable_attribute(ONNX_NAMESPACE::FunctionProto* p, int index) = 0; virtual int FunctionProto__attribute_size(const ONNX_NAMESPACE::FunctionProto* p) = 0; - virtual void FunctionProto__add_attribute(ONNX_NAMESPACE::FunctionProto* p, const string& value) = 0; + virtual void FunctionProto__add_attribute(ONNX_NAMESPACE::FunctionProto* p, const std::string& value) = 0; virtual const ONNX_NAMESPACE::AttributeProto& FunctionProto__attribute_proto(const ONNX_NAMESPACE::FunctionProto* p, int index) = 0; virtual ONNX_NAMESPACE::AttributeProto* FunctionProto__mutable_attribute_proto(ONNX_NAMESPACE::FunctionProto* p, int index) = 0; @@ -610,8 +610,8 @@ struct ProviderHost { virtual void IndexedSubGraph__SetMetaDef(IndexedSubGraph* p, std::unique_ptr&& meta_def_) = 0; virtual const IndexedSubGraph_MetaDef* IndexedSubGraph__GetMetaDef(const IndexedSubGraph* p) = 0; - virtual void IndexedSubGraph__SetSchemaSource(IndexedSubGraph* p, SourceOfSchema schema_source) = 0; - virtual SourceOfSchema IndexedSubGraph__GetSchemaSource(const IndexedSubGraph* p) = 0; + virtual void IndexedSubGraph__SetSchemaSource(IndexedSubGraph* p, IndexedSubGraph::SourceOfSchema schema_source) = 0; + virtual IndexedSubGraph::SourceOfSchema IndexedSubGraph__GetSchemaSource(const IndexedSubGraph* p) = 0; // KernelDef virtual void KernelDef__operator_delete(KernelDef* p) = 0; diff --git a/onnxruntime/core/providers/vitisai/imp/ep_context_utils.cc b/onnxruntime/core/providers/vitisai/imp/ep_context_utils.cc index bac6f699fc801..109a1055d911a 100644 --- a/onnxruntime/core/providers/vitisai/imp/ep_context_utils.cc +++ b/onnxruntime/core/providers/vitisai/imp/ep_context_utils.cc @@ -1,8 +1,5 @@ #include "ep_context_utils.h" -// 1st-party headers/libs. -#include "core/graph/graph.h" - namespace fs = std::filesystem; diff --git a/onnxruntime/core/session/provider_bridge_ort.cc b/onnxruntime/core/session/provider_bridge_ort.cc index ad24e0b643f30..b544bb031e87d 100644 --- a/onnxruntime/core/session/provider_bridge_ort.cc +++ b/onnxruntime/core/session/provider_bridge_ort.cc @@ -807,8 +807,8 @@ struct ProviderHostImpl : ProviderHost { void IndexedSubGraph__SetMetaDef(IndexedSubGraph* p, std::unique_ptr&& meta_def_) override { p->SetMetaDef(std::move(meta_def_)); } const IndexedSubGraph_MetaDef* IndexedSubGraph__GetMetaDef(const IndexedSubGraph* p) override { return p->GetMetaDef(); } - void IndexedSubGraph__SetSchemaSource(IndexedSubGraph* p, SourceOfSchema schema_source) override { p->schema_source = schema_source; } - SourceOfSchema IndexedSubGraph__GetSchemaSource(const IndexedSubGraph* p) override { return p->schema_source; } + void IndexedSubGraph__SetSchemaSource(IndexedSubGraph* p, IndexedSubGraph::SourceOfSchema schema_source) override { p->schema_source = schema_source; } + IndexedSubGraph::SourceOfSchema IndexedSubGraph__GetSchemaSource(const IndexedSubGraph* p) override { return p->schema_source; } // KernelDef (wrapped) void KernelDef__operator_delete(KernelDef* p) override { delete p; } From 0c0e2963217b682f728737b63255bdb2cffad738 Mon Sep 17 00:00:00 2001 From: Glen Cao Date: Mon, 3 Jun 2024 14:30:32 -0700 Subject: [PATCH 03/11] Trying address issues of unrecognized ONNX_NAMESPACE::FunctionProto Tried addressing part of the issues with wrapped classes Fixed extra forward declaration --- include/onnxruntime/core/graph/basic_types.h | 1 + .../shared_library/provider_interfaces.h | 6 ++--- .../shared_library/provider_wrappedtypes.h | 23 +++++++++---------- .../core/session/provider_bridge_ort.cc | 6 +++-- 4 files changed, 19 insertions(+), 17 deletions(-) diff --git a/include/onnxruntime/core/graph/basic_types.h b/include/onnxruntime/core/graph/basic_types.h index 36984d0405bbd..67c7db7979f81 100644 --- a/include/onnxruntime/core/graph/basic_types.h +++ b/include/onnxruntime/core/graph/basic_types.h @@ -19,6 +19,7 @@ class TensorProto; class SparseTensorProto; class TypeProto; class AttributeProto; +class FunctionProto; // define types that would come from the ONNX library if we were building against it. #if defined(ORT_MINIMAL_BUILD) using OperatorSetVersion = int; diff --git a/onnxruntime/core/providers/shared_library/provider_interfaces.h b/onnxruntime/core/providers/shared_library/provider_interfaces.h index ad140c03eca68..6793ff610020f 100644 --- a/onnxruntime/core/providers/shared_library/provider_interfaces.h +++ b/onnxruntime/core/providers/shared_library/provider_interfaces.h @@ -540,7 +540,7 @@ struct ProviderHost { virtual const ONNX_NAMESPACE::AttributeProto& FunctionProto__attribute_proto(const ONNX_NAMESPACE::FunctionProto* p, int index) = 0; virtual ONNX_NAMESPACE::AttributeProto* FunctionProto__mutable_attribute_proto(ONNX_NAMESPACE::FunctionProto* p, int index) = 0; virtual int FunctionProto__attribute_proto_size(const ONNX_NAMESPACE::FunctionProto* p) = 0; - virtual ONNX_NAMESPACE::AttributeProto* FunctionProto__add_attribution_proto(ONNX_NAMESPACE::FunctionProto* p) = 0; + virtual ONNX_NAMESPACE::AttributeProto* FunctionProto__add_attribute_proto(ONNX_NAMESPACE::FunctionProto* p) = 0; virtual const ONNX_NAMESPACE::NodeProto& FunctionProto__node(const ONNX_NAMESPACE::FunctionProto* p, int index) = 0; virtual ONNX_NAMESPACE::NodeProto* FunctionProto__mutable_node(ONNX_NAMESPACE::FunctionProto* p, int index) = 0; @@ -610,8 +610,8 @@ struct ProviderHost { virtual void IndexedSubGraph__SetMetaDef(IndexedSubGraph* p, std::unique_ptr&& meta_def_) = 0; virtual const IndexedSubGraph_MetaDef* IndexedSubGraph__GetMetaDef(const IndexedSubGraph* p) = 0; - virtual void IndexedSubGraph__SetSchemaSource(IndexedSubGraph* p, IndexedSubGraph::SourceOfSchema schema_source) = 0; - virtual IndexedSubGraph::SourceOfSchema IndexedSubGraph__GetSchemaSource(const IndexedSubGraph* p) = 0; + virtual void IndexedSubGraph__SetSchemaSource(IndexedSubGraph* p, IndexedSubGraph_SourceOfSchema schema_source) = 0; + virtual IndexedSubGraph_SourceOfSchema IndexedSubGraph__GetSchemaSource(const IndexedSubGraph* p) = 0; // KernelDef virtual void KernelDef__operator_delete(KernelDef* p) = 0; diff --git a/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h b/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h index 831ec6235503d..083a4428add91 100644 --- a/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h +++ b/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h @@ -404,19 +404,19 @@ struct FunctionProto final { void set_domain(const std::string& domain) { g_host->FunctionProto__set_domain(this, domain); } const std::string& input(int index) const { return g_host->FunctionProto__input(this, index); } - std::vector* mutable_input() { return g_host->FunctionProto__input(this); } + std::vector* mutable_input() { return g_host->FunctionProto__mutable_input(this); } std::string* mutable_input(int index) { return g_host->FunctionProto__mutable_input(this, index); } int input_size() const { return g_host->FunctionProto__input_size(this); } void add_input(const std::string& value) { g_host->FunctionProto__add_input(this, value); } const std::string& output(int index) const { return g_host->FunctionProto__output(this, index); } - std::vector* mutable_output() { return g_host->FunctionProto__output(this); } + std::vector* mutable_output() { return g_host->FunctionProto__mutable_output(this); } std::string* mutable_output(int index) { return g_host->FunctionProto__mutable_output(this, index); } int output_size() const { return g_host->FunctionProto__output_size(this); } void add_output(const std::string& value) { g_host->FunctionProto__add_output(this, value); } const std::string& attribute(int index) const { return g_host->FunctionProto__attribute(this, index); } - std::vector* mutable_attribute() { return g_host->FunctionProto__attribute(this); } + std::vector* mutable_attribute() { return g_host->FunctionProto__mutable_attribute(this); } std::string* mutable_attribute(int index) { return g_host->FunctionProto__mutable_attribute(this, index); } int attribute_size() const { return g_host->FunctionProto__attribute_size(this); } void add_attribute(const std::string& value) { g_host->FunctionProto__add_attribute(this, value); } @@ -524,14 +524,13 @@ struct IndexedSubGraph_MetaDef final { void operator=(const IndexedSubGraph_MetaDef&) = delete; }; -struct IndexedSubGraph final { - // Mirroring the enum defined in indexed_sub_graph.h. - enum class SourceOfSchema : uint8_t { - CREATE, - REUSE_OR_CREATE, - EXISTING, - }; +enum class IndexedSubGraph_SourceOfSchema : uint8_t { + CREATE, + REUSE_OR_CREATE, + EXISTING, +}; +struct IndexedSubGraph final { static std::unique_ptr Create() { return g_host->IndexedSubGraph__construct(); } static void operator delete(void* p) { g_host->IndexedSubGraph__operator_delete(reinterpret_cast(p)); } @@ -540,8 +539,8 @@ struct IndexedSubGraph final { void SetMetaDef(std::unique_ptr&& meta_def_) { return g_host->IndexedSubGraph__SetMetaDef(this, std::move(*reinterpret_cast*>(&meta_def_))); } const IndexedSubGraph_MetaDef* GetMetaDef() const { return reinterpret_cast(g_host->IndexedSubGraph__GetMetaDef(this)); } - void SetSchemaSource(SourceOfSchema schema_source) { return g_host->IndexedSubGraph__SetSchemaSource(this, schema_source); } - SourceOfSchema GetSchemaSource() const { return g_host->IndexedSubGraph__GetSchemaSource(this); } + void SetSchemaSource(IndexedSubGraph_SourceOfSchema schema_source) { return g_host->IndexedSubGraph__SetSchemaSource(this, schema_source); } + IndexedSubGraph_SetSchemaSource GetSchemaSource() const { return g_host->IndexedSubGraph__GetSchemaSource(this); } IndexedSubGraph() = delete; IndexedSubGraph(const IndexedSubGraph&) = delete; diff --git a/onnxruntime/core/session/provider_bridge_ort.cc b/onnxruntime/core/session/provider_bridge_ort.cc index b544bb031e87d..b9ae83d72df92 100644 --- a/onnxruntime/core/session/provider_bridge_ort.cc +++ b/onnxruntime/core/session/provider_bridge_ort.cc @@ -66,10 +66,12 @@ using StringStringEntryProtos = google::protobuf::RepeatedPtrField; using TensorShapeProto_Dimensions = google::protobuf::RepeatedPtrField; using ValueInfoProtos = google::protobuf::RepeatedPtrField; +using FunctionProtos = google::protobuf::RepeatedPtrField; } // namespace ONNX_NAMESPACE namespace onnxruntime { using IndexedSubGraph_MetaDef = IndexedSubGraph::MetaDef; +using IndexedSubGraph_SourceOfSchema = IndexedSubGraph::SourceOfSchema; } // namespace onnxruntime #include "core/common/cpuid_info.h" @@ -807,8 +809,8 @@ struct ProviderHostImpl : ProviderHost { void IndexedSubGraph__SetMetaDef(IndexedSubGraph* p, std::unique_ptr&& meta_def_) override { p->SetMetaDef(std::move(meta_def_)); } const IndexedSubGraph_MetaDef* IndexedSubGraph__GetMetaDef(const IndexedSubGraph* p) override { return p->GetMetaDef(); } - void IndexedSubGraph__SetSchemaSource(IndexedSubGraph* p, IndexedSubGraph::SourceOfSchema schema_source) override { p->schema_source = schema_source; } - IndexedSubGraph::SourceOfSchema IndexedSubGraph__GetSchemaSource(const IndexedSubGraph* p) override { return p->schema_source; } + void IndexedSubGraph__SetSchemaSource(IndexedSubGraph* p, IndexedSubGraph_SourceOfSchema schema_source) override { p->schema_source = schema_source; } + IndexedSubGraph_SourceOfSchema IndexedSubGraph__GetSchemaSource(const IndexedSubGraph* p) override { return p->schema_source; } // KernelDef (wrapped) void KernelDef__operator_delete(KernelDef* p) override { delete p; } From c1a2a590e4a44fd5af495bfde441006f8d78d016 Mon Sep 17 00:00:00 2001 From: Glen Cao Date: Mon, 3 Jun 2024 15:32:44 -0700 Subject: [PATCH 04/11] Tried addressing the issues with defining IndexedSubGraph_SourceOfSchema Fixed some piecemeal issues Fixed an incorrect header inclusion Got rid of NodeAttrHelper Fixed piecemeal issues --- .../providers/shared_library/provider_api.h | 1 + .../shared_library/provider_wrappedtypes.h | 2 +- .../providers/vitisai/imp/ep_context_utils.cc | 65 ++++++++++--------- .../vitisai/include/ep_context_utils.h | 2 +- .../vitisai/vitisai_execution_provider.cc | 10 ++- 5 files changed, 46 insertions(+), 34 deletions(-) diff --git a/onnxruntime/core/providers/shared_library/provider_api.h b/onnxruntime/core/providers/shared_library/provider_api.h index 6959b8a59efa3..0bb11ec74fe32 100644 --- a/onnxruntime/core/providers/shared_library/provider_api.h +++ b/onnxruntime/core/providers/shared_library/provider_api.h @@ -147,6 +147,7 @@ struct ConfigOptions; struct DataTransferManager; struct IndexedSubGraph; struct IndexedSubGraph_MetaDef; +enum class IndexedSubGraph_SourceOfSchema : uint8_t; struct KernelCreateInfo; struct KernelDef; struct KernelDefBuilder; diff --git a/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h b/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h index 083a4428add91..5ea18e6c1753f 100644 --- a/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h +++ b/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h @@ -540,7 +540,7 @@ struct IndexedSubGraph final { const IndexedSubGraph_MetaDef* GetMetaDef() const { return reinterpret_cast(g_host->IndexedSubGraph__GetMetaDef(this)); } void SetSchemaSource(IndexedSubGraph_SourceOfSchema schema_source) { return g_host->IndexedSubGraph__SetSchemaSource(this, schema_source); } - IndexedSubGraph_SetSchemaSource GetSchemaSource() const { return g_host->IndexedSubGraph__GetSchemaSource(this); } + IndexedSubGraph_SourceOfSchema GetSchemaSource() const { return g_host->IndexedSubGraph__GetSchemaSource(this); } IndexedSubGraph() = delete; IndexedSubGraph(const IndexedSubGraph&) = delete; diff --git a/onnxruntime/core/providers/vitisai/imp/ep_context_utils.cc b/onnxruntime/core/providers/vitisai/imp/ep_context_utils.cc index 109a1055d911a..11faa442e924a 100644 --- a/onnxruntime/core/providers/vitisai/imp/ep_context_utils.cc +++ b/onnxruntime/core/providers/vitisai/imp/ep_context_utils.cc @@ -1,3 +1,8 @@ +// Standard headers/libs. +#include +#include +#include + #include "ep_context_utils.h" @@ -5,11 +10,10 @@ namespace fs = std::filesystem; namespace onnxruntime { -std::unique_ptr -VitisAIExecutionProvider::ConvertIndexedSubGraphToFunctionProto( +std::unique_ptr ConvertIndexedSubGraphToFunctionProto( const IndexedSubGraph& sub_graph, const Graph& parent_graph) { - auto p_func_proto = FunctionProto::Create(); - const auto* p_meta_def = sub_graph.GetMetaDef(); + auto p_func_proto = ONNX_NAMESPACE::FunctionProto::Create(); + auto* p_meta_def = const_cast(sub_graph.GetMetaDef()); if (p_meta_def) { p_func_proto->set_name(p_meta_def->name()); p_func_proto->set_domain(p_meta_def->domain()); @@ -38,13 +42,13 @@ VitisAIExecutionProvider::ConvertIndexedSubGraphToFunctionProto( // TODO: `MetaDef::type_and_shape_inference_function`. } auto p_parent_graph_proto = parent_graph.ToGraphProto(); - for (auto node_index : sub_graph.Nodes()) { + for (auto node_index : const_cast(sub_graph).Nodes()) { auto* p_node_proto = p_parent_graph_proto->mutable_node(node_index); auto* p_attr_proto = p_node_proto->add_attribute(); p_attr_proto->set_name("parent_graph_node_index"); p_attr_proto->set_type(ONNX_NAMESPACE::AttributeProto::INT); p_attr_proto->set_i(node_index); - *(p_func_proto.add_node()) = *p_node_proto; + *(p_func_proto->add_node()) = *p_node_proto; } #if 0 // Alternative. @@ -73,7 +77,7 @@ std::unique_ptr ConvertFunctionProtoToIndexedSubGraph( // Precisely, func_metadata_props_size == 2, which implies // `IndexedSubGraph::meta_def_` is not null and `IndexedSubGraph::nodes` > 1. if (func_metadata_props_size > 1) { - auto& prop = p_func_proto->metadata_props(0); + auto& prop = const_cast(p_func_proto->metadata_props(0)); int isg_meta_def_inputs_size = std::stoi(*(prop.mutable_value())); auto p_meta_def = IndexedSubGraph_MetaDef::Create(); p_meta_def->name() = p_func_proto->name(); @@ -89,7 +93,7 @@ std::unique_ptr ConvertFunctionProtoToIndexedSubGraph( meta_def_outputs.push_back(p_func_proto->output(i)); } auto& meta_def_initializers = p_meta_def->constant_initializers(); - for (int i = isg_metadef_inputs_size, l = p_func_proto->input_size(); i < l; i++) { + for (int i = isg_meta_def_inputs_size, l = p_func_proto->input_size(); i < l; i++) { meta_def_initializers.push_back(p_func_proto->input(i)); } auto& meta_def_attrs = p_meta_def->attributes(); @@ -101,11 +105,12 @@ std::unique_ptr ConvertFunctionProtoToIndexedSubGraph( p_isg->SetMetaDef(std::move(p_meta_def)); } auto& isg_nodes = p_isg->Nodes(); - for (int j = 0, l = p_func_proto->node_size(); j < l; j++) { - isg_nodes.push_back(p_func_proto->node(j).i()); + for (int i = 0, l = p_func_proto->node_size(); i < l; i++) { + const auto& node_proto = p_func_proto->node(i); + isg_nodes.push_back(node_proto.attribute(const_cast(node_proto).attribute_size() - 1).i()); } - auto schema_source = static_cast( - std::stoi(*(p_func_proto->metadata_props(func_metadata_props_size - 1).mutable_value()))); + auto schema_source = static_cast( + std::stoi(*(const_cast(p_func_proto->metadata_props(func_metadata_props_size - 1)).mutable_value()))); p_isg->SetSchemaSource(schema_source); return p_isg; } @@ -115,21 +120,23 @@ std::string SerializeCapabilities( const Graph& graph) { std::stringstream ss; for (const auto& p : capability_ptrs) { - auto& p_subgraph = p->Subgraph(); + auto& p_subgraph = p->SubGraph(); auto p_func_proto = ConvertIndexedSubGraphToFunctionProto(*p_subgraph, graph); std::string func_proto_buf; - p_func_proto->SerializeToString(&func_proto_buf); + p_func_proto->SerializeToString(func_proto_buf); size_t buf_len = func_proto_buf.length(); ss.write(reinterpret_cast(&buf_len), sizeof(buf_len)); ss.write(func_proto_buf.data(), buf_len); } - static_assert(ss.good(), "Serialization stream bad"); + if (!ss.good()) { + ORT_THROW("Serialization stream bad"); + } return ss.str(); } -void DeserializeCapabilities(const string& ser_capabilities, +void DeserializeCapabilities(const std::string& ser_capabilities, std::vector>& capability_ptrs) { - std::stringstream ss(ser_capabilities); + std::istringstream ss(ser_capabilities); while (!ss.eof()) { size_t buf_len; ss.read(reinterpret_cast(&buf_len), sizeof(buf_len)); @@ -162,24 +169,24 @@ std::unique_ptr CreateEPContexModel( } std::vector output_node_arg_ptrs; for (const auto* p_node_arg: graph_viewer.GetOutputs()) { - auto& temp_node_arg = ep_ctx_graph.GetOrCreateNodeArg(output->Name(), output->TypeAsProto()); + auto& temp_node_arg = ep_ctx_graph.GetOrCreateNodeArg(p_node_arg->Name(), p_node_arg->TypeAsProto()); output_node_arg_ptrs.push_back(&temp_node_arg); } // Attr "embed_mode". - auto* p_attr_0 = ONNX_NAMESPACE::AttributeProto::Create(); + auto p_attr_0 = ONNX_NAMESPACE::AttributeProto::Create(); p_attr_0->set_name(kEmbedModeAttr); //p_attr_0->set_type(onnx::AttributeProto_AttributeType_INT); p_attr_0->set_type(ONNX_NAMESPACE::AttributeProto::INT); - p_attr_0->seti(embed_mode); + p_attr_0->set_i(embed_mode); // Attr "ep_cache_context". - auto* p_attr_1 = ONNX_NAMESPACE::AttributeProto::Create(); + auto p_attr_1 = ONNX_NAMESPACE::AttributeProto::Create(); p_attr_1->set_name(kEPCacheContextAttr); //p_attr_1->set_type(onnx::AttributeProto_AttributeType_STRING); p_attr_1->set_type(ONNX_NAMESPACE::AttributeProto::STRING); p_attr_1->set_s(embed_mode == 0 ? ctx_cache_file_loc : serialized_ctx_cache); // Attr "source". - auto* p_attr_2 = ONNX_NAMESPACE::AttributeProto::Create(); + auto p_attr_2 = ONNX_NAMESPACE::AttributeProto::Create(); p_attr_2->set_name(kSourceAttr); //p_attr_2->set_type(onnx::AttributeProto_AttributeType_STRING); p_attr_2->set_type(ONNX_NAMESPACE::AttributeProto::STRING); @@ -228,9 +235,9 @@ std::string RetrieveEPContextCache(const Graph& graph) { } // TODO: Support for multi-node EP context model. auto* p_node = graph.GetNode(0); - auto& attrs = p_node->GetAttributes(); + const auto& attrs = p_node->GetAttributes(); int64_t embed_mode = attrs.at(kEmbedModeAttr).i(); - std::string& ep_ctx_cache = attrs.at(kEPCacheContextAttr).s(); + const std::string& ep_ctx_cache = attrs.at(kEPCacheContextAttr).s(); if (embed_mode) { return ep_ctx_cache; } @@ -262,8 +269,8 @@ bool GraphHasEPContextNode(const GraphViewer& graph_viewer) { for (size_t i = 0, l = static_cast(graph_viewer.MaxNodeIndex()); i < l; i++) { auto* p_node = graph_viewer.GetNode(i); if (p_node != nullptr && p_node->OpType() == kEPContextOp) { - NodeAttrHelper node_attr_helper(*p_node); - if (node_attr_helper.Get(kSourceAttr, "") == kVitisAIExecutionProvider) { + const auto& attrs = p_node->GetAttributes(); + if (attrs.count(kSourceAttr) > 0 && attrs.at(kSourceAttr).s() == kVitisAIExecutionProvider) { return true; } } @@ -299,9 +306,9 @@ bool GetEPContextModelFileLocation( //if (!ep_ctx_model_file_loc.empty()) { // return true; //} - if (!ep_ctx_model_path_cfg.emtpy()) { + if (!ep_ctx_model_path_cfg.empty()) { ep_ctx_model_file_loc = ToPathString(ep_ctx_model_path_cfg); - } else if (!model_path_str.emtpy()) { + } else if (!model_path_str.empty()) { if (is_ep_ctx_model) { ep_ctx_model_file_loc = model_path_str; } else { @@ -315,7 +322,7 @@ bool GetEPContextModelFileLocation( // The file for EP context binary is in the same folder as the EP context model file. PathString GetEPContextCacheFileLocation( const PathString& ep_ctx_model_file_loc, const PathString& model_path_str) { - if (!ep_ctx_model_file_loc.emtpy()) { + if (!ep_ctx_model_file_loc.empty()) { fs::path ep_ctx_model_fs_path(ep_ctx_model_file_loc); auto ep_ctx_cache_fs_path = ep_ctx_model_fs_path.replace_extension(fs::path("__ep_ctx_cache.bin")); diff --git a/onnxruntime/core/providers/vitisai/include/ep_context_utils.h b/onnxruntime/core/providers/vitisai/include/ep_context_utils.h index 4531354b7e693..441d21ac928e1 100644 --- a/onnxruntime/core/providers/vitisai/include/ep_context_utils.h +++ b/onnxruntime/core/providers/vitisai/include/ep_context_utils.h @@ -30,7 +30,7 @@ std::string SerializeCapabilities( const std::vector>&, const Graph&); void DeserializeCapabilities( - const string&, std::vector>&); + const std::string&, std::vector>&); std::unique_ptr CreateEPContexModel(const GraphViewer&, const std::string&, const std::string&, const int64_t, const logging::Logger*); diff --git a/onnxruntime/core/providers/vitisai/vitisai_execution_provider.cc b/onnxruntime/core/providers/vitisai/vitisai_execution_provider.cc index ffc55dbbe9e69..11e9609ea077c 100644 --- a/onnxruntime/core/providers/vitisai/vitisai_execution_provider.cc +++ b/onnxruntime/core/providers/vitisai/vitisai_execution_provider.cc @@ -63,7 +63,9 @@ void VitisAIExecutionProvider::LoadEPContexModelFromFile() { if (!p_ep_ctx_model_ && !ep_ctx_model_file_loc_.empty()) { auto p_model_proto = ONNX_NAMESPACE::ModelProto::Create(); auto status = Model::Load(ep_ctx_model_file_loc_, *p_model_proto); - static_assert(status.IsOK(), "Loading EP context model failed"); + if (!status.IsOK()) { + ORT_THROW("Loading EP context model failed from ", ep_ctx_model_file_loc_); + } auto& logger = logging::LoggingManager::DefaultLogger(); p_ep_ctx_model_ = Model::Create(std::move(*p_model_proto), ep_ctx_model_file_loc_, nullptr, logger); LOGS_DEFAULT(VERBOSE) << "Loaded EP context model from: " << ep_ctx_model_file_loc_; @@ -106,11 +108,13 @@ void VitisAIExecutionProvider::FulfillEPContextEnablement( GetEPContextModelFileLocation(ep_ctx_model_path_cfg_, model_path_str, false, ep_ctx_model_file_loc_); auto ep_ctx_cache_fs_path = GetEPContextCacheFileLocation(ep_ctx_model_file_loc_, model_path_str); std::ofstream ep_ctx_cache_ofs(ep_ctx_cache_fs_path.c_str(), std::ios::trunc | std::ios::binary); - static_assert(ep_ctx_cache_ofs.is_open(), "Failed to open a file to write EP context cache"); + if (!ep_ctx_cache_ofs.is_open()) { + ORT_THROW("Failed to open a file to write EP context cache: ", ep_ctx_cache_fs_path.c_str()); + } ep_ctx_cache_ofs.write(ep_ctx_payload.c_str(), ep_ctx_payload.length()); if (!ep_ctx_cache_ofs.good()) { ep_ctx_cache_ofs.close(); - ORT_THROW("Exception writing EP context cache file"); + ORT_THROW("Exception writing EP context cache file: ", ep_ctx_cache_fs_path.c_str()); } ep_ctx_cache_ofs.close(); p_ep_ctx_model_ = CreateEPContexModel(graph_viewer, ep_ctx_payload, ep_ctx_cache_fs_path.string(), 0, &logger); From e40ff6e84b5a48ef59c5a7d224eaa10ae3f1f06a Mon Sep 17 00:00:00 2001 From: Glen Cao Date: Mon, 3 Jun 2024 23:45:38 -0700 Subject: [PATCH 05/11] Worked around unused variable warning Commented out an explicit and direct header inclusion --- onnxruntime/core/providers/vitisai/imp/ep_context_utils.cc | 1 + onnxruntime/core/providers/vitisai/vitisai_execution_provider.h | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/onnxruntime/core/providers/vitisai/imp/ep_context_utils.cc b/onnxruntime/core/providers/vitisai/imp/ep_context_utils.cc index 11faa442e924a..10da03250c61b 100644 --- a/onnxruntime/core/providers/vitisai/imp/ep_context_utils.cc +++ b/onnxruntime/core/providers/vitisai/imp/ep_context_utils.cc @@ -226,6 +226,7 @@ bool ValidateEPContextNode(const Graph& graph) { assert(attrs.count(kEmbedModeAttr) > 0); assert(attrs.count(kEPCacheContextAttr) > 0); assert(attrs.count(kSourceAttr) > 0); + (void)attrs; return true; } diff --git a/onnxruntime/core/providers/vitisai/vitisai_execution_provider.h b/onnxruntime/core/providers/vitisai/vitisai_execution_provider.h index 15ec59c9691b0..b0c16e3283cdb 100644 --- a/onnxruntime/core/providers/vitisai/vitisai_execution_provider.h +++ b/onnxruntime/core/providers/vitisai/vitisai_execution_provider.h @@ -9,7 +9,7 @@ #include #include -#include "core/framework/session_options.h" +//#include "core/framework/session_options.h" #include "core/providers/shared_library/provider_api.h" #include "core/session/onnxruntime_c_api.h" #include "core/common/inlined_containers_fwd.h" From 4aaac631f7ecf3f717898348be633a0820a37dfe Mon Sep 17 00:00:00 2001 From: Glen Cao Date: Tue, 4 Jun 2024 10:10:53 -0700 Subject: [PATCH 06/11] const & non-const compatability of class methods --- .../providers/vitisai/vitisai_execution_provider.cc | 13 +++++++++---- .../providers/vitisai/vitisai_execution_provider.h | 9 ++++++--- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/onnxruntime/core/providers/vitisai/vitisai_execution_provider.cc b/onnxruntime/core/providers/vitisai/vitisai_execution_provider.cc index 11e9609ea077c..30781a81aa428 100644 --- a/onnxruntime/core/providers/vitisai/vitisai_execution_provider.cc +++ b/onnxruntime/core/providers/vitisai/vitisai_execution_provider.cc @@ -2,15 +2,20 @@ // Licensed under the MIT License. #include "vitisai_execution_provider.h" +// Standard headers/libs. #include #include #include #include +// 1st-party headers/libs. +#include "core/framework/session_options.h" + #include "vaip/capability.h" #include "vaip/global_api.h" #include "ep_context_utils.h" + using namespace ONNX_NAMESPACE; namespace fs = std::filesystem; @@ -58,7 +63,7 @@ VitisAIExecutionProvider::~VitisAIExecutionProvider() { } #endif -void VitisAIExecutionProvider::LoadEPContexModelFromFile() { +void VitisAIExecutionProvider::LoadEPContexModelFromFile() const { // XXX: should "p_ep_ctx_model_" be checked or not? if (!p_ep_ctx_model_ && !ep_ctx_model_file_loc_.empty()) { auto p_model_proto = ONNX_NAMESPACE::ModelProto::Create(); @@ -101,8 +106,8 @@ const InlinedVector VitisAIExecutionProvider::GetEpContextNodes() c void VitisAIExecutionProvider::FulfillEPContextEnablement( const std::vector>& capability_ptrs, const onnxruntime::GraphViewer& graph_viewer) { - auto& logger = logging::LoggingManager::DefaultLogger() - auto& model_path_str = GetTopLevelModelPath(graph_viewer).ToPathString(); + auto& logger = logging::LoggingManager::DefaultLogger(); + auto model_path_str = GetTopLevelModelPath(graph_viewer).ToPathString(); auto ep_ctx_payload = SerializeCapabilities(capability_ptrs, graph_viewer.GetGraph()); if (!ep_ctx_embed_mode_) { GetEPContextModelFileLocation(ep_ctx_model_path_cfg_, model_path_str, false, ep_ctx_model_file_loc_); @@ -135,7 +140,7 @@ std::vector> VitisAIExecutionProvider::GetCap } else { // FIXME: Will it make sense to do this? // One of the potential problems is the existing EP-context model file may be stale. - auto& model_path_str = GetTopLevelModelPath(graph_viewer).ToPathString(); + auto model_path_str = GetTopLevelModelPath(graph_viewer).ToPathString(); if (GetEPContextModelFileLocation( ep_ctx_model_path_cfg_, model_path_str, false, ep_ctx_model_file_loc_)) { LOGS_DEFAULT(WARNING) << "The inference session was created with a normal ONNX model " diff --git a/onnxruntime/core/providers/vitisai/vitisai_execution_provider.h b/onnxruntime/core/providers/vitisai/vitisai_execution_provider.h index b0c16e3283cdb..1729b62fd27ee 100644 --- a/onnxruntime/core/providers/vitisai/vitisai_execution_provider.h +++ b/onnxruntime/core/providers/vitisai/vitisai_execution_provider.h @@ -3,17 +3,20 @@ #pragma once +// Standard headers/libs. #include #include #include #include #include +// 1st-party headers/libs. //#include "core/framework/session_options.h" #include "core/providers/shared_library/provider_api.h" #include "core/session/onnxruntime_c_api.h" #include "core/common/inlined_containers_fwd.h" + // we cannot include vaip/vaip.hpp here because header file referred by // onnxruntime_pybind_state_common.cc namespace vaip_core { @@ -58,15 +61,15 @@ class VitisAIExecutionProvider : public IExecutionProvider { std::string ep_ctx_model_path_cfg_{""}; PathString ep_ctx_model_file_loc_{""}; // FIXME: This might not be needed. - std::unique_ptr p_ep_ctx_model_; + mutable std::unique_ptr p_ep_ctx_model_; // It might need to be called before loading // the EP context model that is compiled AOT/offline. - void LoadEPContexModelFromFile(); + void LoadEPContexModelFromFile() const; // Create EP context model and dump it for future use. // This implementation here is only working for non-compilation-based EPs. void FulfillEPContextEnablement( const std::vector>&, - const onnxruntime::GraphViewer&); + const onnxruntime::GraphViewer&) const; }; } // namespace onnxruntime From 4e8e0436493c818970ec9238f9af408a25c43b18 Mon Sep 17 00:00:00 2001 From: Glen Cao Date: Tue, 4 Jun 2024 10:32:04 -0700 Subject: [PATCH 07/11] Commented out SessionOptions related pieces in VAI EP ctor Dealt with const-ness of class members Fixed mismatched types due to previous changes Added missing namespace prefixes Removed incorrect methods --- .../shared_library/provider_interfaces.h | 4 +--- .../shared_library/provider_wrappedtypes.h | 2 -- .../vitisai/vitisai_execution_provider.cc | 22 ++++++++++--------- .../vitisai/vitisai_execution_provider.h | 7 +++--- .../core/session/provider_bridge_ort.cc | 11 +++++----- 5 files changed, 22 insertions(+), 24 deletions(-) diff --git a/onnxruntime/core/providers/shared_library/provider_interfaces.h b/onnxruntime/core/providers/shared_library/provider_interfaces.h index 6793ff610020f..1e5f0a5986e59 100644 --- a/onnxruntime/core/providers/shared_library/provider_interfaces.h +++ b/onnxruntime/core/providers/shared_library/provider_interfaces.h @@ -503,11 +503,9 @@ struct ProviderHost { virtual const std::string& FunctionProto__name(const ONNX_NAMESPACE::FunctionProto* p) const = 0; virtual void FunctionProto__set_name(ONNX_NAMESPACE::FunctionProto* p, const ::std::string& name) = 0; - virtual bool FunctionProto__has_since_version(const ONNX_NAMESPACE::FunctionProto* p) = 0; virtual int FunctionProto__since_version(const ONNX_NAMESPACE::FunctionProto* p) const = 0; virtual void FunctionProto__set_since_version(ONNX_NAMESPACE::FunctionProto* p, int since_version) = 0; - virtual bool FunctionProto__has_status(const ONNX_NAMESPACE::FunctionProto* p) = 0; virtual const ONNX_NAMESPACE::OperatorStatus& FunctionProto__status(const ONNX_NAMESPACE::FunctionProto* p) const = 0; virtual void FunctionProto__set_status(ONNX_NAMESPACE::FunctionProto* p, const ONNX_NAMESPACE::OperatorStatus& status) = 0; @@ -516,7 +514,7 @@ struct ProviderHost { virtual void FunctionProto__set_doc_string(ONNX_NAMESPACE::FunctionProto* p, const ::std::string& doc_string) = 0; virtual bool FunctionProto__has_domain(const ONNX_NAMESPACE::FunctionProto* p) = 0; - virtual const std::string& FunctionProto__domain(const ONNX_NAMESPACE::FunctionProto* p) = 0; + virtual const std::string& FunctionProto__domain(const ONNX_NAMESPACE::FunctionProto* p) const = 0; virtual void FunctionProto__set_domain(ONNX_NAMESPACE::FunctionProto* p, const ::std::string& domain) = 0; virtual const std::string& FunctionProto__input(const ONNX_NAMESPACE::FunctionProto* p, int index) = 0; diff --git a/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h b/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h index 5ea18e6c1753f..a15946e4d356b 100644 --- a/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h +++ b/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h @@ -387,11 +387,9 @@ struct FunctionProto final { const std::string& name() const { return g_host->FunctionProto__name(this); } void set_name(const std::string& name) { g_host->FunctionProto__set_name(this, name); } - bool has_since_version() const { return g_host->FunctionProto__has_since_version(this); } int since_version() const { return g_host->FunctionProto__since_version(this); } void set_since_version(int since_version) { g_host->FunctionProto__set_since_version(this, since_version); } - bool has_status() const { return g_host->FunctionProto__has_status(this); } const OperatorStatus& status() const { return g_host->FunctionProto__status(this); } void set_status(const OperatorStatus& status) { g_host->FunctionProto__set_status(this, status); } diff --git a/onnxruntime/core/providers/vitisai/vitisai_execution_provider.cc b/onnxruntime/core/providers/vitisai/vitisai_execution_provider.cc index 30781a81aa428..7cfd258a3b557 100644 --- a/onnxruntime/core/providers/vitisai/vitisai_execution_provider.cc +++ b/onnxruntime/core/providers/vitisai/vitisai_execution_provider.cc @@ -8,9 +8,6 @@ #include #include -// 1st-party headers/libs. -#include "core/framework/session_options.h" - #include "vaip/capability.h" #include "vaip/global_api.h" #include "ep_context_utils.h" @@ -24,10 +21,12 @@ namespace onnxruntime { constexpr const char* VITISAI = "VITISAI"; VitisAIExecutionProvider::VitisAIExecutionProvider( - const ProviderOptions& info, const SessionOptions* p_sess_opts) + const ProviderOptions& info) + //const ProviderOptions& info, const SessionOptions* p_sess_opts) : IExecutionProvider{onnxruntime::kVitisAIExecutionProvider}, info_(info) { CreateKernelRegistry(); +#if 0 if (p_sess_opts) { ep_ctx_enabled_ = p_sess_opts->config_options.GetConfigOrDefault( kOrtSessionOptionEpContextEnable, "0") == "1"; @@ -44,6 +43,7 @@ VitisAIExecutionProvider::VitisAIExecutionProvider( ep_ctx_model_path_cfg_ = p_sess_opts->config_options.GetConfigOrDefault( kOrtSessionOptionEpContextFilePath, ""); } else { +#endif auto it = info_.find("ep_context_enable"); ep_ctx_enabled_ = it != info_.end() && it->second == "1"; it = info_.find("ep_context_embed_mode"); @@ -51,7 +51,9 @@ VitisAIExecutionProvider::VitisAIExecutionProvider( //ep_ctx_embed_mode_ = it == info_.end() || it->second != "0"; it = info_.find("ep_context_file_path"); ep_ctx_model_path_cfg_ = it == info_.end() ? "" : it->second; +#if 0 } +#endif LOGS_DEFAULT(VERBOSE) << "EP Context cache enabled: " << ep_ctx_enabled_; LOGS_DEFAULT(VERBOSE) << "EP context cache embed mode: " << ep_ctx_embed_mode_; LOGS_DEFAULT(VERBOSE) << "User specified EP context cache path: " << ep_ctx_model_path_cfg_; @@ -105,24 +107,24 @@ const InlinedVector VitisAIExecutionProvider::GetEpContextNodes() c // This implementation here is only working for non-compilation-based EPs. void VitisAIExecutionProvider::FulfillEPContextEnablement( const std::vector>& capability_ptrs, - const onnxruntime::GraphViewer& graph_viewer) { + const onnxruntime::GraphViewer& graph_viewer) const { auto& logger = logging::LoggingManager::DefaultLogger(); auto model_path_str = GetTopLevelModelPath(graph_viewer).ToPathString(); auto ep_ctx_payload = SerializeCapabilities(capability_ptrs, graph_viewer.GetGraph()); if (!ep_ctx_embed_mode_) { GetEPContextModelFileLocation(ep_ctx_model_path_cfg_, model_path_str, false, ep_ctx_model_file_loc_); - auto ep_ctx_cache_fs_path = GetEPContextCacheFileLocation(ep_ctx_model_file_loc_, model_path_str); - std::ofstream ep_ctx_cache_ofs(ep_ctx_cache_fs_path.c_str(), std::ios::trunc | std::ios::binary); + auto ep_ctx_cache_path_str = GetEPContextCacheFileLocation(ep_ctx_model_file_loc_, model_path_str); + std::ofstream ep_ctx_cache_ofs(ep_ctx_cache_path_str.c_str(), std::ios::trunc | std::ios::binary); if (!ep_ctx_cache_ofs.is_open()) { - ORT_THROW("Failed to open a file to write EP context cache: ", ep_ctx_cache_fs_path.c_str()); + ORT_THROW("Failed to open a file to write EP context cache: ", ep_ctx_cache_path_str.c_str()); } ep_ctx_cache_ofs.write(ep_ctx_payload.c_str(), ep_ctx_payload.length()); if (!ep_ctx_cache_ofs.good()) { ep_ctx_cache_ofs.close(); - ORT_THROW("Exception writing EP context cache file: ", ep_ctx_cache_fs_path.c_str()); + ORT_THROW("Exception writing EP context cache file: ", ep_ctx_cache_path_str.c_str()); } ep_ctx_cache_ofs.close(); - p_ep_ctx_model_ = CreateEPContexModel(graph_viewer, ep_ctx_payload, ep_ctx_cache_fs_path.string(), 0, &logger); + p_ep_ctx_model_ = CreateEPContexModel(graph_viewer, ep_ctx_payload, ep_ctx_cache_path_str, 0, &logger); } else { p_ep_ctx_model_ = CreateEPContexModel(graph_viewer, ep_ctx_payload, "", 1, &logger); } diff --git a/onnxruntime/core/providers/vitisai/vitisai_execution_provider.h b/onnxruntime/core/providers/vitisai/vitisai_execution_provider.h index 1729b62fd27ee..c2391be9eee75 100644 --- a/onnxruntime/core/providers/vitisai/vitisai_execution_provider.h +++ b/onnxruntime/core/providers/vitisai/vitisai_execution_provider.h @@ -28,8 +28,9 @@ namespace onnxruntime { // Logical device representation. class VitisAIExecutionProvider : public IExecutionProvider { public: - explicit VitisAIExecutionProvider(const ProviderOptions& info, - const SessionOptions* p_sess_opts = nullptr); + explicit VitisAIExecutionProvider(const ProviderOptions& info); + //explicit VitisAIExecutionProvider(const ProviderOptions& info, + // const SessionOptions* p_sess_opts = nullptr); ~VitisAIExecutionProvider() = default; std::vector> GetCapability(const onnxruntime::GraphViewer& graph_viewer, @@ -59,7 +60,7 @@ class VitisAIExecutionProvider : public IExecutionProvider { bool ep_ctx_enabled_ = false; bool ep_ctx_embed_mode_ = true; std::string ep_ctx_model_path_cfg_{""}; - PathString ep_ctx_model_file_loc_{""}; + mutable PathString ep_ctx_model_file_loc_{""}; // FIXME: This might not be needed. mutable std::unique_ptr p_ep_ctx_model_; // It might need to be called before loading diff --git a/onnxruntime/core/session/provider_bridge_ort.cc b/onnxruntime/core/session/provider_bridge_ort.cc index b9ae83d72df92..00bf1d9c5e939 100644 --- a/onnxruntime/core/session/provider_bridge_ort.cc +++ b/onnxruntime/core/session/provider_bridge_ort.cc @@ -27,6 +27,7 @@ #include "core/session/inference_session.h" #include "core/session/abi_session_options_impl.h" #include "core/session/ort_apis.h" +#include "core/session/onnxruntime_session_options_config_keys.h" #include "core/session/provider_bridge_ort.h" #include "core/util/math.h" #include "core/framework/sparse_utils.h" @@ -616,11 +617,9 @@ struct ProviderHostImpl : ProviderHost { const std::string& FunctionProto__name(const ONNX_NAMESPACE::FunctionProto* p) const override { return p->name(); } void FunctionProto__set_name(ONNX_NAMESPACE::FunctionProto* p, const std::string& name) override { p->set_name(name); } - bool FunctionProto__has_since_version(const ONNX_NAMESPACE::FunctionProto* p) override { return p->has_since_version(); } int FunctionProto__since_version(const ONNX_NAMESPACE::FunctionProto* p) const override { return p->since_version(); } void FunctionProto__set_since_version(ONNX_NAMESPACE::FunctionProto* p, int since_version) override { p->set_since_version(since_version); } - bool FunctionProto__has_status(const ONNX_NAMESPACE::FunctionProto* p) override { return p->has_status(); } const ONNX_NAMESPACE::OperatorStatus& FunctionProto__status(const ONNX_NAMESPACE::FunctionProto* p) const override { return p->status(); } void FunctionProto__set_status(ONNX_NAMESPACE::FunctionProto* p, const ONNX_NAMESPACE::OperatorStatus& status) override { p->set_status(status); } @@ -633,26 +632,26 @@ struct ProviderHostImpl : ProviderHost { void FunctionProto__set_domain(ONNX_NAMESPACE::FunctionProto* p, const std::string& domain) override { p->set_domain(domain); } const std::string& FunctionProto__input(const ONNX_NAMESPACE::FunctionProto* p, int index) override { return p->input(index); } - std::vector* FunctionProto__mutable_input(ONNX_NAMESPACE::FunctionProto* p) override { return p->mutable_input(); } + std::vector* FunctionProto__mutable_input(ONNX_NAMESPACE::FunctionProto* p) override { return p->mutable_input(); } std::string* FunctionProto__mutable_input(ONNX_NAMESPACE::FunctionProto* p, int index) override { return p->mutable_input(index); } int FunctionProto__input_size(const ONNX_NAMESPACE::FunctionProto* p) override { return p->input_size(); } void FunctionProto__add_input(ONNX_NAMESPACE::FunctionProto* p, const std::string& value) override { p->add_input(value); } const std::string& FunctionProto__output(const ONNX_NAMESPACE::FunctionProto* p, int index) override { return p->output(index); } - std::vector* FunctionProto__mutable_output(ONNX_NAMESPACE::FunctionProto* p) override { return p->mutable_output(); } + std::vector* FunctionProto__mutable_output(ONNX_NAMESPACE::FunctionProto* p) override { return p->mutable_output(); } std::string* FunctionProto__mutable_output(ONNX_NAMESPACE::FunctionProto* p, int index) override { return p->mutable_output(index); } int FunctionProto__output_size(const ONNX_NAMESPACE::FunctionProto* p) override { return p->output_size(); } void FunctionProto__add_output(ONNX_NAMESPACE::FunctionProto* p, const std::string& value) override { p->add_output(value); } const std::string& FunctionProto__attribute(const ONNX_NAMESPACE::FunctionProto* p, int index) override { return p->attribute(index); } - std::vector* FunctionProto__mutable_attribute(ONNX_NAMESPACE::FunctionProto* p) override { return p->mutable_attribute(); } + std::vector* FunctionProto__mutable_attribute(ONNX_NAMESPACE::FunctionProto* p) override { return p->mutable_attribute(); } std::string* FunctionProto__mutable_attribute(ONNX_NAMESPACE::FunctionProto* p, int index) override { return p->mutable_attribute(index); } int FunctionProto__attribute_size(const ONNX_NAMESPACE::FunctionProto* p) override { return p->attribute_size(); } void FunctionProto__add_attribute(ONNX_NAMESPACE::FunctionProto* p, const std::string& value) override { p->add_attribute(value); } const ONNX_NAMESPACE::AttributeProto& FunctionProto__attribute_proto(const ONNX_NAMESPACE::FunctionProto* p, int index) override { return p->attribute_proto(index); } ONNX_NAMESPACE::AttributeProto* FunctionProto__mutable_attribute_proto(ONNX_NAMESPACE::FunctionProto* p, int index) override { return p->mutable_attribute_proto(index); } - int FunctionProto__attribute_size(const ONNX_NAMESPACE::FunctionProto* p) override { return p->attribute_proto_size(); } + int FunctionProto__attribute_proto_size(const ONNX_NAMESPACE::FunctionProto* p) override { return p->attribute_proto_size(); } ONNX_NAMESPACE::AttributeProto* FunctionProto__add_attribute_proto(ONNX_NAMESPACE::FunctionProto* p) override { return p->add_attribute_proto(); } const ONNX_NAMESPACE::NodeProto& FunctionProto__node(const ONNX_NAMESPACE::FunctionProto* p, int index) override { return p->node(index); } From ebc02905e87f057e187c0b7c9e8d11a5bb86cdfb Mon Sep 17 00:00:00 2001 From: Glen Cao Date: Tue, 4 Jun 2024 12:46:46 -0700 Subject: [PATCH 08/11] Removed or commented out deprecated fields of FunctionProto --- .../core/providers/shared_library/provider_interfaces.h | 9 --------- .../providers/shared_library/provider_wrappedtypes.h | 9 --------- .../core/providers/vitisai/imp/ep_context_utils.cc | 8 ++++---- onnxruntime/core/session/provider_bridge_ort.cc | 9 --------- 4 files changed, 4 insertions(+), 31 deletions(-) diff --git a/onnxruntime/core/providers/shared_library/provider_interfaces.h b/onnxruntime/core/providers/shared_library/provider_interfaces.h index 1e5f0a5986e59..de741f752abae 100644 --- a/onnxruntime/core/providers/shared_library/provider_interfaces.h +++ b/onnxruntime/core/providers/shared_library/provider_interfaces.h @@ -503,12 +503,6 @@ struct ProviderHost { virtual const std::string& FunctionProto__name(const ONNX_NAMESPACE::FunctionProto* p) const = 0; virtual void FunctionProto__set_name(ONNX_NAMESPACE::FunctionProto* p, const ::std::string& name) = 0; - virtual int FunctionProto__since_version(const ONNX_NAMESPACE::FunctionProto* p) const = 0; - virtual void FunctionProto__set_since_version(ONNX_NAMESPACE::FunctionProto* p, int since_version) = 0; - - virtual const ONNX_NAMESPACE::OperatorStatus& FunctionProto__status(const ONNX_NAMESPACE::FunctionProto* p) const = 0; - virtual void FunctionProto__set_status(ONNX_NAMESPACE::FunctionProto* p, const ONNX_NAMESPACE::OperatorStatus& status) = 0; - virtual bool FunctionProto__has_doc_string(const ONNX_NAMESPACE::FunctionProto* p) = 0; virtual const std::string& FunctionProto__doc_string(const ONNX_NAMESPACE::FunctionProto* p) const = 0; virtual void FunctionProto__set_doc_string(ONNX_NAMESPACE::FunctionProto* p, const ::std::string& doc_string) = 0; @@ -518,19 +512,16 @@ struct ProviderHost { virtual void FunctionProto__set_domain(ONNX_NAMESPACE::FunctionProto* p, const ::std::string& domain) = 0; virtual const std::string& FunctionProto__input(const ONNX_NAMESPACE::FunctionProto* p, int index) = 0; - virtual std::vector* FunctionProto__mutable_input(ONNX_NAMESPACE::FunctionProto* p) = 0; virtual std::string* FunctionProto__mutable_input(ONNX_NAMESPACE::FunctionProto* p, int index) = 0; virtual int FunctionProto__input_size(const ONNX_NAMESPACE::FunctionProto* p) = 0; virtual void FunctionProto__add_input(ONNX_NAMESPACE::FunctionProto* p, const std::string& value) = 0; virtual const std::string& FunctionProto__output(const ONNX_NAMESPACE::FunctionProto* p, int index) = 0; - virtual std::vector* FunctionProto__mutable_output(ONNX_NAMESPACE::FunctionProto* p) = 0; virtual std::string* FunctionProto__mutable_output(ONNX_NAMESPACE::FunctionProto* p, int index) = 0; virtual int FunctionProto__output_size(const ONNX_NAMESPACE::FunctionProto* p) = 0; virtual void FunctionProto__add_output(ONNX_NAMESPACE::FunctionProto* p, const std::string& value) = 0; virtual const std::string& FunctionProto__attribute(const ONNX_NAMESPACE::FunctionProto* p, int index) = 0; - virtual std::vector* FunctionProto__mutable_attribute(ONNX_NAMESPACE::FunctionProto* p) = 0; virtual std::string* FunctionProto__mutable_attribute(ONNX_NAMESPACE::FunctionProto* p, int index) = 0; virtual int FunctionProto__attribute_size(const ONNX_NAMESPACE::FunctionProto* p) = 0; virtual void FunctionProto__add_attribute(ONNX_NAMESPACE::FunctionProto* p, const std::string& value) = 0; diff --git a/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h b/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h index a15946e4d356b..5569ac0dcba6f 100644 --- a/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h +++ b/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h @@ -387,12 +387,6 @@ struct FunctionProto final { const std::string& name() const { return g_host->FunctionProto__name(this); } void set_name(const std::string& name) { g_host->FunctionProto__set_name(this, name); } - int since_version() const { return g_host->FunctionProto__since_version(this); } - void set_since_version(int since_version) { g_host->FunctionProto__set_since_version(this, since_version); } - - const OperatorStatus& status() const { return g_host->FunctionProto__status(this); } - void set_status(const OperatorStatus& status) { g_host->FunctionProto__set_status(this, status); } - bool has_doc_string() const { return g_host->FunctionProto__has_doc_string(this); } const std::string& doc_string() const { return g_host->FunctionProto__doc_string(this); } void set_doc_string(const std::string& doc_string) { g_host->FunctionProto__set_doc_string(this, doc_string); } @@ -402,19 +396,16 @@ struct FunctionProto final { void set_domain(const std::string& domain) { g_host->FunctionProto__set_domain(this, domain); } const std::string& input(int index) const { return g_host->FunctionProto__input(this, index); } - std::vector* mutable_input() { return g_host->FunctionProto__mutable_input(this); } std::string* mutable_input(int index) { return g_host->FunctionProto__mutable_input(this, index); } int input_size() const { return g_host->FunctionProto__input_size(this); } void add_input(const std::string& value) { g_host->FunctionProto__add_input(this, value); } const std::string& output(int index) const { return g_host->FunctionProto__output(this, index); } - std::vector* mutable_output() { return g_host->FunctionProto__mutable_output(this); } std::string* mutable_output(int index) { return g_host->FunctionProto__mutable_output(this, index); } int output_size() const { return g_host->FunctionProto__output_size(this); } void add_output(const std::string& value) { g_host->FunctionProto__add_output(this, value); } const std::string& attribute(int index) const { return g_host->FunctionProto__attribute(this, index); } - std::vector* mutable_attribute() { return g_host->FunctionProto__mutable_attribute(this); } std::string* mutable_attribute(int index) { return g_host->FunctionProto__mutable_attribute(this, index); } int attribute_size() const { return g_host->FunctionProto__attribute_size(this); } void add_attribute(const std::string& value) { g_host->FunctionProto__add_attribute(this, value); } diff --git a/onnxruntime/core/providers/vitisai/imp/ep_context_utils.cc b/onnxruntime/core/providers/vitisai/imp/ep_context_utils.cc index 10da03250c61b..27f37ddbca923 100644 --- a/onnxruntime/core/providers/vitisai/imp/ep_context_utils.cc +++ b/onnxruntime/core/providers/vitisai/imp/ep_context_utils.cc @@ -17,8 +17,8 @@ std::unique_ptr ConvertIndexedSubGraphToFunctionP if (p_meta_def) { p_func_proto->set_name(p_meta_def->name()); p_func_proto->set_domain(p_meta_def->domain()); - p_func_proto->set_since_version(p_meta_def->since_version()); - p_func_proto->set_status(p_meta_def->status()); + //p_func_proto->set_since_version(p_meta_def->since_version()); + //p_func_proto->set_status(p_meta_def->status()); for (const auto& input : p_meta_def->inputs()) { p_func_proto->add_input(input); } @@ -82,8 +82,8 @@ std::unique_ptr ConvertFunctionProtoToIndexedSubGraph( auto p_meta_def = IndexedSubGraph_MetaDef::Create(); p_meta_def->name() = p_func_proto->name(); p_meta_def->domain() = p_func_proto->domain(); - p_meta_def->since_version() = p_func_proto->since_version(); - p_meta_def->status() = p_func_proto->status(); + //p_meta_def->since_version() = p_func_proto->since_version(); + //p_meta_def->status() = p_func_proto->status(); auto& meta_def_inputs = p_meta_def->inputs(); for (int i = 0; i < isg_meta_def_inputs_size; i++) { meta_def_inputs.push_back(p_func_proto->input(i)); diff --git a/onnxruntime/core/session/provider_bridge_ort.cc b/onnxruntime/core/session/provider_bridge_ort.cc index 00bf1d9c5e939..68cab2d86518d 100644 --- a/onnxruntime/core/session/provider_bridge_ort.cc +++ b/onnxruntime/core/session/provider_bridge_ort.cc @@ -617,12 +617,6 @@ struct ProviderHostImpl : ProviderHost { const std::string& FunctionProto__name(const ONNX_NAMESPACE::FunctionProto* p) const override { return p->name(); } void FunctionProto__set_name(ONNX_NAMESPACE::FunctionProto* p, const std::string& name) override { p->set_name(name); } - int FunctionProto__since_version(const ONNX_NAMESPACE::FunctionProto* p) const override { return p->since_version(); } - void FunctionProto__set_since_version(ONNX_NAMESPACE::FunctionProto* p, int since_version) override { p->set_since_version(since_version); } - - const ONNX_NAMESPACE::OperatorStatus& FunctionProto__status(const ONNX_NAMESPACE::FunctionProto* p) const override { return p->status(); } - void FunctionProto__set_status(ONNX_NAMESPACE::FunctionProto* p, const ONNX_NAMESPACE::OperatorStatus& status) override { p->set_status(status); } - bool FunctionProto__has_doc_string(const ONNX_NAMESPACE::FunctionProto* p) override { return p->has_doc_string(); } const std::string& FunctionProto__doc_string(const ONNX_NAMESPACE::FunctionProto* p) const override { return p->doc_string(); } void FunctionProto__set_doc_string(ONNX_NAMESPACE::FunctionProto* p, const std::string& doc_string) override { p->set_doc_string(doc_string); } @@ -632,19 +626,16 @@ struct ProviderHostImpl : ProviderHost { void FunctionProto__set_domain(ONNX_NAMESPACE::FunctionProto* p, const std::string& domain) override { p->set_domain(domain); } const std::string& FunctionProto__input(const ONNX_NAMESPACE::FunctionProto* p, int index) override { return p->input(index); } - std::vector* FunctionProto__mutable_input(ONNX_NAMESPACE::FunctionProto* p) override { return p->mutable_input(); } std::string* FunctionProto__mutable_input(ONNX_NAMESPACE::FunctionProto* p, int index) override { return p->mutable_input(index); } int FunctionProto__input_size(const ONNX_NAMESPACE::FunctionProto* p) override { return p->input_size(); } void FunctionProto__add_input(ONNX_NAMESPACE::FunctionProto* p, const std::string& value) override { p->add_input(value); } const std::string& FunctionProto__output(const ONNX_NAMESPACE::FunctionProto* p, int index) override { return p->output(index); } - std::vector* FunctionProto__mutable_output(ONNX_NAMESPACE::FunctionProto* p) override { return p->mutable_output(); } std::string* FunctionProto__mutable_output(ONNX_NAMESPACE::FunctionProto* p, int index) override { return p->mutable_output(index); } int FunctionProto__output_size(const ONNX_NAMESPACE::FunctionProto* p) override { return p->output_size(); } void FunctionProto__add_output(ONNX_NAMESPACE::FunctionProto* p, const std::string& value) override { p->add_output(value); } const std::string& FunctionProto__attribute(const ONNX_NAMESPACE::FunctionProto* p, int index) override { return p->attribute(index); } - std::vector* FunctionProto__mutable_attribute(ONNX_NAMESPACE::FunctionProto* p) override { return p->mutable_attribute(); } std::string* FunctionProto__mutable_attribute(ONNX_NAMESPACE::FunctionProto* p, int index) override { return p->mutable_attribute(index); } int FunctionProto__attribute_size(const ONNX_NAMESPACE::FunctionProto* p) override { return p->attribute_size(); } void FunctionProto__add_attribute(ONNX_NAMESPACE::FunctionProto* p, const std::string& value) override { p->add_attribute(value); } From 3b0b2b037748f226e5f27df7bea6c444bc8cc96b Mon Sep 17 00:00:00 2001 From: Glen Cao Date: Tue, 4 Jun 2024 14:51:20 -0700 Subject: [PATCH 09/11] Compitible with the deprecated FunctionProto fields --- .../providers/vitisai/imp/ep_context_utils.cc | 28 ++++++++++++------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/onnxruntime/core/providers/vitisai/imp/ep_context_utils.cc b/onnxruntime/core/providers/vitisai/imp/ep_context_utils.cc index 27f37ddbca923..7b8ce337f3ac6 100644 --- a/onnxruntime/core/providers/vitisai/imp/ep_context_utils.cc +++ b/onnxruntime/core/providers/vitisai/imp/ep_context_utils.cc @@ -17,8 +17,6 @@ std::unique_ptr ConvertIndexedSubGraphToFunctionP if (p_meta_def) { p_func_proto->set_name(p_meta_def->name()); p_func_proto->set_domain(p_meta_def->domain()); - //p_func_proto->set_since_version(p_meta_def->since_version()); - //p_func_proto->set_status(p_meta_def->status()); for (const auto& input : p_meta_def->inputs()) { p_func_proto->add_input(input); } @@ -39,6 +37,14 @@ std::unique_ptr ConvertIndexedSubGraphToFunctionP *p_attr_proto = attr_pair.second; } p_func_proto->set_doc_string(p_meta_def->doc_string()); + // "since_version" + auto* p_metadata_props_1 = p_func_proto->add_metadata_props(); + *(p_metadata_props_1->mutable_key()) = "meta_def_since_version"; + *(p_metadata_props_1->mutable_value()) = std::to_string(p_meta_def->since_version()); + // "status" + auto* p_metadata_props_2 = p_func_proto->add_metadata_props(); + *(p_metadata_props_2->mutable_key()) = "meta_def_status"; + *(p_metadata_props_2->mutable_value()) = std::to_string(static_cast(p_meta_def->status())); // TODO: `MetaDef::type_and_shape_inference_function`. } auto p_parent_graph_proto = parent_graph.ToGraphProto(); @@ -63,9 +69,9 @@ std::unique_ptr ConvertIndexedSubGraphToFunctionP *(p_func_proto.add_node()) = *p_node_proto; } #endif - auto* p_metadata_props_1 = p_func_proto->add_metadata_props(); - *p_metadata_props_1->mutable_key() = "schema_source"; - *p_metadata_props_1->mutable_value() = std::to_string(static_cast(sub_graph.GetSchemaSource())); + auto* p_metadata_props_3 = p_func_proto->add_metadata_props(); + *p_metadata_props_3->mutable_key() = "schema_source"; + *p_metadata_props_3->mutable_value() = std::to_string(static_cast(sub_graph.GetSchemaSource())); return p_func_proto; } @@ -74,16 +80,18 @@ std::unique_ptr ConvertFunctionProtoToIndexedSubGraph( auto p_isg = IndexedSubGraph::Create(); // "meta_def_inputs_size" (optional) and "schema_source". int func_metadata_props_size = p_func_proto->metadata_props_size(); - // Precisely, func_metadata_props_size == 2, which implies + // Precisely, func_metadata_props_size == 4, which implies // `IndexedSubGraph::meta_def_` is not null and `IndexedSubGraph::nodes` > 1. if (func_metadata_props_size > 1) { - auto& prop = const_cast(p_func_proto->metadata_props(0)); - int isg_meta_def_inputs_size = std::stoi(*(prop.mutable_value())); + auto& prop0 = const_cast(p_func_proto->metadata_props(0)); + int isg_meta_def_inputs_size = std::stoi(*(prop0.mutable_value())); auto p_meta_def = IndexedSubGraph_MetaDef::Create(); p_meta_def->name() = p_func_proto->name(); p_meta_def->domain() = p_func_proto->domain(); - //p_meta_def->since_version() = p_func_proto->since_version(); - //p_meta_def->status() = p_func_proto->status(); + auto& prop1 = const_cast(p_func_proto->metadata_props(1)); + p_meta_def->since_version() = std::stoi(*(prop1.mutable_value())); + auto& prop2 = const_cast(p_func_proto->metadata_props(2)); + p_meta_def->status() = static_cast(std::stoi(*(prop2.mutable_value()))); auto& meta_def_inputs = p_meta_def->inputs(); for (int i = 0; i < isg_meta_def_inputs_size; i++) { meta_def_inputs.push_back(p_func_proto->input(i)); From d86a3eb2d2165d357ba6306624b6e8fb3413d8ab Mon Sep 17 00:00:00 2001 From: Glen Cao Date: Tue, 4 Jun 2024 15:37:20 -0700 Subject: [PATCH 10/11] lintrunner -a --- .../providers/vitisai/imp/ep_context_utils.cc | 23 +++++++++---------- .../vitisai/include/ep_context_utils.h | 3 +-- .../vitisai/vitisai_execution_provider.cc | 23 +++++++++---------- .../vitisai/vitisai_execution_provider.h | 5 ++-- .../core/session/provider_bridge_ort.cc | 2 +- 5 files changed, 26 insertions(+), 30 deletions(-) diff --git a/onnxruntime/core/providers/vitisai/imp/ep_context_utils.cc b/onnxruntime/core/providers/vitisai/imp/ep_context_utils.cc index 7b8ce337f3ac6..2094967cd34f8 100644 --- a/onnxruntime/core/providers/vitisai/imp/ep_context_utils.cc +++ b/onnxruntime/core/providers/vitisai/imp/ep_context_utils.cc @@ -5,7 +5,6 @@ #include "ep_context_utils.h" - namespace fs = std::filesystem; namespace onnxruntime { @@ -143,7 +142,7 @@ std::string SerializeCapabilities( } void DeserializeCapabilities(const std::string& ser_capabilities, - std::vector>& capability_ptrs) { + std::vector>& capability_ptrs) { std::istringstream ss(ser_capabilities); while (!ss.eof()) { size_t buf_len; @@ -176,7 +175,7 @@ std::unique_ptr CreateEPContexModel( input_node_arg_ptrs.push_back(&temp_node_arg); } std::vector output_node_arg_ptrs; - for (const auto* p_node_arg: graph_viewer.GetOutputs()) { + for (const auto* p_node_arg : graph_viewer.GetOutputs()) { auto& temp_node_arg = ep_ctx_graph.GetOrCreateNodeArg(p_node_arg->Name(), p_node_arg->TypeAsProto()); output_node_arg_ptrs.push_back(&temp_node_arg); } @@ -184,19 +183,19 @@ std::unique_ptr CreateEPContexModel( // Attr "embed_mode". auto p_attr_0 = ONNX_NAMESPACE::AttributeProto::Create(); p_attr_0->set_name(kEmbedModeAttr); - //p_attr_0->set_type(onnx::AttributeProto_AttributeType_INT); + // p_attr_0->set_type(onnx::AttributeProto_AttributeType_INT); p_attr_0->set_type(ONNX_NAMESPACE::AttributeProto::INT); p_attr_0->set_i(embed_mode); // Attr "ep_cache_context". auto p_attr_1 = ONNX_NAMESPACE::AttributeProto::Create(); p_attr_1->set_name(kEPCacheContextAttr); - //p_attr_1->set_type(onnx::AttributeProto_AttributeType_STRING); + // p_attr_1->set_type(onnx::AttributeProto_AttributeType_STRING); p_attr_1->set_type(ONNX_NAMESPACE::AttributeProto::STRING); p_attr_1->set_s(embed_mode == 0 ? ctx_cache_file_loc : serialized_ctx_cache); // Attr "source". auto p_attr_2 = ONNX_NAMESPACE::AttributeProto::Create(); p_attr_2->set_name(kSourceAttr); - //p_attr_2->set_type(onnx::AttributeProto_AttributeType_STRING); + // p_attr_2->set_type(onnx::AttributeProto_AttributeType_STRING); p_attr_2->set_type(ONNX_NAMESPACE::AttributeProto::STRING); p_attr_2->set_s(kVitisAIExecutionProvider); @@ -312,9 +311,9 @@ bool GetEPContextModelFileLocation( const PathString& model_path_str, bool is_ep_ctx_model, PathString& ep_ctx_model_file_loc) { - //if (!ep_ctx_model_file_loc.empty()) { - // return true; - //} + // if (!ep_ctx_model_file_loc.empty()) { + // return true; + // } if (!ep_ctx_model_path_cfg.empty()) { ep_ctx_model_file_loc = ToPathString(ep_ctx_model_path_cfg); } else if (!model_path_str.empty()) { @@ -322,7 +321,7 @@ bool GetEPContextModelFileLocation( ep_ctx_model_file_loc = model_path_str; } else { ep_ctx_model_file_loc = - ToPathString(fs::path(model_path_str).stem().string() + "_ctx.onnx"); + ToPathString(fs::path(model_path_str).stem().string() + "_ctx.onnx"); } } return !ep_ctx_model_file_loc.empty() && fs::exists(ep_ctx_model_file_loc) && fs::is_regular_file(ep_ctx_model_file_loc); @@ -334,12 +333,12 @@ PathString GetEPContextCacheFileLocation( if (!ep_ctx_model_file_loc.empty()) { fs::path ep_ctx_model_fs_path(ep_ctx_model_file_loc); auto ep_ctx_cache_fs_path = - ep_ctx_model_fs_path.replace_extension(fs::path("__ep_ctx_cache.bin")); + ep_ctx_model_fs_path.replace_extension(fs::path("__ep_ctx_cache.bin")); return ToPathString(ep_ctx_cache_fs_path.string()); } fs::path model_fs_path(model_path_str); auto ep_ctx_cache_fs_path = - model_fs_path.replace_extension(fs::path("__ep_ctx_cache.bin")); + model_fs_path.replace_extension(fs::path("__ep_ctx_cache.bin")); return ToPathString(ep_ctx_cache_fs_path.string()); } diff --git a/onnxruntime/core/providers/vitisai/include/ep_context_utils.h b/onnxruntime/core/providers/vitisai/include/ep_context_utils.h index 441d21ac928e1..1e3bf8e53ba3f 100644 --- a/onnxruntime/core/providers/vitisai/include/ep_context_utils.h +++ b/onnxruntime/core/providers/vitisai/include/ep_context_utils.h @@ -8,7 +8,6 @@ // 1st-party headers/libs. #include "core/providers/shared_library/provider_api.h" - namespace onnxruntime { static constexpr const char* kEPContextOp = "EPContext"; @@ -33,7 +32,7 @@ void DeserializeCapabilities( const std::string&, std::vector>&); std::unique_ptr CreateEPContexModel(const GraphViewer&, const std::string&, - const std::string&, const int64_t, const logging::Logger*); + const std::string&, const int64_t, const logging::Logger*); void DumpEPContextModel(std::unique_ptr&, const std::string&); diff --git a/onnxruntime/core/providers/vitisai/vitisai_execution_provider.cc b/onnxruntime/core/providers/vitisai/vitisai_execution_provider.cc index 7cfd258a3b557..26036bb34d13c 100644 --- a/onnxruntime/core/providers/vitisai/vitisai_execution_provider.cc +++ b/onnxruntime/core/providers/vitisai/vitisai_execution_provider.cc @@ -12,7 +12,6 @@ #include "vaip/global_api.h" #include "ep_context_utils.h" - using namespace ONNX_NAMESPACE; namespace fs = std::filesystem; @@ -22,8 +21,8 @@ constexpr const char* VITISAI = "VITISAI"; VitisAIExecutionProvider::VitisAIExecutionProvider( const ProviderOptions& info) - //const ProviderOptions& info, const SessionOptions* p_sess_opts) - : IExecutionProvider{onnxruntime::kVitisAIExecutionProvider}, info_(info) { + // const ProviderOptions& info, const SessionOptions* p_sess_opts) + : IExecutionProvider{onnxruntime::kVitisAIExecutionProvider}, info_(info) { CreateKernelRegistry(); #if 0 @@ -44,13 +43,13 @@ VitisAIExecutionProvider::VitisAIExecutionProvider( kOrtSessionOptionEpContextFilePath, ""); } else { #endif - auto it = info_.find("ep_context_enable"); - ep_ctx_enabled_ = it != info_.end() && it->second == "1"; - it = info_.find("ep_context_embed_mode"); - ep_ctx_embed_mode_ = it != info_.end() && it->second != "0"; - //ep_ctx_embed_mode_ = it == info_.end() || it->second != "0"; - it = info_.find("ep_context_file_path"); - ep_ctx_model_path_cfg_ = it == info_.end() ? "" : it->second; + auto it = info_.find("ep_context_enable"); + ep_ctx_enabled_ = it != info_.end() && it->second == "1"; + it = info_.find("ep_context_embed_mode"); + ep_ctx_embed_mode_ = it != info_.end() && it->second != "0"; + // ep_ctx_embed_mode_ = it == info_.end() || it->second != "0"; + it = info_.find("ep_context_file_path"); + ep_ctx_model_path_cfg_ = it == info_.end() ? "" : it->second; #if 0 } #endif @@ -144,9 +143,9 @@ std::vector> VitisAIExecutionProvider::GetCap // One of the potential problems is the existing EP-context model file may be stale. auto model_path_str = GetTopLevelModelPath(graph_viewer).ToPathString(); if (GetEPContextModelFileLocation( - ep_ctx_model_path_cfg_, model_path_str, false, ep_ctx_model_file_loc_)) { + ep_ctx_model_path_cfg_, model_path_str, false, ep_ctx_model_file_loc_)) { LOGS_DEFAULT(WARNING) << "The inference session was created with a normal ONNX model " - << "but a model file with EP context cache exists at " << ep_ctx_model_file_loc_.c_str(); + << "but a model file with EP context cache exists at " << ep_ctx_model_file_loc_.c_str(); LoadEPContexModelFromFile(); auto ep_ctx_payload = RetrieveEPContextCache(p_ep_ctx_model_->MainGraph()); std::vector> capability_ptrs; diff --git a/onnxruntime/core/providers/vitisai/vitisai_execution_provider.h b/onnxruntime/core/providers/vitisai/vitisai_execution_provider.h index c2391be9eee75..e2b6996d7887c 100644 --- a/onnxruntime/core/providers/vitisai/vitisai_execution_provider.h +++ b/onnxruntime/core/providers/vitisai/vitisai_execution_provider.h @@ -16,7 +16,6 @@ #include "core/session/onnxruntime_c_api.h" #include "core/common/inlined_containers_fwd.h" - // we cannot include vaip/vaip.hpp here because header file referred by // onnxruntime_pybind_state_common.cc namespace vaip_core { @@ -29,8 +28,8 @@ namespace onnxruntime { class VitisAIExecutionProvider : public IExecutionProvider { public: explicit VitisAIExecutionProvider(const ProviderOptions& info); - //explicit VitisAIExecutionProvider(const ProviderOptions& info, - // const SessionOptions* p_sess_opts = nullptr); + // explicit VitisAIExecutionProvider(const ProviderOptions& info, + // const SessionOptions* p_sess_opts = nullptr); ~VitisAIExecutionProvider() = default; std::vector> GetCapability(const onnxruntime::GraphViewer& graph_viewer, diff --git a/onnxruntime/core/session/provider_bridge_ort.cc b/onnxruntime/core/session/provider_bridge_ort.cc index 68cab2d86518d..6a8d8aad5b2df 100644 --- a/onnxruntime/core/session/provider_bridge_ort.cc +++ b/onnxruntime/core/session/provider_bridge_ort.cc @@ -411,7 +411,7 @@ struct ProviderHostImpl : ProviderHost { int32_t TypeProto_Tensor__elem_type(const ONNX_NAMESPACE::TypeProto_Tensor* p) override { return p->elem_type(); } void TypeProto_Tensor__set_elem_type(ONNX_NAMESPACE::TypeProto_Tensor* p, int32_t value) override { p->set_elem_type(value); }; - // TypeProto_SparseTensor (wrapped) + // TypeProto_SparseTensor (wrapped) #if !defined(DISABLE_SPARSE_TENSORS) bool TypeProto_SparseTensor__has_shape(const ONNX_NAMESPACE::TypeProto_SparseTensor* p) override { return p->has_shape(); } const ONNX_NAMESPACE::TensorShapeProto& TypeProto_SparseTensor__shape(const ONNX_NAMESPACE::TypeProto_SparseTensor* p) override { From 98d7208bfe8b697d3ac25960b3f355b2ce2a86fe Mon Sep 17 00:00:00 2001 From: Glen Cao Date: Tue, 4 Jun 2024 16:05:10 -0700 Subject: [PATCH 11/11] lintrunner -a; merge upstream main --- onnxruntime/core/providers/vitisai/vitisai_execution_provider.h | 2 +- onnxruntime/core/session/provider_bridge_ort.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/onnxruntime/core/providers/vitisai/vitisai_execution_provider.h b/onnxruntime/core/providers/vitisai/vitisai_execution_provider.h index e2b6996d7887c..2c852b9853a36 100644 --- a/onnxruntime/core/providers/vitisai/vitisai_execution_provider.h +++ b/onnxruntime/core/providers/vitisai/vitisai_execution_provider.h @@ -11,7 +11,7 @@ #include // 1st-party headers/libs. -//#include "core/framework/session_options.h" +// #include "core/framework/session_options.h" #include "core/providers/shared_library/provider_api.h" #include "core/session/onnxruntime_c_api.h" #include "core/common/inlined_containers_fwd.h" diff --git a/onnxruntime/core/session/provider_bridge_ort.cc b/onnxruntime/core/session/provider_bridge_ort.cc index 6a8d8aad5b2df..68cab2d86518d 100644 --- a/onnxruntime/core/session/provider_bridge_ort.cc +++ b/onnxruntime/core/session/provider_bridge_ort.cc @@ -411,7 +411,7 @@ struct ProviderHostImpl : ProviderHost { int32_t TypeProto_Tensor__elem_type(const ONNX_NAMESPACE::TypeProto_Tensor* p) override { return p->elem_type(); } void TypeProto_Tensor__set_elem_type(ONNX_NAMESPACE::TypeProto_Tensor* p, int32_t value) override { p->set_elem_type(value); }; - // TypeProto_SparseTensor (wrapped) + // TypeProto_SparseTensor (wrapped) #if !defined(DISABLE_SPARSE_TENSORS) bool TypeProto_SparseTensor__has_shape(const ONNX_NAMESPACE::TypeProto_SparseTensor* p) override { return p->has_shape(); } const ONNX_NAMESPACE::TensorShapeProto& TypeProto_SparseTensor__shape(const ONNX_NAMESPACE::TypeProto_SparseTensor* p) override {