Release 1.10.0 Cherrypick Round 2 (#9928)

* Misc InstanceNorm CUDA kernel changes (#9879) (cherry picked from commit c161813) * Fix Training Packaging pipeline (#9885) * Fix Training Packaging pipeline (cherry picked from commit 6de79d8) * Update doc for operators/opsets supported by mobile package (#9899) (cherry picked from commit 00c979d) * QDQ tool modification part3 (#9904) * refine per channel quantization for qdq * remove old option * add comment * add import itertools (cherry picked from commit 02aa16e) * Update to include the Xamarin targets for internal ORT builds so the managed nuget package is consistent as both CPU and GPU builds produce a package called Microsoft.ML.OnnxRuntime.Managed. (#9906) (cherry picked from commit 3f5c1e1) Co-authored-by: Hariharan Seshadri <shariharan91@gmail.com> Co-authored-by: Sherlock <baihan.huang@gmail.com> Co-authored-by: Scott McKay <skottmckay@gmail.com> Co-authored-by: Chi Lo <54722500+chilo-ms@users.noreply.github.com>
microsoft · Dec 3, 2021 · 1e31b2c · 1e31b2c
1 parent 31a4742
commit 1e31b2c
Show file tree

Hide file tree

Showing 23 changed files with 472 additions and 167 deletions.
diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/Microsoft.ML.OnnxRuntime.csproj b/csharp/src/Microsoft.ML.OnnxRuntime/Microsoft.ML.OnnxRuntime.csproj
@@ -4,10 +4,10 @@
     <OrtPackageId Condition="'$(OrtPackageId)' == ''">Microsoft.ML.OnnxRuntime</OrtPackageId>
   </PropertyGroup>
 
-  <!-- only include the Xamarin mobile targets for the main ORT package, 
+  <!-- only include the Xamarin mobile targets if we're building an ORT package, 
        and only if the mobile workloads are installed -->
   <Choose>
-    <When Condition="'$(OrtPackageId)' == 'Microsoft.ML.OnnxRuntime' AND Exists('$(MSBuildExtensionsPath)\Xamarin\Android') AND Exists('$(MSBuildExtensionsPath)\Xamarin\iOS')">
+    <When Condition="('$(OrtPackageId)' == 'Microsoft.ML.OnnxRuntime' OR '$(OrtPackageId)' == 'Microsoft.ML.OnnxRuntime.Gpu') AND Exists('$(MSBuildExtensionsPath)\Xamarin\Android') AND Exists('$(MSBuildExtensionsPath)\Xamarin\iOS')">
       <PropertyGroup>
         <TargetFrameworks>netstandard1.1;netstandard2.0;xamarinios10;monoandroid11.0;net5.0;netcoreapp3.1</TargetFrameworks>
       </PropertyGroup>      

diff --git a/docs/ORTMobilePackageOperatorTypeSupport.md b/docs/ORTMobilePackageOperatorTypeSupport.md
@@ -17,104 +17,109 @@ NOTE: Operators used to manipulate dimensions and indices will support int32 and
 |Operator|Opsets|
 |--------|------|
 |**ai.onnx**||
-|ai.onnx:Abs|12, 13|
-|ai.onnx:Add|12, 13|
-|ai.onnx:And|12, 13|
-|ai.onnx:ArgMax|12, 13|
-|ai.onnx:ArgMin|12, 13|
-|ai.onnx:AveragePool|12, 13|
-|ai.onnx:Cast|12, 13|
-|ai.onnx:Ceil|12, 13|
-|ai.onnx:Clip|12, 13|
-|ai.onnx:Concat|12, 13|
-|ai.onnx:ConstantOfShape|12, 13|
-|ai.onnx:Conv|12, 13|
-|ai.onnx:ConvTranspose|12, 13|
-|ai.onnx:Cos|12, 13|
-|ai.onnx:CumSum|12, 13|
-|ai.onnx:DepthToSpace|12, 13|
-|ai.onnx:DequantizeLinear|12, 13|
-|ai.onnx:Div|12, 13|
-|ai.onnx:DynamicQuantizeLinear|12, 13|
-|ai.onnx:Elu|12, 13|
-|ai.onnx:Equal|12, 13|
-|ai.onnx:Exp|12, 13|
-|ai.onnx:Expand|12, 13|
-|ai.onnx:Flatten|12, 13|
-|ai.onnx:Floor|12, 13|
-|ai.onnx:Gather|12, 13|
-|ai.onnx:GatherND|12, 13|
-|ai.onnx:Gemm|12, 13|
-|ai.onnx:GlobalAveragePool|12, 13|
-|ai.onnx:Greater|12, 13|
-|ai.onnx:GreaterOrEqual|12, 13|
-|ai.onnx:Identity|12, 13|
-|ai.onnx:If|12, 13|
-|ai.onnx:LRN|12, 13|
-|ai.onnx:LeakyRelu|12, 13|
-|ai.onnx:Less|12, 13|
-|ai.onnx:LessOrEqual|12, 13|
-|ai.onnx:Log|12, 13|
-|ai.onnx:LogSoftmax|12, 13|
-|ai.onnx:Loop|12, 13|
-|ai.onnx:MatMul|12, 13|
-|ai.onnx:MatMulInteger|12, 13|
-|ai.onnx:Max|12, 13|
-|ai.onnx:MaxPool|12, 13|
-|ai.onnx:Mean|12, 13|
-|ai.onnx:Min|12, 13|
-|ai.onnx:Mul|12, 13|
-|ai.onnx:Neg|12, 13|
-|ai.onnx:NonMaxSuppression|12, 13|
-|ai.onnx:NonZero|12, 13|
-|ai.onnx:Not|12, 13|
-|ai.onnx:Or|12, 13|
-|ai.onnx:PRelu|12, 13|
-|ai.onnx:Pad|12, 13|
-|ai.onnx:Pow|12, 13|
-|ai.onnx:QLinearConv|12, 13|
-|ai.onnx:QLinearMatMul|12, 13|
-|ai.onnx:QuantizeLinear|12, 13|
-|ai.onnx:Range|12, 13|
-|ai.onnx:Reciprocal|12, 13|
-|ai.onnx:ReduceMax|12, 13|
-|ai.onnx:ReduceMean|12, 13|
-|ai.onnx:ReduceMin|12, 13|
-|ai.onnx:ReduceProd|12, 13|
-|ai.onnx:ReduceSum|12, 13|
-|ai.onnx:Relu|12, 13|
-|ai.onnx:Reshape|12, 13|
-|ai.onnx:Resize|12, 13|
-|ai.onnx:ReverseSequence|12, 13|
-|ai.onnx:Round|12, 13|
-|ai.onnx:Scan|12, 13|
-|ai.onnx:ScatterND|12, 13|
-|ai.onnx:Shape|12, 13|
-|ai.onnx:Sigmoid|12, 13|
-|ai.onnx:Sin|12, 13|
-|ai.onnx:Size|12, 13|
-|ai.onnx:Slice|12, 13|
-|ai.onnx:Softmax|12, 13|
-|ai.onnx:SpaceToDepth|12, 13|
-|ai.onnx:Split|12, 13|
-|ai.onnx:Sqrt|12, 13|
-|ai.onnx:Squeeze|12, 13|
-|ai.onnx:Sub|12, 13|
-|ai.onnx:Sum|12, 13|
-|ai.onnx:Tanh|12, 13|
-|ai.onnx:ThresholdedRelu|12, 13|
-|ai.onnx:Tile|12, 13|
-|ai.onnx:TopK|12, 13|
-|ai.onnx:Transpose|12, 13|
-|ai.onnx:Unique|12, 13|
-|ai.onnx:Unsqueeze|12, 13|
-|ai.onnx:Where|12, 13|
+|ai.onnx:Abs|12, 13, 14, 15|
+|ai.onnx:Add|12, 13, 14, 15|
+|ai.onnx:And|12, 13, 14, 15|
+|ai.onnx:ArgMax|12, 13, 14, 15|
+|ai.onnx:ArgMin|12, 13, 14, 15|
+|ai.onnx:AveragePool|12, 13, 14, 15|
+|ai.onnx:Cast|12, 13, 14, 15|
+|ai.onnx:Ceil|12, 13, 14, 15|
+|ai.onnx:Clip|12, 13, 14, 15|
+|ai.onnx:Concat|12, 13, 14, 15|
+|ai.onnx:ConstantOfShape|12, 13, 14, 15|
+|ai.onnx:Conv|12, 13, 14, 15|
+|ai.onnx:ConvTranspose|12, 13, 14, 15|
+|ai.onnx:Cos|12, 13, 14, 15|
+|ai.onnx:CumSum|12, 13, 14, 15|
+|ai.onnx:DepthToSpace|12, 13, 14, 15|
+|ai.onnx:DequantizeLinear|12, 13, 14, 15|
+|ai.onnx:Div|12, 13, 14, 15|
+|ai.onnx:DynamicQuantizeLinear|12, 13, 14, 15|
+|ai.onnx:Elu|12, 13, 14, 15|
+|ai.onnx:Equal|12, 13, 14, 15|
+|ai.onnx:Erf|12, 13, 14, 15|
+|ai.onnx:Exp|12, 13, 14, 15|
+|ai.onnx:Expand|12, 13, 14, 15|
+|ai.onnx:Flatten|12, 13, 14, 15|
+|ai.onnx:Floor|12, 13, 14, 15|
+|ai.onnx:Gather|12, 13, 14, 15|
+|ai.onnx:GatherND|12, 13, 14, 15|
+|ai.onnx:Gemm|12, 13, 14, 15|
+|ai.onnx:GlobalAveragePool|12, 13, 14, 15|
+|ai.onnx:Greater|12, 13, 14, 15|
+|ai.onnx:GreaterOrEqual|12, 13, 14, 15|
+|ai.onnx:HardSigmoid|12, 13, 14, 15|
+|ai.onnx:Identity|12, 13, 14, 15|
+|ai.onnx:If|12, 13, 14, 15|
+|ai.onnx:InstanceNormalization|12, 13, 14, 15|
+|ai.onnx:LRN|12, 13, 14, 15|
+|ai.onnx:LayerNormalization|1|
+|ai.onnx:LeakyRelu|12, 13, 14, 15|
+|ai.onnx:Less|12, 13, 14, 15|
+|ai.onnx:LessOrEqual|12, 13, 14, 15|
+|ai.onnx:Log|12, 13, 14, 15|
+|ai.onnx:LogSoftmax|12, 13, 14, 15|
+|ai.onnx:Loop|12, 13, 14, 15|
+|ai.onnx:MatMul|12, 13, 14, 15|
+|ai.onnx:MatMulInteger|12, 13, 14, 15|
+|ai.onnx:Max|12, 13, 14, 15|
+|ai.onnx:MaxPool|12, 13, 14, 15|
+|ai.onnx:Mean|12, 13, 14, 15|
+|ai.onnx:Min|12, 13, 14, 15|
+|ai.onnx:Mul|12, 13, 14, 15|
+|ai.onnx:Neg|12, 13, 14, 15|
+|ai.onnx:NonMaxSuppression|12, 13, 14, 15|
+|ai.onnx:NonZero|12, 13, 14, 15|
+|ai.onnx:Not|12, 13, 14, 15|
+|ai.onnx:Or|12, 13, 14, 15|
+|ai.onnx:PRelu|12, 13, 14, 15|
+|ai.onnx:Pad|12, 13, 14, 15|
+|ai.onnx:Pow|12, 13, 14, 15|
+|ai.onnx:QLinearConv|12, 13, 14, 15|
+|ai.onnx:QLinearMatMul|12, 13, 14, 15|
+|ai.onnx:QuantizeLinear|12, 13, 14, 15|
+|ai.onnx:Range|12, 13, 14, 15|
+|ai.onnx:Reciprocal|12, 13, 14, 15|
+|ai.onnx:ReduceMax|12, 13, 14, 15|
+|ai.onnx:ReduceMean|12, 13, 14, 15|
+|ai.onnx:ReduceMin|12, 13, 14, 15|
+|ai.onnx:ReduceProd|12, 13, 14, 15|
+|ai.onnx:ReduceSum|12, 13, 14, 15|
+|ai.onnx:Relu|12, 13, 14, 15|
+|ai.onnx:Reshape|12, 13, 14, 15|
+|ai.onnx:Resize|12, 13, 14, 15|
+|ai.onnx:ReverseSequence|12, 13, 14, 15|
+|ai.onnx:Round|12, 13, 14, 15|
+|ai.onnx:Scan|12, 13, 14, 15|
+|ai.onnx:ScatterND|12, 13, 14, 15|
+|ai.onnx:Shape|12, 13, 14, 15|
+|ai.onnx:Sigmoid|12, 13, 14, 15|
+|ai.onnx:Sin|12, 13, 14, 15|
+|ai.onnx:Size|12, 13, 14, 15|
+|ai.onnx:Slice|12, 13, 14, 15|
+|ai.onnx:Softmax|12, 13, 14, 15|
+|ai.onnx:SpaceToDepth|12, 13, 14, 15|
+|ai.onnx:Split|12, 13, 14, 15|
+|ai.onnx:Sqrt|12, 13, 14, 15|
+|ai.onnx:Squeeze|12, 13, 14, 15|
+|ai.onnx:Sub|12, 13, 14, 15|
+|ai.onnx:Sum|12, 13, 14, 15|
+|ai.onnx:Tanh|12, 13, 14, 15|
+|ai.onnx:ThresholdedRelu|12, 13, 14, 15|
+|ai.onnx:Tile|12, 13, 14, 15|
+|ai.onnx:TopK|12, 13, 14, 15|
+|ai.onnx:Transpose|12, 13, 14, 15|
+|ai.onnx:Unique|12, 13, 14, 15|
+|ai.onnx:Unsqueeze|12, 13, 14, 15|
+|ai.onnx:Where|12, 13, 14, 15|
 |||
 |**com.microsoft**||
 |com.microsoft:DynamicQuantizeMatMul|1|
 |com.microsoft:FusedConv|1|
 |com.microsoft:FusedGemm|1|
 |com.microsoft:FusedMatMul|1|
+|com.microsoft:Gelu|1|
 |com.microsoft:MatMulIntegerToFloat|1|
 |com.microsoft:NhwcMaxPool|1|
 |com.microsoft:QLinearAdd|1|

diff --git a/docs/python/inference/examples/plot_common_errors.py b/docs/python/inference/examples/plot_common_errors.py
@@ -21,7 +21,7 @@
 from onnxruntime.datasets import get_example
 
 example2 = get_example("logreg_iris.onnx")
-sess = rt.InferenceSession(example2)
+sess = rt.InferenceSession(example2, providers=rt.get_available_providers())
 
 input_name = sess.get_inputs()[0].name
 output_name = sess.get_outputs()[0].name

diff --git a/docs/python/inference/examples/plot_convert_pipeline_vectorizer.py b/docs/python/inference/examples/plot_convert_pipeline_vectorizer.py
@@ -72,7 +72,7 @@
 import onnxruntime as rt
 from onnxruntime.capi.onnxruntime_pybind11_state import InvalidArgument
 
-sess = rt.InferenceSession("pipeline_vectorize.onnx")
+sess = rt.InferenceSession("pipeline_vectorize.onnx", providers=rt.get_available_providers())
 
 import numpy
 inp, out = sess.get_inputs()[0], sess.get_outputs()[0]

diff --git a/docs/python/inference/examples/plot_load_and_predict.py b/docs/python/inference/examples/plot_load_and_predict.py
@@ -21,7 +21,7 @@
 # The model is available on github `onnx...test_sigmoid <https://github.com/onnx/onnx/tree/master/onnx/backend/test/data/node/test_sigmoid>`_.
 
 example1 = get_example("sigmoid.onnx")
-sess = rt.InferenceSession(example1)
+sess = rt.InferenceSession(example1, providers=rt.get_available_providers())
 
 #########################
 # Let's see the input name and shape.

diff --git a/docs/python/inference/examples/plot_metadata.py b/docs/python/inference/examples/plot_metadata.py
@@ -31,8 +31,8 @@
 #############################
 # With *ONNX Runtime*:
 
-from onnxruntime import InferenceSession
-sess = InferenceSession(example)
+import onnxruntime as rt
+sess = rt.InferenceSession(example, providers=rt.get_available_providers())
 meta = sess.get_modelmeta()
 
 print("custom_metadata_map={}".format(meta.custom_metadata_map))

diff --git a/docs/python/inference/examples/plot_profiling.py b/docs/python/inference/examples/plot_profiling.py
@@ -35,7 +35,7 @@ def change_ir_version(filename, ir_version=6):
 example1 = get_example("mul_1.onnx")
 onnx_model = change_ir_version(example1)
 onnx_model_str = onnx_model.SerializeToString()
-sess = rt.InferenceSession(onnx_model_str)
+sess = rt.InferenceSession(onnx_model_str, providers=rt.get_available_providers())
 input_name = sess.get_inputs()[0].name
 
 x = numpy.array([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], dtype=numpy.float32)
@@ -48,7 +48,7 @@ def change_ir_version(filename, ir_version=6):
 
 options = rt.SessionOptions()
 options.enable_profiling = True
-sess_profile = rt.InferenceSession(onnx_model_str, options)
+sess_profile = rt.InferenceSession(onnx_model_str, options, providers=rt.get_available_providers())
 input_name = sess.get_inputs()[0].name
 
 x = numpy.array([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], dtype=numpy.float32)

diff --git a/docs/python/inference/examples/plot_train_convert_predict.py b/docs/python/inference/examples/plot_train_convert_predict.py
@@ -64,7 +64,7 @@
 # its input and output.
 
 import onnxruntime as rt
-sess = rt.InferenceSession("logreg_iris.onnx")
+sess = rt.InferenceSession("logreg_iris.onnx", providers=rt.get_available_providers())
 
 print("input name='{}' and shape={}".format(
     sess.get_inputs()[0].name, sess.get_inputs()[0].shape))
@@ -180,7 +180,7 @@ def sess_predict_proba(x):
 ###################################
 # We compare.
 
-sess = rt.InferenceSession("rf_iris.onnx")
+sess = rt.InferenceSession("rf_iris.onnx", providers=rt.get_available_providers())
 
 def sess_predict_proba_rf(x):
     return sess.run([prob_name], {input_name: x.astype(numpy.float32)})[0]
@@ -204,7 +204,7 @@ def sess_predict_proba_rf(x):
     onx = convert_sklearn(rf, initial_types=initial_type)
     with open("rf_iris_%d.onnx" % n_trees, "wb") as f:
         f.write(onx.SerializeToString())
-    sess = rt.InferenceSession("rf_iris_%d.onnx" % n_trees)
+    sess = rt.InferenceSession("rf_iris_%d.onnx" % n_trees, providers=rt.get_available_providers())
     def sess_predict_proba_loop(x):
         return sess.run([prob_name], {input_name: x.astype(numpy.float32)})[0]
     tsk = speed("loop(X_test, rf.predict_proba, 100)", number=5, repeat=5)

diff --git a/docs/python/inference/tutorial.rst b/docs/python/inference/tutorial.rst
@@ -82,7 +82,7 @@ for this machine learning model.
     import numpy
     import onnxruntime as rt
 
-    sess = rt.InferenceSession("logreg_iris.onnx")
+    sess = rt.InferenceSession("logreg_iris.onnx", providers=rt.get_available_providers())
     input_name = sess.get_inputs()[0].name
     pred_onx = sess.run(None, {input_name: X_test.astype(numpy.float32)})[0]
     print(pred_onx)
@@ -97,7 +97,7 @@ by specifying its name into a list.
     import numpy
     import onnxruntime as rt
 
-    sess = rt.InferenceSession("logreg_iris.onnx")
+    sess = rt.InferenceSession("logreg_iris.onnx", providers=rt.get_available_providers())
     input_name = sess.get_inputs()[0].name
     label_name = sess.get_outputs()[0].name
     pred_onx = sess.run([label_name], {input_name: X_test.astype(numpy.float32)})[0]