Make changes to fix nightly tests issue. (#2227)

Signed-off-by: Jay Zhang <[email protected]>

Author: fatcat-z

ci_build/azure_pipelines/keras2onnx_application_tests.yml

@@ -25,7 +25,7 @@ jobs:
         python.version: '3.9'
         ONNX_PATH: onnx==1.14.1
         INSTALL_KERAS:
-        UNINSTALL_KERAS: pip uninstall keras -y
+        UNINSTALL_KERAS:
         INSTALL_TENSORFLOW: pip install tensorflow==2.12.0
         INSTALL_ORT: pip install onnxruntime==1.15.0
         INSTALL_KERAS_RESNET: pip install keras-resnet
@@ -97,9 +97,9 @@ jobs:
         python.version: '3.9'
         ONNX_PATH: onnx==1.14.1
         INSTALL_KERAS:
-        UNINSTALL_KERAS: pip uninstall keras -y
+        UNINSTALL_KERAS:
         INSTALL_TENSORFLOW: pip install tensorflow==2.12.0
-        INSTALL_ORT: pip install onnxruntime==1.15.0
+        INSTALL_ORT: pip install onnxruntime==1.15.1
         INSTALL_KERAS_RESNET: pip install keras-resnet
         INSTALL_TRANSFORMERS: pip install transformers==3.4.0
         INSTALL_NUMPY: pip install numpy==1.19.0
@@ -111,7 +111,7 @@ jobs:
         INSTALL_KERAS:
         UNINSTALL_KERAS:
         INSTALL_TENSORFLOW: pip install tensorflow==2.11.0
-        INSTALL_ORT: pip install onnxruntime==1.15.0
+        INSTALL_ORT: pip install onnxruntime==1.15.1
         INSTALL_KERAS_RESNET: pip install keras-resnet
         INSTALL_TRANSFORMERS: pip install transformers==3.4.0
         INSTALL_NUMPY:
@@ -123,7 +123,7 @@ jobs:
         INSTALL_KERAS:
         UNINSTALL_KERAS:
         INSTALL_TENSORFLOW: pip install tensorflow==2.10.0
-        INSTALL_ORT: pip install onnxruntime==1.15.0
+        INSTALL_ORT: pip install onnxruntime==1.15.1
         INSTALL_KERAS_RESNET: pip install keras-resnet
         INSTALL_TRANSFORMERS: pip install transformers==4.12.0
         INSTALL_NUMPY:

ci_build/azure_pipelines/templates/setup.yml

@@ -14,20 +14,6 @@ steps:
     else
         pip install $(CI_PIP_ONNX_BACKEND_NAME) numpy==1.23.4
     fi
-    # TF 1.10 requires numpy <=1.14.5 and >=1.13.3, but onnxruntime 0.2.1 does not work with numpy <= 1.14.5
-    # Upgrade numpy only within constraints from other packages if any.
-    if [[ $CI_TF_VERSION == 1.10* ]] && [[ $CI_ONNX_BACKEND == "onnxruntime" ]] ;
-    then
-        pip install $(CI_PIP_ONNX_NAME) $(CI_PIP_ONNX_BACKEND_NAME) numpy --no-deps -U
-    fi
-
-    if [[ $CI_ONNXRUNTIME_NIGHTLY == "true" ]] ;
-    then
-      pip uninstall -y onnxruntime
-      # install numpy upfront since it is not on https://test.pypi.org/simple/
-      pip install 'numpy>=1.18'
-      pip install -i https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/pypi/simple/ ort-nightly
-    fi
 
     if [[ $CI_SKIP_TFJS_TESTS == "False" ]] ;
     then
@@ -37,12 +23,7 @@ steps:
 
     if [[ $CI_TF_VERSION == 2.* ]] ;
     then
-      # onnxruntime-extensions is not supported Python 3.10 so far.
-      # https://github.com/microsoft/onnxruntime-extensions/issues/273
-      if [[ $CI_PYTHON_VERSION != 3.10 ]] ;
-      then
-        pip install onnxruntime-extensions==0.3.1
-      fi
+      pip install onnxruntime-extensions==0.3.1
       if [[ $CI_TF_VERSION == 2.3* ]] ;
       then
         pip install tensorflow-text==${CI_TF_VERSION}
@@ -84,6 +65,14 @@ steps:
       fi
     fi
 
+    if [[ $CI_ONNXRUNTIME_NIGHTLY == "true" ]] ;
+    then
+      pip uninstall -y onnxruntime
+      # install numpy upfront since it is not on https://test.pypi.org/simple/
+      pip install 'numpy>=1.18'
+      pip install -i https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/pypi/simple/ ort-nightly
+    fi
+
     pip uninstall -y protobuf
     pip install "protobuf~=3.20"
 

ci_build/azure_pipelines/unit_test-matrix.yml

@@ -60,3 +60,11 @@ stages:
         report_coverage: 'True'
 
   - template: 'templates/combine_test_coverage.yml'
+
+schedules:
+- cron: "0 10 * * *"
+  displayName: Daily nightly Unit Test, full matrix.
+  branches:
+    include:
+    - main
+  always: true

tests/test_einsum_helper.py

@@ -25,9 +25,10 @@ def assert_raise(self, fct, exc_type):
         raise AssertionError("%r was not raised." % exc_type)
 
     def apply_einsum_sequence(self, seq, *inputs):
+        providers = ['CPUExecutionProvider']
         names = ["X%d" % i for i in range(len(inputs))]
         onx = seq.to_onnx('Y', *names, opset=self.config.opset)
-        sess = InferenceSession(onx.SerializeToString())
+        sess = InferenceSession(onx.SerializeToString(), providers=providers)
         inps = {n: i.astype(np.float32) for n, i in zip(names, inputs)}
         res = sess.run(None, inps)
         return res[0]