From 2cd7cd3c500673dd9856990f7767deb350f2da00 Mon Sep 17 00:00:00 2001 From: Xiaoyu Date: Wed, 8 Jan 2025 20:26:29 +0000 Subject: [PATCH] address comments --- ...late.yaml => olive-test-cpu-template.yaml} | 1 - .../olive-test-linux-gpu-template.yaml | 4 - .../olive-test-win-template.yaml | 95 ------------------- .azure_pipelines/olive-ci.yaml | 10 +- .azure_pipelines/olive-examples.yaml | 8 ++ .azure_pipelines/olive-ort-nightly.yaml | 5 +- examples/test/local/test_qnn_tooklit.py | 1 - test/requirements-test.txt | 4 +- .../unit_test/passes/onnx/test_split_model.py | 2 - 9 files changed, 19 insertions(+), 111 deletions(-) rename .azure_pipelines/job_templates/{olive-test-linux-cpu-template.yaml => olive-test-cpu-template.yaml} (99%) delete mode 100644 .azure_pipelines/job_templates/olive-test-win-template.yaml diff --git a/.azure_pipelines/job_templates/olive-test-linux-cpu-template.yaml b/.azure_pipelines/job_templates/olive-test-cpu-template.yaml similarity index 99% rename from .azure_pipelines/job_templates/olive-test-linux-cpu-template.yaml rename to .azure_pipelines/job_templates/olive-test-cpu-template.yaml index 998ae245c..0095fac17 100644 --- a/.azure_pipelines/job_templates/olive-test-linux-cpu-template.yaml +++ b/.azure_pipelines/job_templates/olive-test-cpu-template.yaml @@ -3,7 +3,6 @@ parameters: pool: '' test_type: '' windows: False - device: 'cpu' python_version: '3.10' onnxruntime: 'onnxruntime' torch: 'torch' diff --git a/.azure_pipelines/job_templates/olive-test-linux-gpu-template.yaml b/.azure_pipelines/job_templates/olive-test-linux-gpu-template.yaml index 2d5603e5a..7d66de592 100644 --- a/.azure_pipelines/job_templates/olive-test-linux-gpu-template.yaml +++ b/.azure_pipelines/job_templates/olive-test-linux-gpu-template.yaml @@ -27,10 +27,6 @@ jobs: HF_HOME: $(Pipeline.Workspace)/.cache/huggingface steps: - - script: docker system df && docker system prune -a -f && docker system df - displayName: Clean docker images - continueOnError: true - - template: build-docker-image-template.yaml parameters: python_version: ${{ parameters.python_version }} diff --git a/.azure_pipelines/job_templates/olive-test-win-template.yaml b/.azure_pipelines/job_templates/olive-test-win-template.yaml deleted file mode 100644 index 86137ab5e..000000000 --- a/.azure_pipelines/job_templates/olive-test-win-template.yaml +++ /dev/null @@ -1,95 +0,0 @@ -# Example Windows test template for Olive pipeline - -parameters: - name: '' - pool: '' - test_type: '' - windows: False - device: 'cpu' - python_version: '3.10' - onnxruntime: 'onnxruntime' - torch: 'torch' - requirements_file: 'requirements-test.txt' - -jobs: -- job: ${{parameters.name}} - timeoutInMinutes: 300 - pool: - name: ${{ parameters.pool}} - variables: - WINDOWS: ${{ parameters.windows}} - testType: ${{ parameters.test_type }} - python_version: ${{ parameters.python_version }} - requirements_file: ${{ parameters.requirements_file }} - PIP_CACHE_DIR: $(Pipeline.Workspace)/.cache/pip - HF_HOME: $(Pipeline.Workspace)/.cache/huggingface - PYTEST_BASETEMP: $(Pipeline.Workspace)/.pytest_basetemp - - steps: - - script: docker system df && docker system prune -a -f && docker system df - displayName: Clean docker images - continueOnError: true - - - template: olive-setup-template.yaml - parameters: - python_version: ${{ parameters.python_version }} - onnxruntime: ${{ parameters.onnxruntime }} - torch: ${{ parameters.torch }} - - - ${{ if and(eq(variables.WINDOWS, 'True'), eq(variables.testType, 'multiple_ep')) }}: - - script: | - call python -m pip install pytest - call curl --output openvino_toolkit.zip https://storage.openvinotoolkit.org/repositories/openvino/packages/2023.0.1/windows/w_openvino_toolkit_windows_2023.0.1.11005.fa1c41994f3_x86_64.zip - call 7z x openvino_toolkit.zip - call w_openvino_toolkit_windows_2023.0.1.11005.fa1c41994f3_x86_64\\setupvars.bat - call python -m pip install numpy psutil coverage protobuf==3.20.3 - call coverage run --source=$(Build.SourcesDirectory)/olive -m pytest -v -s --log-cli-level=WARNING --junitxml=$(Build.SourcesDirectory)/logs/test-TestOlive.xml $(Build.SourcesDirectory)/test/$(testType) --basetemp $(PYTEST_BASETEMP) - call coverage xml - displayName: Test Olive - env: - WORKSPACE_SUBSCRIPTION_ID: $(workspace-subscription-id) - WORKSPACE_RESOURCE_GROUP: $(workspace-resource-group) - WORKSPACE_NAME: $(workspace-name) - MANAGED_IDENTITY_CLIENT_ID: $(olive-1es-identity-client-id) - - ${{ else }}: - - script: | - python -m pip install pytest - python -m pip install -r $(Build.SourcesDirectory)/test/$(requirements_file) - - coverage run --source=$(Build.SourcesDirectory)/olive -m pytest -v -s --log-cli-level=WARNING --junitxml=$(Build.SourcesDirectory)/logs/test-TestOlive.xml $(Build.SourcesDirectory)/test/$(testType) --basetemp $(PYTEST_BASETEMP) - coverage xml - displayName: Test Olive - env: - WORKSPACE_SUBSCRIPTION_ID: $(workspace-subscription-id) - WORKSPACE_RESOURCE_GROUP: $(workspace-resource-group) - WORKSPACE_NAME: $(workspace-name) - MANAGED_IDENTITY_CLIENT_ID: $(olive-1es-identity-client-id) - - - task: CredScan@3 - displayName: 'Run CredScan' - inputs: - debugMode: false - continueOnError: true - - - task: PublishTestResults@2 - condition: succeededOrFailed() - inputs: - testResultsFiles: '**/*TestOlive*.xml' - testRunTitle: '$(Build.BuildNumber)[$(Agent.JobName)]' - failTaskOnFailedTests: true - displayName: Upload pipeline run test results - - # Code coverage requires - - task: UseDotNet@2 - displayName: 'Use .NET Core sdk 7.0.x' - inputs: - version: 7.0.x - - - task: PublishCodeCoverageResults@2 - inputs: - summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/coverage.xml' - displayName: Publish code coverage results - - - script: git clean -dfX - condition: always() - displayName: Clean remaining artifacts diff --git a/.azure_pipelines/olive-ci.yaml b/.azure_pipelines/olive-ci.yaml index 913a4bac4..b0bd1e739 100644 --- a/.azure_pipelines/olive-ci.yaml +++ b/.azure_pipelines/olive-ci.yaml @@ -51,7 +51,7 @@ variables: jobs: # Linux unit tests -- template: job_templates/olive-test-linux-cpu-template.yaml +- template: job_templates/olive-test-cpu-template.yaml parameters: name: Linux_CPU_CI_Unit_Test pool: $(OLIVE_POOL_UBUNTU2004) @@ -69,7 +69,7 @@ jobs: requirements_file: 'requirements-test-gpu.txt' # Windows unit tests -- template: job_templates/olive-test-win-template.yaml +- template: job_templates/olive-test-cpu-template.yaml parameters: name: Windows_CPU_CI_Unit_Test pool: $(OLIVE_POOL_WIN2019) @@ -137,13 +137,13 @@ jobs: # these jobs need secrets not available in forks - ${{ if ne(variables['System.PullRequest.IsFork'], 'True') }}: # integration tests - - template: job_templates/olive-test-linux-cpu-template.yaml + - template: job_templates/olive-test-cpu-template.yaml parameters: name: Linux_CPU_CI_Integration_Test pool: $(OLIVE_POOL_UBUNTU2004) test_type: 'integ_test' - - template: job_templates/olive-test-win-template.yaml + - template: job_templates/olive-test-cpu-template.yaml parameters: name: Windows_CPU_CI_Integration_Test pool: $(OLIVE_POOL_WIN2019) @@ -151,7 +151,7 @@ jobs: windows: True # Multiple EP Linux testing - - template: job_templates/olive-test-linux-cpu-template.yaml + - template: job_templates/olive-test-cpu-template.yaml parameters: name: Linux_CI_Multiple_EP_Test pool: $(OLIVE_POOL_UBUNTU2004) diff --git a/.azure_pipelines/olive-examples.yaml b/.azure_pipelines/olive-examples.yaml index c33b6f493..41c428acf 100644 --- a/.azure_pipelines/olive-examples.yaml +++ b/.azure_pipelines/olive-examples.yaml @@ -33,6 +33,14 @@ jobs: super_resolution: exampleFolder: super_resolution exampleName: super_resolution + +- template: job_templates/olive-example-linux-template.yaml + parameters: + name: Linux_Examples_Py38 + pool: $(OLIVE_POOL_UBUNTU2004) + onnxruntime: onnxruntime + python_version: '3.8' + examples: mobilenet_qnn_toolkit: exampleFolder: mobilenet exampleName: qnn_tooklit diff --git a/.azure_pipelines/olive-ort-nightly.yaml b/.azure_pipelines/olive-ort-nightly.yaml index 11fcfa968..72c215861 100644 --- a/.azure_pipelines/olive-ort-nightly.yaml +++ b/.azure_pipelines/olive-ort-nightly.yaml @@ -38,14 +38,15 @@ jobs: requirements_file: 'requirements-test-gpu.txt' # Windows unit test -- template: job_templates/olive-test-win-template.yaml +- template: job_templates/olive-test-cpu-template.yaml parameters: name: Windows_CI_Unit_Test_Olive pool: $(OLIVE_POOL_WIN2019) device: 'cpu' windows: True test_type: 'unit_test' - onnxruntime: ort-nightly + onnxruntime: onnxruntime-gpu + onnxruntime_nightly: true # Linux examples test - template: job_templates/olive-example-linux-template.yaml diff --git a/examples/test/local/test_qnn_tooklit.py b/examples/test/local/test_qnn_tooklit.py index 093715e5c..8b895df66 100644 --- a/examples/test/local/test_qnn_tooklit.py +++ b/examples/test/local/test_qnn_tooklit.py @@ -17,7 +17,6 @@ set_verbosity_debug() -@pytest.mark.skip(reason="Disable failing tests") class TestQnnToolkit: @pytest.fixture(autouse=True) def setup(self, tmp_path): diff --git a/test/requirements-test.txt b/test/requirements-test.txt index 812a60abb..f2d073aef 100644 --- a/test/requirements-test.txt +++ b/test/requirements-test.txt @@ -43,4 +43,6 @@ pytorch_lightning sentencepiece tabulate torchvision -transformers>=4.42.0 +# num_logits_to_keep is causing extra input. +# TODO(anyone): Remove this once the issue is resolved +transformers>=4.42.0, <4.45.0 diff --git a/test/unit_test/passes/onnx/test_split_model.py b/test/unit_test/passes/onnx/test_split_model.py index 0d13ef291..d38ce4946 100644 --- a/test/unit_test/passes/onnx/test_split_model.py +++ b/test/unit_test/passes/onnx/test_split_model.py @@ -80,8 +80,6 @@ def input_model_info_fixture(request, tmp_path_factory): return all_models, request.param, 4 if request.param else 2 -# TODO(team): 55319 Failed on CI. Need to investigate. -@pytest.mark.skipif(True, reason="Fails on CI. Need to investigate.") @pytest.mark.parametrize( "model_type", ["convert_fp32", "opt_fp32", "opt_fp16"],