build: update to 1ES PT (#1503)

* code mirror template for worker * converting to 1ES pipeline * basic 1ES pipeline * basic format changed 1ES pipeline * same template as openai pipeline * official build template * official, public, e2e builds * yml formatting * tests typo * formatting & feedback * removed nuget internal reference * removed official folder * feedback, updating public build * updating build cmd and dotnet version --------- Co-authored-by: Victoria Hall <victoria.hall@microsoft.com> Co-authored-by: Manvir Kaur <67894494+manvkaur@users.noreply.github.com>
Azure · Jun 14, 2024 · c244569 · c244569
1 parent 255fc24
commit c244569
Show file tree

Hide file tree

Showing 7 changed files with 366 additions and 3 deletions.
diff --git a/.github/workflows/ci_e2e_workflow.yml b/.github/workflows/ci_e2e_workflow.yml
@@ -64,7 +64,7 @@ jobs:
           python -m pip install --upgrade pip
           python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple -U azure-functions --pre
           python -m pip install -U -e .[dev]
- 
+
           if [[ "${{ matrix.python-version }}" != "3.7" ]]; then
              python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple --pre -U -e .[test-http-v2]
           fi
@@ -79,7 +79,7 @@ jobs:
           mkdir logs
       - name: Grant execute permission
         run: |
-            chmod +x .github/Scripts/${{ matrix.test-type }}.sh
+          chmod +x .github/Scripts/${{ matrix.test-type }}.sh
       - name: Running 3.7 ${{ matrix.test-type }}
         if: matrix.python-version == 3.7
         env:
@@ -158,4 +158,4 @@ jobs:
         with:
           name: Test WebHost Logs ${{ github.run_id }} ${{ matrix.python-version }}
           path: logs/*.log
-          if-no-files-found: ignore
+          if-no-files-found: ignore
diff --git a/eng/ci/code-mirror.yml b/eng/ci/code-mirror.yml
@@ -0,0 +1,18 @@
+trigger:
+  branches:
+    include:
+      - dev
+      - release/*
+
+resources:
+  repositories:
+    - repository: eng
+      type: git
+      name: engineering
+      ref: refs/tags/release
+
+variables:
+  - template: ci/variables/cfs.yml@eng
+
+extends:
+  template: ci/code-mirror.yml@eng
diff --git a/eng/ci/official-build.yml b/eng/ci/official-build.yml
@@ -0,0 +1,50 @@
+trigger:
+  batch: true
+  branches:
+    include:
+      - dev
+      - release/*
+
+# CI only, does not trigger on PRs.
+pr: none
+
+schedules:
+  - cron: '0 0 * * MON'
+    displayName: At 12:00 AM, only on Monday
+    branches:
+      include:
+        - dev
+    always: true
+
+resources:
+  repositories:
+    - repository: 1es
+      type: git
+      name: 1ESPipelineTemplates/1ESPipelineTemplates
+      ref: refs/tags/release
+    - repository: eng
+      type: git
+      name: engineering
+      ref: refs/tags/release
+
+variables:
+  - template: ci/variables/build.yml@eng
+  - template: ci/variables/cfs.yml@eng
+
+extends:
+  template: v1/1ES.Official.PipelineTemplate.yml@1es
+  parameters:
+    pool:
+      name: 1es-pool-azfunc
+      image: 1es-windows-2022
+      os: windows
+
+    stages:
+      - stage: Build
+        jobs:
+          - template: /eng/ci/templates/official/jobs/build-artifacts.yml@self
+
+      - stage: RunTests
+        dependsOn: Build
+        jobs:
+          - template: /eng/ci/templates/official/jobs/ci-e2e-tests.yml@self
diff --git a/eng/ci/public-build.yml b/eng/ci/public-build.yml
@@ -0,0 +1,38 @@
+trigger:
+  batch: true
+  branches:
+    include:
+      - dev
+
+pr:
+  branches:
+    include:
+      - dev
+
+schedules:
+  - cron: '0 0 * * MON'
+    displayName: At 12:00 AM, only on Monday
+    branches:
+      include:
+        - dev
+    always: true
+
+resources:
+  repositories:
+    - repository: 1es
+      type: git
+      name: 1ESPipelineTemplates/1ESPipelineTemplates
+      ref: refs/tags/release
+
+extends:
+  template: v1/1ES.Unofficial.PipelineTemplate.yml@1es
+  parameters:
+    pool:
+      name: 1es-pool-azfunc-public
+      image: 1es-windows-2022
+      os: windows
+
+    stages:
+      - stage: Build
+        jobs:
+          - template: /eng/ci/templates/jobs/build.yml@self
diff --git a/eng/templates/jobs/build.yml b/eng/templates/jobs/build.yml
@@ -0,0 +1,22 @@
+jobs:
+  - job: "Build"
+    displayName: 'Build python worker'
+
+    pool:
+      name: 1es-pool-azfunc-public
+      image: 1es-windows-2022
+      os: windows
+
+    steps:
+      - task: UsePythonVersion@0
+        inputs:
+          versionSpec: "3.11"
+      - bash: |
+          python --version
+        displayName: 'Check python version'
+      - bash: |
+          python -m venv .env
+          .env\Scripts\Activate.ps1
+          python -m pip install --upgrade pip==23.0
+          python -m pip install .
+        displayName: 'Build python worker'
diff --git a/eng/templates/official/jobs/build-artifacts.yml b/eng/templates/official/jobs/build-artifacts.yml
@@ -0,0 +1,80 @@
+jobs:
+  - job: "Build"
+    displayName: "Build python worker"
+
+    templateContext:
+      outputParentDirectory: $(Build.ArtifactStagingDirectory)
+      outputs:
+        - output: pipelineArtifact
+          targetPath: $(Build.ArtifactStagingDirectory)
+          artifactName: "drop"
+        - output: nuget
+          condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/dev'), eq(variables['UPLOADPACKAGETOPRERELEASEFEED'], true))
+          useDotNetTask: false
+          packagesToPush: "$(Build.ArtifactStagingDirectory)/**/*.nupkg;!$(Build.ArtifactStagingDirectory)/**/*.symbols.nupkg"
+          packageParentPath: "$(Build.ArtifactStagingDirectory)"
+          publishVstsFeed: "e6a70c92-4128-439f-8012-382fe78d6396/f37f760c-aebd-443e-9714-ce725cd427df"
+          nuGetFeedType: "internal"
+          allowPackageConflicts: true
+
+    pool:
+      name: 1es-pool-azfunc
+      image: 1es-windows-2022
+      os: windows
+
+    variables:
+      ${{ if contains(variables['Build.SourceBranch'], '/tags/' ) }}:
+        isTagTemp: true
+      isTag: $[variables.isTagTemp]
+
+    strategy:
+      matrix:
+        Python37V4:
+          pythonVersion: "3.7"
+          workerPath: $(PROD_V4_WORKER_PY)
+        Python38V4:
+          pythonVersion: "3.8"
+          workerPath: $(PROD_V4_WORKER_PY)
+        Python39V4:
+          pythonVersion: "3.9"
+          workerPath: $(PROD_V4_WORKER_PY)
+        Python310V4:
+          pythonVersion: "3.10"
+          workerPath: $(PROD_V4_WORKER_PY)
+        Python311V4:
+          pythonVersion: "3.11"
+          workerPath: $(PROD_V4_WORKER_PY)
+
+    steps:
+      - template: pack/templates/win_env_gen.yml
+        displayName: "Build Windows x64"
+        parameters:
+          pythonVersion: "$(pythonVersion)"
+          workerPath: "$(workerPath)"
+          architecture: "x64"
+          artifactName: "$(pythonVersion)_WINDOWS_X64"
+      - template: pack/templates/win_env_gen.yml
+        displayName: "Build Windows x86"
+        parameters:
+          pythonVersion: "$(pythonVersion)"
+          workerPath: "$(workerPath)"
+          architecture: "x86"
+          artifactName: "$(pythonVersion)_WINDOWS_x86"
+      - template: pack/templates/nix_env_gen.yml
+        displayName: "Build Linux x64"
+        parameters:
+          pythonVersion: "$(pythonVersion)"
+          workerPath: "$(workerPath)"
+          artifactName: "$(pythonVersion)_LINUX_X64"
+      - template: pack/templates/nix_env_gen.yml
+        displayName: "Build OSX X64"
+        parameters:
+          pythonVersion: "$(pythonVersion)"
+          workerPath: "$(workerPath)"
+          artifactName: "$(pythonVersion)_OSX_X64"
+      - template: pack/templates/macos_64_env_gen.yml
+        displayName: "Build OSX ARM64"
+        parameters:
+          pythonVersion: "$(pythonVersion)"
+          workerPath: "$(workerPath)"
+          artifactName: "$(pythonVersion)_OSX_ARM64"
diff --git a/eng/templates/official/jobs/ci-e2e-tests.yml b/eng/templates/official/jobs/ci-e2e-tests.yml
@@ -0,0 +1,155 @@
+jobs:
+  - job: "TestPython"
+    displayName: "Run Python E2E Tests"
+
+    pool:
+      name: 1es-pool-azfunc
+      image: 1es-ubuntu-22.04
+      os: linux
+
+    strategy:
+      matrix:
+        python-37-fwpc:
+          PYTHON_VERSION: '3.7'
+          TEST_TYPE: 'fwpc-e2e-tests'
+        python-37-e2e:
+          PYTHON_VERSION: '3.7'
+          TEST_TYPE: 'e2e-tests'
+        python-37-deferred-bindings:
+          PYTHON_VERSION: '3.7'
+          TEST_TYPE: 'deferred-bindings-e2e-tests'
+        python-38-fwpc:
+          PYTHON_VERSION: '3.8'
+          TEST_TYPE: 'fwpc-e2e-tests'
+        python-38-e2e:
+          PYTHON_VERSION: '3.8'
+          TEST_TYPE: 'e2e-tests'
+        python-38-deferred-bindings:
+          PYTHON_VERSION: '3.8'
+          TEST_TYPE: 'deferred-bindings-e2e-tests'
+        python-39-fwpc:
+          PYTHON_VERSION: '3.9'
+          TEST_TYPE: 'fwpc-e2e-tests'
+        python-39-e2e:
+          PYTHON_VERSION: '3.9'
+          TEST_TYPE: 'e2e-tests'
+        python-39-deferred-bindings:
+          PYTHON_VERSION: '3.9'
+          TEST_TYPE: 'deferred-bindings-e2e-tests'
+        python-310-fwpc:
+          PYTHON_VERSION: '3.10'
+          TEST_TYPE: 'fwpc-e2e-tests'
+        python-310-e2e:
+          PYTHON_VERSION: '3.10'
+          TEST_TYPE: 'e2e-tests'
+        python-310-deferred-bindings:
+          PYTHON_VERSION: '3.10'
+          TEST_TYPE: 'deferred-bindings-e2e-tests'
+        python-311-fwpc:
+          PYTHON_VERSION: '3.11'
+          TEST_TYPE: 'fwpc-e2e-tests'
+        python-311-e2e:
+          PYTHON_VERSION: '3.11'
+          TEST_TYPE: 'e2e-tests'
+        python-311-deferred-bindings:
+          PYTHON_VERSION: '3.11'
+          TEST_TYPE: 'deferred-bindings-e2e-tests'
+    steps:
+      - task: UsePythonVersion@0
+        inputs:
+          versionSpec: $(PYTHON_VERSION)
+      - task: UseDotNet@2
+        displayName: 'Install .NET 8'
+        inputs:
+          version: 8.0.x
+      - bash: |
+          python -m pip install --upgrade pip
+          python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple -U azure-functions --pre
+          python -m pip install -U -e .[dev]
+
+          if [[ "${{ PYTHON_VERSION }}" != "3.7" ]]; then
+             python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple --pre -U -e .[test-http-v2]
+          fi
+          if [[ "${{ PYTHON_VERSION }}" != "3.7" && "${{ PYTHON_VERSION }}" != "3.8" ]]; then
+              python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple --pre -U -e .[test-deferred-bindings]
+          fi
+
+          # Retry a couple times to avoid certificate issue
+          retry 5 python setup.py build
+          retry 5 python setup.py webhost --branch-name=dev
+          retry 5 python setup.py extension
+          mkdir logs
+        displayName: 'Install dependencies and the worker'
+      - bash: |
+          chmod +x .github/Scripts/${{ TEST_TYPE }}.sh
+        displayName: 'Grant execute permission'
+      - bash: |
+          if [[ "${{ PYTHON_VERSION }}" == "3.7" ]]; then
+             .github/Scripts/${{ TEST_TYPE }}.sh
+          fi
+        env:
+          AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString37 }}
+          AzureWebJobsCosmosDBConnectionString: ${{ secrets.LinuxCosmosDBConnectionString37 }}
+          AzureWebJobsEventHubConnectionString: ${{ secrets.LinuxEventHubConnectionString37 }}
+          AzureWebJobsServiceBusConnectionString: ${{ secrets.LinuxServiceBusConnectionString37 }}
+          AzureWebJobsSqlConnectionString: ${{ secrets.LinuxSqlConnectionString37 }}
+          AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString37 }}
+          AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString37 }}
+          ARCHIVE_WEBHOST_LOGS: ${{ github.event.inputs.archive_webhost_logging }}
+        displayName: "Running 3.7 $(TEST_TYPE) tests"
+      - bash: |
+          if [[ "${{ PYTHON_VERSION }}" == "3.8" ]]; then
+             .github/Scripts/${{ TEST_TYPE }}.sh
+          fi
+        env:
+          AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString38 }}
+          AzureWebJobsCosmosDBConnectionString: ${{ secrets.LinuxCosmosDBConnectionString38 }}
+          AzureWebJobsEventHubConnectionString: ${{ secrets.LinuxEventHubConnectionString38 }}
+          AzureWebJobsServiceBusConnectionString: ${{ secrets.LinuxServiceBusConnectionString38 }}
+          AzureWebJobsSqlConnectionString: ${{ secrets.LinuxSqlConnectionString38 }}
+          AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString38 }}
+          AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString38 }}
+          ARCHIVE_WEBHOST_LOGS: ${{ github.event.inputs.archive_webhost_logging }}
+        displayName: "Running 3.8 $(TEST_TYPE) tests"
+      - bash: |
+          if [[ "${{ PYTHON_VERSION }}" == "3.9" ]]; then
+             .github/Scripts/${{ TEST_TYPE }}.sh
+          fi
+        env:
+          AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString39 }}
+          AzureWebJobsCosmosDBConnectionString: ${{ secrets.LinuxCosmosDBConnectionString39 }}
+          AzureWebJobsEventHubConnectionString: ${{ secrets.LinuxEventHubConnectionString39 }}
+          AzureWebJobsServiceBusConnectionString: ${{ secrets.LinuxServiceBusConnectionString39 }}
+          AzureWebJobsSqlConnectionString: ${{ secrets.LinuxSqlConnectionString39 }}
+          AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString39 }}
+          AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString39 }}
+          ARCHIVE_WEBHOST_LOGS: ${{ github.event.inputs.archive_webhost_logging }}
+        displayName: "Running 3.9 $(TEST_TYPE) tests"
+      - bash: |
+          if [[ "${{ PYTHON_VERSION }}" == "3.10" ]]; then
+             .github/Scripts/${{ TEST_TYPE }}.sh
+          fi
+        env:
+          AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString310 }}
+          AzureWebJobsCosmosDBConnectionString: ${{ secrets.LinuxCosmosDBConnectionString310 }}
+          AzureWebJobsEventHubConnectionString: ${{ secrets.LinuxEventHubConnectionString310 }}
+          AzureWebJobsServiceBusConnectionString: ${{ secrets.LinuxServiceBusConnectionString310 }}
+          AzureWebJobsSqlConnectionString: ${{ secrets.LinuxSqlConnectionString310 }}
+          AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString310 }}
+          AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString310 }}
+          ARCHIVE_WEBHOST_LOGS: ${{ github.event.inputs.archive_webhost_logging }}
+        displayName: "Running 3.10 $(TEST_TYPE) tests"
+      - bash: |
+          if [[ "${{ PYTHON_VERSION }}" == "3.11" ]]; then
+             .github/Scripts/${{ TEST_TYPE }}.sh
+          fi
+        env:
+          AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString311 }}
+          AzureWebJobsCosmosDBConnectionString: ${{ secrets.LinuxCosmosDBConnectionString311 }}
+          AzureWebJobsEventHubConnectionString: ${{ secrets.LinuxEventHubConnectionString311 }}
+          AzureWebJobsServiceBusConnectionString: ${{ secrets.LinuxServiceBusConnectionString311 }}
+          AzureWebJobsSqlConnectionString: ${{ secrets.LinuxSqlConnectionString311 }}
+          AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString311 }}
+          AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString311 }}
+          ARCHIVE_WEBHOST_LOGS: ${{ github.event.inputs.archive_webhost_logging }}
+        displayName: "Running 3.11 $(TEST_TYPE) tests"