From c244569730d856fcd588111a2492b8e7c5e034d0 Mon Sep 17 00:00:00 2001 From: hallvictoria <59299039+hallvictoria@users.noreply.github.com> Date: Fri, 14 Jun 2024 12:27:17 -0500 Subject: [PATCH] build: update to 1ES PT (#1503) * code mirror template for worker * converting to 1ES pipeline * basic 1ES pipeline * basic format changed 1ES pipeline * same template as openai pipeline * official build template * official, public, e2e builds * yml formatting * tests typo * formatting & feedback * removed nuget internal reference * removed official folder * feedback, updating public build * updating build cmd and dotnet version --------- Co-authored-by: Victoria Hall Co-authored-by: Manvir Kaur <67894494+manvkaur@users.noreply.github.com> --- .github/workflows/ci_e2e_workflow.yml | 6 +- eng/ci/code-mirror.yml | 18 ++ eng/ci/official-build.yml | 50 ++++++ eng/ci/public-build.yml | 38 +++++ eng/templates/jobs/build.yml | 22 +++ .../official/jobs/build-artifacts.yml | 80 +++++++++ eng/templates/official/jobs/ci-e2e-tests.yml | 155 ++++++++++++++++++ 7 files changed, 366 insertions(+), 3 deletions(-) create mode 100644 eng/ci/code-mirror.yml create mode 100644 eng/ci/official-build.yml create mode 100644 eng/ci/public-build.yml create mode 100644 eng/templates/jobs/build.yml create mode 100644 eng/templates/official/jobs/build-artifacts.yml create mode 100644 eng/templates/official/jobs/ci-e2e-tests.yml diff --git a/.github/workflows/ci_e2e_workflow.yml b/.github/workflows/ci_e2e_workflow.yml index 6b4d6979..0d97b962 100644 --- a/.github/workflows/ci_e2e_workflow.yml +++ b/.github/workflows/ci_e2e_workflow.yml @@ -64,7 +64,7 @@ jobs: python -m pip install --upgrade pip python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple -U azure-functions --pre python -m pip install -U -e .[dev] - + if [[ "${{ matrix.python-version }}" != "3.7" ]]; then python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple --pre -U -e .[test-http-v2] fi @@ -79,7 +79,7 @@ jobs: mkdir logs - name: Grant execute permission run: | - chmod +x .github/Scripts/${{ matrix.test-type }}.sh + chmod +x .github/Scripts/${{ matrix.test-type }}.sh - name: Running 3.7 ${{ matrix.test-type }} if: matrix.python-version == 3.7 env: @@ -158,4 +158,4 @@ jobs: with: name: Test WebHost Logs ${{ github.run_id }} ${{ matrix.python-version }} path: logs/*.log - if-no-files-found: ignore + if-no-files-found: ignore \ No newline at end of file diff --git a/eng/ci/code-mirror.yml b/eng/ci/code-mirror.yml new file mode 100644 index 00000000..04c800ea --- /dev/null +++ b/eng/ci/code-mirror.yml @@ -0,0 +1,18 @@ +trigger: + branches: + include: + - dev + - release/* + +resources: + repositories: + - repository: eng + type: git + name: engineering + ref: refs/tags/release + +variables: + - template: ci/variables/cfs.yml@eng + +extends: + template: ci/code-mirror.yml@eng \ No newline at end of file diff --git a/eng/ci/official-build.yml b/eng/ci/official-build.yml new file mode 100644 index 00000000..bb6afa21 --- /dev/null +++ b/eng/ci/official-build.yml @@ -0,0 +1,50 @@ +trigger: + batch: true + branches: + include: + - dev + - release/* + +# CI only, does not trigger on PRs. +pr: none + +schedules: + - cron: '0 0 * * MON' + displayName: At 12:00 AM, only on Monday + branches: + include: + - dev + always: true + +resources: + repositories: + - repository: 1es + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + - repository: eng + type: git + name: engineering + ref: refs/tags/release + +variables: + - template: ci/variables/build.yml@eng + - template: ci/variables/cfs.yml@eng + +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1es + parameters: + pool: + name: 1es-pool-azfunc + image: 1es-windows-2022 + os: windows + + stages: + - stage: Build + jobs: + - template: /eng/ci/templates/official/jobs/build-artifacts.yml@self + + - stage: RunTests + dependsOn: Build + jobs: + - template: /eng/ci/templates/official/jobs/ci-e2e-tests.yml@self \ No newline at end of file diff --git a/eng/ci/public-build.yml b/eng/ci/public-build.yml new file mode 100644 index 00000000..929634b6 --- /dev/null +++ b/eng/ci/public-build.yml @@ -0,0 +1,38 @@ +trigger: + batch: true + branches: + include: + - dev + +pr: + branches: + include: + - dev + +schedules: + - cron: '0 0 * * MON' + displayName: At 12:00 AM, only on Monday + branches: + include: + - dev + always: true + +resources: + repositories: + - repository: 1es + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + +extends: + template: v1/1ES.Unofficial.PipelineTemplate.yml@1es + parameters: + pool: + name: 1es-pool-azfunc-public + image: 1es-windows-2022 + os: windows + + stages: + - stage: Build + jobs: + - template: /eng/ci/templates/jobs/build.yml@self \ No newline at end of file diff --git a/eng/templates/jobs/build.yml b/eng/templates/jobs/build.yml new file mode 100644 index 00000000..dadd88fb --- /dev/null +++ b/eng/templates/jobs/build.yml @@ -0,0 +1,22 @@ +jobs: + - job: "Build" + displayName: 'Build python worker' + + pool: + name: 1es-pool-azfunc-public + image: 1es-windows-2022 + os: windows + + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: "3.11" + - bash: | + python --version + displayName: 'Check python version' + - bash: | + python -m venv .env + .env\Scripts\Activate.ps1 + python -m pip install --upgrade pip==23.0 + python -m pip install . + displayName: 'Build python worker' \ No newline at end of file diff --git a/eng/templates/official/jobs/build-artifacts.yml b/eng/templates/official/jobs/build-artifacts.yml new file mode 100644 index 00000000..0d9c4164 --- /dev/null +++ b/eng/templates/official/jobs/build-artifacts.yml @@ -0,0 +1,80 @@ +jobs: + - job: "Build" + displayName: "Build python worker" + + templateContext: + outputParentDirectory: $(Build.ArtifactStagingDirectory) + outputs: + - output: pipelineArtifact + targetPath: $(Build.ArtifactStagingDirectory) + artifactName: "drop" + - output: nuget + condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/dev'), eq(variables['UPLOADPACKAGETOPRERELEASEFEED'], true)) + useDotNetTask: false + packagesToPush: "$(Build.ArtifactStagingDirectory)/**/*.nupkg;!$(Build.ArtifactStagingDirectory)/**/*.symbols.nupkg" + packageParentPath: "$(Build.ArtifactStagingDirectory)" + publishVstsFeed: "e6a70c92-4128-439f-8012-382fe78d6396/f37f760c-aebd-443e-9714-ce725cd427df" + nuGetFeedType: "internal" + allowPackageConflicts: true + + pool: + name: 1es-pool-azfunc + image: 1es-windows-2022 + os: windows + + variables: + ${{ if contains(variables['Build.SourceBranch'], '/tags/' ) }}: + isTagTemp: true + isTag: $[variables.isTagTemp] + + strategy: + matrix: + Python37V4: + pythonVersion: "3.7" + workerPath: $(PROD_V4_WORKER_PY) + Python38V4: + pythonVersion: "3.8" + workerPath: $(PROD_V4_WORKER_PY) + Python39V4: + pythonVersion: "3.9" + workerPath: $(PROD_V4_WORKER_PY) + Python310V4: + pythonVersion: "3.10" + workerPath: $(PROD_V4_WORKER_PY) + Python311V4: + pythonVersion: "3.11" + workerPath: $(PROD_V4_WORKER_PY) + + steps: + - template: pack/templates/win_env_gen.yml + displayName: "Build Windows x64" + parameters: + pythonVersion: "$(pythonVersion)" + workerPath: "$(workerPath)" + architecture: "x64" + artifactName: "$(pythonVersion)_WINDOWS_X64" + - template: pack/templates/win_env_gen.yml + displayName: "Build Windows x86" + parameters: + pythonVersion: "$(pythonVersion)" + workerPath: "$(workerPath)" + architecture: "x86" + artifactName: "$(pythonVersion)_WINDOWS_x86" + - template: pack/templates/nix_env_gen.yml + displayName: "Build Linux x64" + parameters: + pythonVersion: "$(pythonVersion)" + workerPath: "$(workerPath)" + artifactName: "$(pythonVersion)_LINUX_X64" + - template: pack/templates/nix_env_gen.yml + displayName: "Build OSX X64" + parameters: + pythonVersion: "$(pythonVersion)" + workerPath: "$(workerPath)" + artifactName: "$(pythonVersion)_OSX_X64" + - template: pack/templates/macos_64_env_gen.yml + displayName: "Build OSX ARM64" + parameters: + pythonVersion: "$(pythonVersion)" + workerPath: "$(workerPath)" + artifactName: "$(pythonVersion)_OSX_ARM64" \ No newline at end of file diff --git a/eng/templates/official/jobs/ci-e2e-tests.yml b/eng/templates/official/jobs/ci-e2e-tests.yml new file mode 100644 index 00000000..3ebdaca6 --- /dev/null +++ b/eng/templates/official/jobs/ci-e2e-tests.yml @@ -0,0 +1,155 @@ +jobs: + - job: "TestPython" + displayName: "Run Python E2E Tests" + + pool: + name: 1es-pool-azfunc + image: 1es-ubuntu-22.04 + os: linux + + strategy: + matrix: + python-37-fwpc: + PYTHON_VERSION: '3.7' + TEST_TYPE: 'fwpc-e2e-tests' + python-37-e2e: + PYTHON_VERSION: '3.7' + TEST_TYPE: 'e2e-tests' + python-37-deferred-bindings: + PYTHON_VERSION: '3.7' + TEST_TYPE: 'deferred-bindings-e2e-tests' + python-38-fwpc: + PYTHON_VERSION: '3.8' + TEST_TYPE: 'fwpc-e2e-tests' + python-38-e2e: + PYTHON_VERSION: '3.8' + TEST_TYPE: 'e2e-tests' + python-38-deferred-bindings: + PYTHON_VERSION: '3.8' + TEST_TYPE: 'deferred-bindings-e2e-tests' + python-39-fwpc: + PYTHON_VERSION: '3.9' + TEST_TYPE: 'fwpc-e2e-tests' + python-39-e2e: + PYTHON_VERSION: '3.9' + TEST_TYPE: 'e2e-tests' + python-39-deferred-bindings: + PYTHON_VERSION: '3.9' + TEST_TYPE: 'deferred-bindings-e2e-tests' + python-310-fwpc: + PYTHON_VERSION: '3.10' + TEST_TYPE: 'fwpc-e2e-tests' + python-310-e2e: + PYTHON_VERSION: '3.10' + TEST_TYPE: 'e2e-tests' + python-310-deferred-bindings: + PYTHON_VERSION: '3.10' + TEST_TYPE: 'deferred-bindings-e2e-tests' + python-311-fwpc: + PYTHON_VERSION: '3.11' + TEST_TYPE: 'fwpc-e2e-tests' + python-311-e2e: + PYTHON_VERSION: '3.11' + TEST_TYPE: 'e2e-tests' + python-311-deferred-bindings: + PYTHON_VERSION: '3.11' + TEST_TYPE: 'deferred-bindings-e2e-tests' + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: $(PYTHON_VERSION) + - task: UseDotNet@2 + displayName: 'Install .NET 8' + inputs: + version: 8.0.x + - bash: | + python -m pip install --upgrade pip + python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple -U azure-functions --pre + python -m pip install -U -e .[dev] + + if [[ "${{ PYTHON_VERSION }}" != "3.7" ]]; then + python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple --pre -U -e .[test-http-v2] + fi + if [[ "${{ PYTHON_VERSION }}" != "3.7" && "${{ PYTHON_VERSION }}" != "3.8" ]]; then + python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple --pre -U -e .[test-deferred-bindings] + fi + + # Retry a couple times to avoid certificate issue + retry 5 python setup.py build + retry 5 python setup.py webhost --branch-name=dev + retry 5 python setup.py extension + mkdir logs + displayName: 'Install dependencies and the worker' + - bash: | + chmod +x .github/Scripts/${{ TEST_TYPE }}.sh + displayName: 'Grant execute permission' + - bash: | + if [[ "${{ PYTHON_VERSION }}" == "3.7" ]]; then + .github/Scripts/${{ TEST_TYPE }}.sh + fi + env: + AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString37 }} + AzureWebJobsCosmosDBConnectionString: ${{ secrets.LinuxCosmosDBConnectionString37 }} + AzureWebJobsEventHubConnectionString: ${{ secrets.LinuxEventHubConnectionString37 }} + AzureWebJobsServiceBusConnectionString: ${{ secrets.LinuxServiceBusConnectionString37 }} + AzureWebJobsSqlConnectionString: ${{ secrets.LinuxSqlConnectionString37 }} + AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString37 }} + AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString37 }} + ARCHIVE_WEBHOST_LOGS: ${{ github.event.inputs.archive_webhost_logging }} + displayName: "Running 3.7 $(TEST_TYPE) tests" + - bash: | + if [[ "${{ PYTHON_VERSION }}" == "3.8" ]]; then + .github/Scripts/${{ TEST_TYPE }}.sh + fi + env: + AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString38 }} + AzureWebJobsCosmosDBConnectionString: ${{ secrets.LinuxCosmosDBConnectionString38 }} + AzureWebJobsEventHubConnectionString: ${{ secrets.LinuxEventHubConnectionString38 }} + AzureWebJobsServiceBusConnectionString: ${{ secrets.LinuxServiceBusConnectionString38 }} + AzureWebJobsSqlConnectionString: ${{ secrets.LinuxSqlConnectionString38 }} + AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString38 }} + AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString38 }} + ARCHIVE_WEBHOST_LOGS: ${{ github.event.inputs.archive_webhost_logging }} + displayName: "Running 3.8 $(TEST_TYPE) tests" + - bash: | + if [[ "${{ PYTHON_VERSION }}" == "3.9" ]]; then + .github/Scripts/${{ TEST_TYPE }}.sh + fi + env: + AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString39 }} + AzureWebJobsCosmosDBConnectionString: ${{ secrets.LinuxCosmosDBConnectionString39 }} + AzureWebJobsEventHubConnectionString: ${{ secrets.LinuxEventHubConnectionString39 }} + AzureWebJobsServiceBusConnectionString: ${{ secrets.LinuxServiceBusConnectionString39 }} + AzureWebJobsSqlConnectionString: ${{ secrets.LinuxSqlConnectionString39 }} + AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString39 }} + AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString39 }} + ARCHIVE_WEBHOST_LOGS: ${{ github.event.inputs.archive_webhost_logging }} + displayName: "Running 3.9 $(TEST_TYPE) tests" + - bash: | + if [[ "${{ PYTHON_VERSION }}" == "3.10" ]]; then + .github/Scripts/${{ TEST_TYPE }}.sh + fi + env: + AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString310 }} + AzureWebJobsCosmosDBConnectionString: ${{ secrets.LinuxCosmosDBConnectionString310 }} + AzureWebJobsEventHubConnectionString: ${{ secrets.LinuxEventHubConnectionString310 }} + AzureWebJobsServiceBusConnectionString: ${{ secrets.LinuxServiceBusConnectionString310 }} + AzureWebJobsSqlConnectionString: ${{ secrets.LinuxSqlConnectionString310 }} + AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString310 }} + AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString310 }} + ARCHIVE_WEBHOST_LOGS: ${{ github.event.inputs.archive_webhost_logging }} + displayName: "Running 3.10 $(TEST_TYPE) tests" + - bash: | + if [[ "${{ PYTHON_VERSION }}" == "3.11" ]]; then + .github/Scripts/${{ TEST_TYPE }}.sh + fi + env: + AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString311 }} + AzureWebJobsCosmosDBConnectionString: ${{ secrets.LinuxCosmosDBConnectionString311 }} + AzureWebJobsEventHubConnectionString: ${{ secrets.LinuxEventHubConnectionString311 }} + AzureWebJobsServiceBusConnectionString: ${{ secrets.LinuxServiceBusConnectionString311 }} + AzureWebJobsSqlConnectionString: ${{ secrets.LinuxSqlConnectionString311 }} + AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString311 }} + AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString311 }} + ARCHIVE_WEBHOST_LOGS: ${{ github.event.inputs.archive_webhost_logging }} + displayName: "Running 3.11 $(TEST_TYPE) tests"